From 95544925278508152c10d41a51a87959dadbed68 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:55:26 +0800 Subject: [PATCH 1/3] crop transcript in middle instead of decimate it, use \n instead of , --- .../site-adapters/youtube/index.mjs | 2 +- src/utils/crop-text.mjs | 65 ++++++++++--------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/content-script/site-adapters/youtube/index.mjs b/src/content-script/site-adapters/youtube/index.mjs index 46f1a7fc..02f91ddb 100644 --- a/src/content-script/site-adapters/youtube/index.mjs +++ b/src/content-script/site-adapters/youtube/index.mjs @@ -48,7 +48,7 @@ export default { let subtitleContent = '' while (subtitleData.indexOf('">') !== -1) { subtitleData = subtitleData.substring(subtitleData.indexOf('">') + 2) - subtitleContent += subtitleData.substring(0, subtitleData.indexOf('<')) + ',' + subtitleContent += subtitleData.substring(0, subtitleData.indexOf('<')) + '\n' } subtitleContent = replaceHtmlEntities(subtitleContent) diff --git a/src/utils/crop-text.mjs b/src/utils/crop-text.mjs index 7546a004..79f6f832 100644 --- a/src/utils/crop-text.mjs +++ b/src/utils/crop-text.mjs @@ -28,6 +28,7 @@ const clamp = (v, min, max) => { return Math.min(Math.max(v, min), max) } +/** this function will crop text by keeping the beginning and end */ export async function cropText( text, maxLength = 4000, @@ -48,47 +49,53 @@ export async function cropText( maxLength -= 100 + clamp(userConfig.maxResponseTokenLength, 1, maxLength - 1000) } - const splits = text.split(/[,，。?？!！;；]/).map((s) => s.trim()) + const splits = text.split(/[,，。?？!！;；\n]/).map((s) => s.trim()) const splitsLength = splits.map((s) => (tiktoken ? encode(s).length : s.length)) - const length = splitsLength.reduce((sum, length) => sum + length, 0) - - const cropLength = length - startLength - endLength const cropTargetLength = maxLength - startLength - endLength - const cropPercentage = cropTargetLength / cropLength - const cropStep = Math.max(0, 1 / cropPercentage - 1) - - if (cropStep === 0) return text + let firstHalfTokens = 0 + let secondHalfTokens = 0 + const halfTargetTokens = Math.floor(cropTargetLength / 2) + let middleIndex = -1 + let endStartIndex = splits.length + let totalTokens = splitsLength.reduce((sum, length) => sum + length + 1, 0) + let croppedTokens = 0 let croppedText = '' let currentLength = 0 - let currentIndex = 0 - let currentStep = 0 - for (; currentIndex < splits.length; currentIndex++) { - if (currentLength + splitsLength[currentIndex] + 1 <= startLength) { - croppedText += splits[currentIndex] + ',' - currentLength += splitsLength[currentIndex] + 1 - } else if (currentLength + splitsLength[currentIndex] + 1 + endLength <= maxLength) { - if (currentStep < cropStep) { - currentStep++ - } else { - croppedText += splits[currentIndex] + ',' - currentLength += splitsLength[currentIndex] + 1 - currentStep = currentStep - cropStep - } + // First pass: find the middle + for (let i = 0; i < splits.length; i++) { + if (firstHalfTokens < halfTargetTokens) { + firstHalfTokens += splitsLength[i] + 1 } else { + middleIndex = i + break + } + } + + // Second pass: find the start of the end section + for (let i = splits.length - 1; i >= middleIndex; i--) { + secondHalfTokens += splitsLength[i] + 1 + if (secondHalfTokens >= halfTargetTokens) { + endStartIndex = i break } } - let endPart = '' - let endPartLength = 0 - for (let i = splits.length - 1; endPartLength + splitsLength[i] <= endLength; i--) { - endPart = splits[i] + ',' + endPart - endPartLength += splitsLength[i] + 1 + // Calculate cropped tokens + croppedTokens = totalTokens - firstHalfTokens - secondHalfTokens + + // Construct the cropped text + croppedText = splits.slice(0, middleIndex).join('\n') + if (middleIndex !== endStartIndex) { + croppedText += `\n\n**Important disclaimer**, this text is incomplete! ${croppedTokens} or ${ + (croppedTokens / totalTokens).toFixed(2) * 100 + }% of tokens have been removed from this location in the text due to lack limited model context\n\n` } - currentLength += endPartLength - croppedText += endPart + croppedText += splits.slice(endStartIndex).join('\n') + + currentLength = firstHalfTokens + secondHalfTokens + (middleIndex !== endStartIndex ? 9 : 0) // 9 is the length of "\n[cropped]\n" + // ... existing code ... console.log( `input maxLength: ${maxLength}\n` + From 5a69226a11defcc63fdc21bd51997a2c3f43550d Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:26:17 +0800 Subject: [PATCH 2/3] better max Length --- src/utils/crop-text.mjs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/utils/crop-text.mjs b/src/utils/crop-text.mjs index 79f6f832..214d7c30 100644 --- a/src/utils/crop-text.mjs +++ b/src/utils/crop-text.mjs @@ -31,9 +31,9 @@ const clamp = (v, min, max) => { /** this function will crop text by keeping the beginning and end */ export async function cropText( text, - maxLength = 4000, - startLength = 400, - endLength = 300, + maxLength = 4096, + startLength = 0, + endLength = 0, tiktoken = true, ) { const userConfig = await getUserConfig() @@ -42,12 +42,22 @@ export async function cropText( null, userConfig.customModelName, ).match(/[- (]*([0-9]+)k/)?.[1] + + // for maxlength prefer modelLimit > userLimit > default if (k) { + // if we have the models exact content limit use that maxLength = Number(k) * 1000 - maxLength -= 100 + clamp(userConfig.maxResponseTokenLength, 1, maxLength - 1000) + } else if (userConfig.maxResponseTokenLength) { + // if we don't have the models exact content limit use the default + maxLength = userConfig.maxResponseTokenLength } else { - maxLength -= 100 + clamp(userConfig.maxResponseTokenLength, 1, maxLength - 1000) + // if we don't have the models exact content limit use the default + } + + if (userConfig.maxResponseTokenLength) { + maxLength = clamp(maxLength, 1, userConfig.maxResponseTokenLength) } + maxLength -= 100 // give some buffer const splits = text.split(/[,，。?？!！;；\n]/).map((s) => s.trim()) const splitsLength = splits.map((s) => (tiktoken ? encode(s).length : s.length)) @@ -94,8 +104,7 @@ export async function cropText( } croppedText += splits.slice(endStartIndex).join('\n') - currentLength = firstHalfTokens + secondHalfTokens + (middleIndex !== endStartIndex ? 9 : 0) // 9 is the length of "\n[cropped]\n" - // ... existing code ... + currentLength = firstHalfTokens + secondHalfTokens + (middleIndex !== endStartIndex ? 20 : 0) // 20 is approx the length of the disclaimer console.log( `input maxLength: ${maxLength}\n` + From b5d72476c85b0a892de38e9e7c52f76ad8d947fb Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 1 May 2025 05:15:20 +0800 Subject: [PATCH 3/3] wip --- src/config/index.mjs | 2 +- src/content-script/site-adapters/youtube/index.mjs | 4 ++-- src/utils/crop-text.mjs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/config/index.mjs b/src/config/index.mjs index ecc1620e..b11a987e 100644 --- a/src/config/index.mjs +++ b/src/config/index.mjs @@ -332,7 +332,7 @@ export const defaultConfig = { // advanced - maxResponseTokenLength: 1000, + maxResponseTokenLength: 200000, maxConversationContextLength: 9, temperature: 1, customChatGptWebApiUrl: 'https://chatgpt.com', diff --git a/src/content-script/site-adapters/youtube/index.mjs b/src/content-script/site-adapters/youtube/index.mjs index 02f91ddb..da63d37a 100644 --- a/src/content-script/site-adapters/youtube/index.mjs +++ b/src/content-script/site-adapters/youtube/index.mjs @@ -54,8 +54,8 @@ export default { subtitleContent = replaceHtmlEntities(subtitleContent) return await cropText( - `Provide a structured summary of the following video in markdown format, focusing on key takeaways and crucial information, and ensuring to include the video title. The summary should be easy to read and concise, yet comprehensive.` + - `The video title is "${title}". The subtitle content is as follows:\n${subtitleContent}`, + `Provide a structured summary of the content of the following video in markdown format, focusing on key takeaways and crucial information for the viewer Gwern, and ensure to include the video title and completeness of transcript if needed. Ignore promotions, bio's, and other uninteresting parts. The summary should be easy to read and concise, yet comprehensive. You should include key text as markdown quotes after tidying them up.` + + `The video title is "${title}". Add a tldr and BLUF. The subtitle content is as follows:\n${subtitleContent}`, ) } catch (e) { console.log(e) diff --git a/src/utils/crop-text.mjs b/src/utils/crop-text.mjs index 214d7c30..bd561601 100644 --- a/src/utils/crop-text.mjs +++ b/src/utils/crop-text.mjs @@ -31,7 +31,7 @@ const clamp = (v, min, max) => { /** this function will crop text by keeping the beginning and end */ export async function cropText( text, - maxLength = 4096, + maxLength = 200000, startLength = 0, endLength = 0, tiktoken = true, @@ -97,10 +97,10 @@ export async function cropText( // Construct the cropped text croppedText = splits.slice(0, middleIndex).join('\n') - if (middleIndex !== endStartIndex) { + if (croppedTokens > 0) { croppedText += `\n\n**Important disclaimer**, this text is incomplete! ${croppedTokens} or ${ (croppedTokens / totalTokens).toFixed(2) * 100 - }% of tokens have been removed from this location in the text due to lack limited model context\n\n` + }% of tokens have been removed from this location in the text due to lack limited model context of ${maxLength}\n\n` } croppedText += splits.slice(endStartIndex).join('\n')