diff --git a/components/Composer.tsx b/components/Composer.tsx index edc607a87..f4ca2f4ef 100644 --- a/components/Composer.tsx +++ b/components/Composer.tsx @@ -515,7 +515,7 @@ export function Composer(props: { {/* Content reducer modal */} {reducerText?.length >= 1 && chatModelId && } diff --git a/components/dialogs/ContentReducerModal.tsx b/components/dialogs/ContentReducerModal.tsx index 532f30147..fcee35b04 100644 --- a/components/dialogs/ContentReducerModal.tsx +++ b/components/dialogs/ContentReducerModal.tsx @@ -1,8 +1,8 @@ import * as React from 'react'; -import { Alert, Box, Button, CircularProgress, Divider, FormControl, FormHelperText, FormLabel, Modal, ModalClose, ModalDialog, Slider, Textarea, Typography } from '@mui/joy'; +import { Alert, Box, Button, CircularProgress, Divider, FormControl, FormHelperText, FormLabel, Modal, ModalClose, ModalDialog, Option, Select, Slider, Stack, Textarea, Typography } from '@mui/joy'; -import { ChatModelId } from '@/lib/data'; +import { ChatModelId, ChatModels, fastChatModelId } from '@/lib/data'; import { Section } from '@/components/dialogs/SettingsModal'; import { TokenBadge } from '@/components/util/TokenBadge'; import { countModelTokens } from '@/lib/llm/tokens'; @@ -27,37 +27,36 @@ export function ContentReducerModal(props: { initialText: string, initialTokens: number, tokenLimit: number, - chatModelId: ChatModelId, onClose: () => void, onReducedText: (text: string) => void, }) { // state + const [reducerModelId, setReducerModelId] = React.useState(fastChatModelId); const [compressionLevel, setCompressionLevel] = React.useState(3); const [reducedText, setReducedText] = React.useState(''); const [processing, setProcessing] = React.useState(false); - // external state - // ... // derived state - const reducedTokens = countModelTokens(reducedText, props.chatModelId, 'content reducer reduce'); + const reducedTokens = countModelTokens(reducedText, reducerModelId, 'content reducer reduce'); const remainingTokens = props.tokenLimit - reducedTokens; - const handleCompressionLevelChange = (event: Event, newValue: number | number[]) => - setCompressionLevel(newValue as number); + + const handleChatModelChange = (event: any, value: ChatModelId | null) => value && setReducerModelId(value); + + const handleCompressionLevelChange = (event: Event, newValue: number | number[]) => setCompressionLevel(newValue as number); const handlePreviewClicked = async () => { - console.log('props.tokenBudget', props.tokenLimit); setProcessing(true); - const reducedText = await summerizeToFitContextBudget(props.initialText, props.tokenLimit, props.chatModelId); + const reducedText = await summerizeToFitContextBudget(props.initialText, props.tokenLimit, reducerModelId); setReducedText(reducedText); setProcessing(false); }; - const handleUseReducedTextClicked = () => - props.onReducedText(reducedText); + const handleUseReducedTextClicked = () => props.onReducedText(reducedText); + // DISABLED: user shall select the model and compression level first // upon load, click the preview button // React.useEffect(() => { // // noinspection JSIgnoredPromiseFromCall @@ -75,50 +74,61 @@ export function ContentReducerModal(props: { - Content Reducer [Pre-Alpha] + Content Reducer (preview) -
+ {/* Settings */} +
+ - - Text: {props.initialTokens.toLocaleString()} tokens - - - Limit: {props.tokenLimit.toLocaleString()} tokens - + + Input: {props.initialTokens.toLocaleString()} tokens · Limit: {props.tokenLimit.toLocaleString()} tokens +
+ compression needed ≥ {props.tokenLimit ? Math.round(100 * props.initialTokens / props.tokenLimit) : 0} % +
-
+ + + Reducer model + {ChatModels[reducerModelId]?.tradeoff} + + {reducerModelId && } + + + + Compression + {compressionLevel < 2 ? 'Low' : compressionLevel > 4 ? 'High' : 'Medium'} + + + - {/* Example User settings */} -
- - - - Compression - {compressionLevel < 2 ? 'Low' : compressionLevel > 4 ? 'High' : 'Medium'} + + - - - - - - +
-
+ {/* Outputs */} +
{/* Readonly output and token counter */} diff --git a/lib/data.ts b/lib/data.ts index 9def01c0e..dfed57188 100644 --- a/lib/data.ts +++ b/lib/data.ts @@ -73,7 +73,8 @@ type ChatModelData = { description: string | JSX.Element; title: string; fullName: string; // seems unused - contextWindowSize: number, + contextWindowSize: number; + tradeoff: string; } export const ChatModels: { [key in ChatModelId]: ChatModelData } = { @@ -82,11 +83,13 @@ export const ChatModels: { [key in ChatModelId]: ChatModelData } = { title: 'GPT-4', fullName: 'GPT-4', contextWindowSize: 8192, + tradeoff: 'Precise, slow and expensive', }, 'gpt-3.5-turbo': { description: 'A good balance between speed and insight', title: '3.5-Turbo', fullName: 'GPT-3.5 Turbo', contextWindowSize: 4097, + tradeoff: 'Faster and cheaper', }, }; \ No newline at end of file diff --git a/lib/llm/summerize.ts b/lib/llm/summerize.ts index 0b4b053af..70e7f4c5d 100644 --- a/lib/llm/summerize.ts +++ b/lib/llm/summerize.ts @@ -1,7 +1,7 @@ -// summerize.ts import { ApiChatInput, ApiChatResponse } from '../../pages/api/openai/chat'; import { cleanupPrompt } from './prompts'; import { useSettingsStore } from '@/lib/stores/store-settings'; +import { ChatModelId, ChatModels } from '@/lib/data'; function breakDownChunk(chunk: string, targetWordCount: number): string[] { const words = chunk.split(' '); @@ -14,11 +14,7 @@ function breakDownChunk(chunk: string, targetWordCount: number): string[] { return subChunks; } -export async function summerizeToFitContextBudget(text: string, targetWordCount: number, modelId: string): Promise { - if (typeof text !== 'string' || typeof targetWordCount !== 'number') { - throw new Error('Invalid input. Please provide a string and a number.'); - } - +export async function summerizeToFitContextBudget(text: string, targetWordCount: number, modelId: ChatModelId): Promise { if (targetWordCount < 0) { throw new Error('Target word count must be a non-negative number.'); } @@ -65,9 +61,15 @@ export async function summerizeToFitContextBudget(text: string, targetWordCount: return summarizedChunks.join('\n'); } -async function cleanUpContent(chunk: string, modelId: string, max_tokens: number): Promise { +async function cleanUpContent(chunk: string, modelId: ChatModelId, ignored_was_targetWordCount: number): Promise { const { apiKey, apiHost, apiOrganizationId } = useSettingsStore.getState(); + + // auto-adjust the tokens assuming the output would be half the size of the input (a bit dangerous, + // but at this stage we are not guaranteed the input nor output would fit) + const outputTokenShare = 1 / 3; + const autoResponseTokensSize = Math.floor(ChatModels[modelId].contextWindowSize * outputTokenShare); + const input: ApiChatInput = { api: { ...(apiKey && { apiKey }), @@ -78,7 +80,7 @@ async function cleanUpContent(chunk: string, modelId: string, max_tokens: number messages: [ { role: 'system', content: cleanupPrompt }, { role: 'user', content: chunk }], - max_tokens: max_tokens, // Adjust the max tokens as needed + max_tokens: autoResponseTokensSize, // note: before was 'targetWordCount', but it's not correct }; const response = await fetch('/api/openai/chat', { @@ -91,14 +93,14 @@ async function cleanUpContent(chunk: string, modelId: string, max_tokens: number if (!response.ok) { console.error('Error from API call: ', response.status, response.statusText); - return ""; + return ''; } const data: ApiChatResponse = await response.json(); return data.message.content; } -async function recursiveSummerize(text: string, modelId: string, targetWordCount: number): Promise { +async function recursiveSummerize(text: string, modelId: ChatModelId, targetWordCount: number): Promise { const words = text.split(' '); if (words.length <= targetWordCount || words.length <= 1) {