zack3d/big-agi
1
0
Fork 0
mirror of https://github.com/enricoros/big-AGI.git synced 2026-05-11 14:10:15 -07:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup Reducer UI, better support for 3.5-Turbo

Browse Source
This commit is contained in:
Enrico Ros
2023-04-18 01:33:35 -07:00
parent 8e6ca76b37
commit fb2d7bea42
4 changed files with 70 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/Composer.tsx
+1 -1
View File
@@ -515,7 +515,7 @@ export function Composer(props: {
{/* Content reducer modal */}
{reducerText?.length >= 1 && chatModelId &&
<ContentReducerModal
initialText={reducerText} initialTokens={reducerTextTokens} tokenLimit={remainingTokens} chatModelId={chatModelId}
initialText={reducerText} initialTokens={reducerTextTokens} tokenLimit={remainingTokens}
onReducedText={handleContentReducerText} onClose={handleContentReducerClose}
/>
}
components/dialogs/ContentReducerModal.tsx
+53 -43
View File
@@ -1,8 +1,8 @@
import * as React from 'react';
import { Alert, Box, Button, CircularProgress, Divider, FormControl, FormHelperText, FormLabel, Modal, ModalClose, ModalDialog, Slider, Textarea, Typography } from '@mui/joy';
import { Alert, Box, Button, CircularProgress, Divider, FormControl, FormHelperText, FormLabel, Modal, ModalClose, ModalDialog, Option, Select, Slider, Stack, Textarea, Typography } from '@mui/joy';
import { ChatModelId } from '@/lib/data';
import { ChatModelId, ChatModels, fastChatModelId } from '@/lib/data';
import { Section } from '@/components/dialogs/SettingsModal';
import { TokenBadge } from '@/components/util/TokenBadge';
import { countModelTokens } from '@/lib/llm/tokens';
@@ -27,37 +27,36 @@ export function ContentReducerModal(props: {
initialText: string,
initialTokens: number,
tokenLimit: number,
chatModelId: ChatModelId,
onClose: () => void,
onReducedText: (text: string) => void,
}) {
// state
const [reducerModelId, setReducerModelId] = React.useState<ChatModelId>(fastChatModelId);
const [compressionLevel, setCompressionLevel] = React.useState(3);
const [reducedText, setReducedText] = React.useState('');
const [processing, setProcessing] = React.useState(false);
// external state
// ...
// derived state
const reducedTokens = countModelTokens(reducedText, props.chatModelId, 'content reducer reduce');
const reducedTokens = countModelTokens(reducedText, reducerModelId, 'content reducer reduce');
const remainingTokens = props.tokenLimit - reducedTokens;
const handleCompressionLevelChange = (event: Event, newValue: number | number[]) =>
setCompressionLevel(newValue as number);
const handleChatModelChange = (event: any, value: ChatModelId | null) => value && setReducerModelId(value);
const handleCompressionLevelChange = (event: Event, newValue: number | number[]) => setCompressionLevel(newValue as number);
const handlePreviewClicked = async () => {
console.log('props.tokenBudget', props.tokenLimit);
setProcessing(true);
const reducedText = await summerizeToFitContextBudget(props.initialText, props.tokenLimit, props.chatModelId);
const reducedText = await summerizeToFitContextBudget(props.initialText, props.tokenLimit, reducerModelId);
setReducedText(reducedText);
setProcessing(false);
};
const handleUseReducedTextClicked = () =>
props.onReducedText(reducedText);
const handleUseReducedTextClicked = () => props.onReducedText(reducedText);
// DISABLED: user shall select the model and compression level first
// upon load, click the preview button
// React.useEffect(() => {
// // noinspection JSIgnoredPromiseFromCall
@@ -75,50 +74,61 @@ export function ContentReducerModal(props: {
<ModalClose />
<Typography level='h5'>Content Reducer [Pre-Alpha]</Typography>
<Typography level='h5'>Content Reducer (preview)</Typography>
<Divider sx={{ my: 2 }} />
<Section title='Inputs'>
{/* Settings */}
<Section>
<Stack direction='column' sx={{ gap: 2 }}>
<Typography level='body2'>
Text: {props.initialTokens.toLocaleString()} tokens
</Typography>
<Typography level='body2'>
Limit: {props.tokenLimit.toLocaleString()} tokens
</Typography>
<Typography level='body2'>
Input: <b>{props.initialTokens.toLocaleString()}</b> tokens · Limit: <b>{props.tokenLimit.toLocaleString()}</b> tokens
<br />
compression needed ≥ <b>{props.tokenLimit ? Math.round(100 * props.initialTokens / props.tokenLimit) : 0}</b> %
</Typography>
</Section>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between' }}>
<Box sx={{ minWidth: 120 }}>
<FormLabel>Reducer model</FormLabel>
<FormHelperText>{ChatModels[reducerModelId]?.tradeoff}</FormHelperText>
</Box>
{reducerModelId && <Select value={reducerModelId} onChange={handleChatModelChange} sx={{ minWidth: 140 }}>
{Object.keys(ChatModels).map((key: string) => (
<Option key={key} value={key}>
{ChatModels[key as ChatModelId].title}
</Option>
))}
</Select>}
</FormControl>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between' }}>
<Box sx={{ minWidth: 120 }}>
<FormLabel>Compression</FormLabel>
<FormHelperText>{compressionLevel < 2 ? 'Low' : compressionLevel > 4 ? 'High' : 'Medium'}</FormHelperText>
</Box>
<Slider
color='neutral' disabled
min={1} max={5} defaultValue={3}
value={compressionLevel} onChange={handleCompressionLevelChange}
valueLabelDisplay='auto'
sx={{ py: 1, mt: 1.1 }}
/>
</FormControl>
{/* Example User settings */}
<Section title='Settings'>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between' }}>
<Box sx={{ minWidth: 120 }}>
<FormLabel>Compression</FormLabel>
<FormHelperText>{compressionLevel < 2 ? 'Low' : compressionLevel > 4 ? 'High' : 'Medium'}</FormHelperText>
<Box sx={{ display: 'flex', justifyContent: 'flex-end' }}>
<Button variant='solid' color='primary' onClick={handlePreviewClicked} disabled={processing}>
Preview
</Button>
</Box>
<Slider
aria-label='Model Temperature' color='neutral'
min={1} max={5} defaultValue={3}
value={compressionLevel} onChange={handleCompressionLevelChange}
valueLabelDisplay='auto'
sx={{ py: 1, mt: 1.1 }}
/>
</FormControl>
<Box sx={{ display: 'flex', justifyContent: 'flex-end' }}>
<Button variant='solid' color='primary' onClick={handlePreviewClicked} disabled={processing}>
Preview
</Button>
</Box>
</Stack>
</Section>
<Section title='Outputs'>
{/* Outputs */}
<Section title='Compressed content'>
{/* Readonly output and token counter */}
<Box sx={{ flexGrow: 1, position: 'relative', minWidth: '30vw' }}>
lib/data.ts
+4 -1
View File
@@ -73,7 +73,8 @@ type ChatModelData = {
description: string | JSX.Element;
title: string;
fullName: string; // seems unused
contextWindowSize: number,
contextWindowSize: number;
tradeoff: string;
}
export const ChatModels: { [key in ChatModelId]: ChatModelData } = {
@@ -82,11 +83,13 @@ export const ChatModels: { [key in ChatModelId]: ChatModelData } = {
title: 'GPT-4',
fullName: 'GPT-4',
contextWindowSize: 8192,
tradeoff: 'Precise, slow and expensive',
},
'gpt-3.5-turbo': {
description: 'A good balance between speed and insight',
title: '3.5-Turbo',
fullName: 'GPT-3.5 Turbo',
contextWindowSize: 4097,
tradeoff: 'Faster and cheaper',
},
};
lib/llm/summerize.ts
+12 -10
View File
@@ -1,7 +1,7 @@
// summerize.ts
import { ApiChatInput, ApiChatResponse } from '../../pages/api/openai/chat';
import { cleanupPrompt } from './prompts';
import { useSettingsStore } from '@/lib/stores/store-settings';
import { ChatModelId, ChatModels } from '@/lib/data';
function breakDownChunk(chunk: string, targetWordCount: number): string[] {
const words = chunk.split(' ');
@@ -14,11 +14,7 @@ function breakDownChunk(chunk: string, targetWordCount: number): string[] {
return subChunks;
}
export async function summerizeToFitContextBudget(text: string, targetWordCount: number, modelId: string): Promise<string> {
if (typeof text !== 'string' || typeof targetWordCount !== 'number') {
throw new Error('Invalid input. Please provide a string and a number.');
}
export async function summerizeToFitContextBudget(text: string, targetWordCount: number, modelId: ChatModelId): Promise<string> {
if (targetWordCount < 0) {
throw new Error('Target word count must be a non-negative number.');
}
@@ -65,9 +61,15 @@ export async function summerizeToFitContextBudget(text: string, targetWordCount:
return summarizedChunks.join('\n');
}
async function cleanUpContent(chunk: string, modelId: string, max_tokens: number): Promise<string> {
async function cleanUpContent(chunk: string, modelId: ChatModelId, ignored_was_targetWordCount: number): Promise<string> {
const { apiKey, apiHost, apiOrganizationId } = useSettingsStore.getState();
// auto-adjust the tokens assuming the output would be half the size of the input (a bit dangerous,
// but at this stage we are not guaranteed the input nor output would fit)
const outputTokenShare = 1 / 3;
const autoResponseTokensSize = Math.floor(ChatModels[modelId].contextWindowSize * outputTokenShare);
const input: ApiChatInput = {
api: {
...(apiKey && { apiKey }),
@@ -78,7 +80,7 @@ async function cleanUpContent(chunk: string, modelId: string, max_tokens: number
messages: [
{ role: 'system', content: cleanupPrompt },
{ role: 'user', content: chunk }],
max_tokens: max_tokens, // Adjust the max tokens as needed
max_tokens: autoResponseTokensSize, // note: before was 'targetWordCount', but it's not correct
};
const response = await fetch('/api/openai/chat', {
@@ -91,14 +93,14 @@ async function cleanUpContent(chunk: string, modelId: string, max_tokens: number
if (!response.ok) {
console.error('Error from API call: ', response.status, response.statusText);
return "";
return '';
}
const data: ApiChatResponse = await response.json();
return data.message.content;
}
async function recursiveSummerize(text: string, modelId: string, targetWordCount: number): Promise<string> {
async function recursiveSummerize(text: string, modelId: ChatModelId, targetWordCount: number): Promise<string> {
const words = text.split(' ');
if (words.length <= targetWordCount || words.length <= 1) {