diff --git a/src/apps/chat/store-app-chat.ts b/src/apps/chat/store-app-chat.ts index 40234897c..d04b863a6 100644 --- a/src/apps/chat/store-app-chat.ts +++ b/src/apps/chat/store-app-chat.ts @@ -3,10 +3,13 @@ import { persist } from 'zustand/middleware'; import { useShallow } from 'zustand/react/shallow'; import type { DLLMId } from '~/common/stores/llms/llms.types'; +import { Is } from '~/common/util/pwaUtils'; export type ChatAutoSpeakType = 'off' | 'firstLine' | 'all'; +export type TokenCountingMethod = 'accurate' | 'approximate'; + // Chat Settings (Chat AI & Chat UI) @@ -38,6 +41,9 @@ interface AppChatStore { chatKeepLastThinkingOnly: boolean, setChatKeepLastThinkingOnly: (chatKeepLastThinkingOnly: boolean) => void; + tokenCountingMethod: TokenCountingMethod; + setTokenCountingMethod: (tokenCountingMethod: TokenCountingMethod) => void; + // chat UI clearFilters: () => void; @@ -107,6 +113,9 @@ const useAppChatStore = create()(persist( chatKeepLastThinkingOnly: true, setChatKeepLastThinkingOnly: (chatKeepLastThinkingOnly: boolean) => _set({ chatKeepLastThinkingOnly }), + tokenCountingMethod: Is.Desktop ? 'accurate' : 'approximate', + setTokenCountingMethod: (tokenCountingMethod: TokenCountingMethod) => _set({ tokenCountingMethod }), + // Chat UI clearFilters: () => _set({ filterIsArchived: false, filterHasDocFragments: false, filterHasImageAssets: false, filterHasStars: false }), @@ -181,6 +190,7 @@ export const useChatAutoAI = () => useAppChatStore(useShallow(state => ({ autoTitleChat: state.autoTitleChat, autoVndAntBreakpoints: state.autoVndAntBreakpoints, chatKeepLastThinkingOnly: state.chatKeepLastThinkingOnly, + tokenCountingMethod: state.tokenCountingMethod, setAutoSpeak: state.setAutoSpeak, setAutoSuggestAttachmentPrompts: state.setAutoSuggestAttachmentPrompts, setAutoSuggestDiagrams: state.setAutoSuggestDiagrams, @@ -189,6 +199,7 @@ export const useChatAutoAI = () => useAppChatStore(useShallow(state => ({ setAutoTitleChat: state.setAutoTitleChat, setAutoVndAntBreakpoints: state.setAutoVndAntBreakpoints, setChatKeepLastThinkingOnly: state.setChatKeepLastThinkingOnly, + setTokenCountingMethod: state.setTokenCountingMethod, }))); export const getChatAutoAI = (): { @@ -208,6 +219,9 @@ export const useChatAutoSuggestHTMLUI = (): boolean => export const useChatAutoSuggestAttachmentPrompts = (): boolean => useAppChatStore(state => state.autoSuggestAttachmentPrompts); +export const getChatTokenCountingMethod = (): TokenCountingMethod => + useAppChatStore.getState().tokenCountingMethod; + export const useChatMicTimeoutMsValue = (): number => useAppChatStore(state => state.micTimeoutMs); diff --git a/src/apps/settings-modal/AppChatSettingsAI.tsx b/src/apps/settings-modal/AppChatSettingsAI.tsx index 1fa417d29..4d6b36158 100644 --- a/src/apps/settings-modal/AppChatSettingsAI.tsx +++ b/src/apps/settings-modal/AppChatSettingsAI.tsx @@ -13,6 +13,7 @@ import { useLLMSelect } from '~/common/components/forms/useLLMSelect'; import { useLabsDevMode } from '~/common/stores/store-ux-labs'; import { useModelDomain } from '~/common/stores/llms/hooks/useModelDomain'; +import type { TokenCountingMethod } from '../chat/store-app-chat'; import { useChatAutoAI } from '../chat/store-app-chat'; @@ -29,6 +30,19 @@ const _keepThinkingBlocksOptions: FormSelectOption<'all' | 'last-only'>[] = [ }, ] as const; +const _tokenCountingMethodOptions: FormSelectOption[] = [ + { + value: 'approximate', + label: 'Fast', + description: 'Lightweight: ~90% approximation', + }, + { + value: 'accurate', + label: 'Precise', + description: 'Accurate tokenizer, heavier', + }, +] as const; + function FormControlDomainModel(props: { domainId: DModelDomainId, @@ -63,6 +77,7 @@ export function AppChatSettingsAI() { // autoSuggestQuestions, setAutoSuggestQuestions, autoTitleChat, setAutoTitleChat, chatKeepLastThinkingOnly, setChatKeepLastThinkingOnly, + tokenCountingMethod, setTokenCountingMethod, } = useChatAutoAI(); const labsDevMode = useLabsDevMode(); @@ -123,6 +138,15 @@ export function AppChatSettingsAI() { /> )} + + 100 ? Math.min(adjustedTokens * 0.05, 10) : 0; + + const finalCount = Math.max(1, Math.round(adjustedTokens - spaceAdjustment - repetitionReduction)); + + DEBUG_TOKEN_COUNT && console.log(`approximateTextTokens: ${debugFrom}, family: ${modelFamily}, type: ${contentType}, chars: ${text.length}, tokens: ${finalCount}`); + + return finalCount; +}