From 2396966740f4e8485e3fe38552b30a2cb4d5c00c Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Fri, 24 May 2024 22:25:14 -0700 Subject: [PATCH] Tokens: rename the text methods --- src/apps/chat/components/composer/Composer.tsx | 4 ++-- .../composer/llmattachments/useLLMAttachmentDrafts.ts | 4 ++-- src/apps/tokens/AppTokens.tsx | 4 ++-- src/common/stores/chat/store-chats.ts | 4 ++-- src/common/util/token-counter.ts | 4 ++-- src/modules/aifn/summarize/ContentReducer.tsx | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index 009ce65fe..1657f7736 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -29,7 +29,7 @@ import { PreferencesTab, useOptimaLayout } from '~/common/layout/optima/useOptim import { SpeechResult, useSpeechRecognition } from '~/common/components/useSpeechRecognition'; import { animationEnterBelow } from '~/common/util/animUtils'; import { conversationTitle, DConversationId } from '~/common/stores/chat/chat.conversation'; -import { countModelTokens } from '~/common/util/token-counter'; +import { textTokensForLLMId } from '~/common/util/token-counter'; import { getConversation, useChatStore } from '~/common/stores/chat/store-chats'; import { isMacUser } from '~/common/util/pwaUtils'; import { launchAppCall } from '~/common/app.routes'; @@ -175,7 +175,7 @@ export function Composer(props: { const tokensComposerText = React.useMemo(() => { if (!debouncedText || !chatLLMId) return 0; - return countModelTokens(debouncedText, chatLLMId, 'composer text') ?? 0; + return textTokensForLLMId(debouncedText, chatLLMId, 'composer text') ?? 0; }, [chatLLMId, debouncedText]); let tokensComposer = tokensComposerText + llmAttachmentDrafts.llmTokenCountApprox; if (tokensComposer > 0) diff --git a/src/apps/chat/components/composer/llmattachments/useLLMAttachmentDrafts.ts b/src/apps/chat/components/composer/llmattachments/useLLMAttachmentDrafts.ts index 3bb413618..0665a85b3 100644 --- a/src/apps/chat/components/composer/llmattachments/useLLMAttachmentDrafts.ts +++ b/src/apps/chat/components/composer/llmattachments/useLLMAttachmentDrafts.ts @@ -3,7 +3,7 @@ import * as React from 'react'; import { DLLM, DLLMId, LLM_IF_OAI_Vision } from '~/modules/llms/store-llms'; import type { DAttachmentPart } from '~/common/stores/chat/chat.message'; -import { countModelTokens } from '~/common/util/token-counter'; +import { textTokensForLLMId } from '~/common/util/token-counter'; import type { AttachmentDraft, AttachmentDraftId } from '~/common/attachment-drafts/attachment.types'; @@ -68,7 +68,7 @@ function toLLMAttachment(attachmentDraft: AttachmentDraft, llmSupportedOutputPar const llmTokenCountApprox = llmForTokenCount ? attachmentCollapsedParts.reduce((acc, output) => { if (output.atype === 'atext') - return acc + (countModelTokens(output.text, llmForTokenCount, 'attachments tokens count') ?? 0); + return acc + (textTokensForLLMId(output.text, llmForTokenCount, 'attachments tokens count') ?? 0); console.warn('Unhandled token preview for output type:', output.atype); return acc; }, 0) diff --git a/src/apps/tokens/AppTokens.tsx b/src/apps/tokens/AppTokens.tsx index 53e08326c..699459c90 100644 --- a/src/apps/tokens/AppTokens.tsx +++ b/src/apps/tokens/AppTokens.tsx @@ -3,7 +3,7 @@ import * as React from 'react'; import { Box, Container, FormControl, Textarea, Typography } from '@mui/joy'; import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; -import { countTokenizerTokens, preloadTiktokenLibrary } from '~/common/util/token-counter'; +import { textTokensForEncodingId, preloadTiktokenLibrary } from '~/common/util/token-counter'; import { lineHeightTextareaMd } from '~/common/app.theme'; import { useTokenizerSelect } from '~/common/components/forms/useTokenizerSelect'; @@ -47,7 +47,7 @@ export function AppTokens() { const updateTokenDetails = (text: string, encodingId: string | null) => { if (encodingId) { - const details = countTokenizerTokens(text, encodingId, 'AppTokens'); + const details = textTokensForEncodingId(text, encodingId, 'AppTokens'); setTokenDetails(details || []); } }; diff --git a/src/common/stores/chat/store-chats.ts b/src/common/stores/chat/store-chats.ts index edd57add6..6f0d1c0b4 100644 --- a/src/common/stores/chat/store-chats.ts +++ b/src/common/stores/chat/store-chats.ts @@ -7,7 +7,7 @@ import { DLLMId, getChatLLMId } from '~/modules/llms/store-llms'; import { SystemPurposeId } from '../../../data'; import { backupIdbV3, idbStateStorage } from '~/common/util/idbUtils'; -import { countModelTokens } from '~/common/util/token-counter'; +import { textTokensForLLMId } from '~/common/util/token-counter'; import { DMessage, DMessageId, DMessageMetadata, reduceContentToText } from './chat.message'; import { conversationTitle, convertCConversation_V3_V4, createDConversation, DConversation, DConversationId, duplicateCConversation } from './chat.conversation'; @@ -341,7 +341,7 @@ function updateMessageTokenCount(message: DMessage, llmId: DLLMId | null, forceU const messageTextParts = reduceContentToText(message.content, ''); // TODO: handle attachments too - message.tokenCount = countModelTokens(messageTextParts, llmId, debugFrom) ?? 0; + message.tokenCount = textTokensForLLMId(messageTextParts, llmId, debugFrom) ?? 0; } return message.tokenCount; } diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts index 2c7c46608..90501fd41 100644 --- a/src/common/util/token-counter.ts +++ b/src/common/util/token-counter.ts @@ -67,7 +67,7 @@ const tokenizerCache: { [encodingId: string]: Tiktoken } = {}; * @param {string} debugFrom - Debug information. * @returns {number | null} The token count or null if not ready. */ -export function countModelTokens(text: string, llmId: DLLMId, debugFrom: string): number | null { +export function textTokensForLLMId(text: string, llmId: DLLMId, debugFrom: string): number | null { // The library shall have been preloaded - if not, attempt to start its loading and return null to indicate we're not ready to count if (!encoding_for_model || !get_encoding) { if (!informTheUser) { @@ -114,7 +114,7 @@ export function countModelTokens(text: string, llmId: DLLMId, debugFrom: string) * @param {string} debugFrom - Debug information. * @returns {Array<{ token: number, bytes: string }> | null} The detailed token information or null if not ready. */ -export function countTokenizerTokens(text: string, encodingId: string, debugFrom: string): Array | null { +export function textTokensForEncodingId(text: string, encodingId: string, debugFrom: string): Array | null { // The library shall have been preloaded - if not, attempt to start its loading and return null to indicate we're not ready to count if (!get_encoding) { if (!informTheUser) { diff --git a/src/modules/aifn/summarize/ContentReducer.tsx b/src/modules/aifn/summarize/ContentReducer.tsx index 43ed0aad0..e8f158881 100644 --- a/src/modules/aifn/summarize/ContentReducer.tsx +++ b/src/modules/aifn/summarize/ContentReducer.tsx @@ -10,7 +10,7 @@ import { TokenBadgeMemo } from '../../../apps/chat/components/composer/TokenBadg import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; import { GoodModal } from '~/common/components/GoodModal'; import { Section } from '~/common/components/Section'; -import { countModelTokens } from '~/common/util/token-counter'; +import { textTokensForLLMId } from '~/common/util/token-counter'; import { lineHeightTextareaMd } from '~/common/app.theme'; import { summerizeToFitContextBudget } from './summerize'; @@ -48,7 +48,7 @@ export function ContentReducer(props: { const [processing, setProcessing] = React.useState(false); // derived state - const reducedTokens = reducerModelId ? countModelTokens(reducedText, reducerModelId, 'content reducer reduce') ?? 0 : 0; + const reducedTokens = reducerModelId ? textTokensForLLMId(reducedText, reducerModelId, 'content reducer reduce') ?? 0 : 0; const remainingTokens = props.tokenLimit - reducedTokens;