diff --git a/src/common/components/forms/useLLMSelect.tsx b/src/common/components/forms/useLLMSelect.tsx index 6a7f5cfa5..83b47ff5e 100644 --- a/src/common/components/forms/useLLMSelect.tsx +++ b/src/common/components/forms/useLLMSelect.tsx @@ -11,7 +11,7 @@ import { findModelVendor } from '~/modules/llms/vendors/vendors.registry'; import { llmsGetVendorIcon, LLMVendorIcon } from '~/modules/llms/components/LLMVendorIcon'; import type { DModelDomainId } from '~/common/stores/llms/model.domains.types'; -import { DLLM, DLLMId, LLM_IF_OAI_Reasoning, LLM_IF_Outputs_Audio, LLM_IF_Outputs_Image, LLM_IF_Tools_WebSearch } from '~/common/stores/llms/llms.types'; +import { DLLM, DLLMId, getLLMPricing, LLM_IF_OAI_Reasoning, LLM_IF_Outputs_Audio, LLM_IF_Outputs_Image, LLM_IF_Tools_WebSearch } from '~/common/stores/llms/llms.types'; import { PhGearSixIcon } from '~/common/components/icons/phosphor/PhGearSixIcon'; import { StarredNoXL2 } from '~/common/components/StarIcons'; import { TooltipOutlined } from '~/common/components/TooltipOutlined'; @@ -182,7 +182,7 @@ export function useLLMSelect( let features = ''; const isNotSymlink = !llm.label.startsWith('๐Ÿ”—'); - const seemsFree = !!llm.pricing?.chat?._isFree; + const seemsFree = !!getLLMPricing(llm)?.chat?._isFree; if (isNotSymlink) { // check features if (seemsFree) features += 'free '; diff --git a/src/common/stores/llms/llms.types.ts b/src/common/stores/llms/llms.types.ts index 9d4baaba6..2948959d6 100644 --- a/src/common/stores/llms/llms.types.ts +++ b/src/common/stores/llms/llms.types.ts @@ -51,6 +51,7 @@ export interface DLLM { userParameters?: DModelParameterValues; // user has set these parameters userContextTokens?: DLLMContextTokens; // user override for context window userMaxOutputTokens?: DLLMMaxOutputTokens; // user override for max output tokens + userPricing?: DModelPricing; // user override for model pricing } @@ -121,6 +122,18 @@ export function getLLMMaxOutputTokens(llm: DLLM | null): DLLMMaxOutputTokens | u return llm.userMaxOutputTokens ?? llm.maxOutputTokens; } +/** + * Returns the effective pricing for a model. + * Checks user override first, then falls back to model default. + */ +export function getLLMPricing(llm: DLLM | null): DModelPricing | undefined { + if (!llm) + return undefined; // undefined if no model + + // Check user override first, then fall back to model default + return llm.userPricing ?? llm.pricing; +} + /// Interfaces /// diff --git a/src/common/stores/llms/store-llms-domains_slice.ts b/src/common/stores/llms/store-llms-domains_slice.ts index f3f82772b..27e637b2a 100644 --- a/src/common/stores/llms/store-llms-domains_slice.ts +++ b/src/common/stores/llms/store-llms-domains_slice.ts @@ -3,7 +3,7 @@ import type { StateCreator } from 'zustand/vanilla'; import type { ModelVendorId } from '~/modules/llms/vendors/vendors.registry'; import type { DModelDomainId } from './model.domains.types'; -import { DLLM, DLLMId, isLLMHidden, isLLMVisible } from './llms.types'; +import { DLLM, DLLMId, getLLMPricing, isLLMHidden, isLLMVisible } from './llms.types'; import { LlmsRootState, useModelsStore } from './store-llms'; import { ModelDomainsList, ModelDomainsRegistry } from './model.domains.registry'; import { createDModelConfiguration, DModelConfiguration } from './modelconfiguration.types'; @@ -275,7 +275,7 @@ function _groupLlmsByVendorRankedByElo(llms: ReadonlyArray): PreferredRank const eloCostItem = { id: llm.id, cbaElo: llm.benchmark?.cbaElo, - costRank: !llm.pricing ? undefined : _getLlmCostBenchmark(llm), + costRank: !getLLMPricing(llm) ? undefined : _getLlmCostBenchmark(llm), }; if (!group) acc.push({ vendorId: llm.vId, llmsByElo: [eloCostItem] }); @@ -295,8 +295,9 @@ function _groupLlmsByVendorRankedByElo(llms: ReadonlyArray): PreferredRank // Hypothetical cost benchmark for a model, based on total cost of 100k input tokens and 10k output tokens. function _getLlmCostBenchmark(llm: DLLM): number | undefined { - if (!llm.pricing?.chat) return undefined; - const costIn = getLlmCostForTokens(100000, 100000, llm.pricing.chat.input); - const costOut = getLlmCostForTokens(100000, 10000, llm.pricing.chat.output); + const pricing = getLLMPricing(llm); + if (!pricing?.chat) return undefined; + const costIn = getLlmCostForTokens(100000, 100000, pricing.chat.input); + const costOut = getLlmCostForTokens(100000, 10000, pricing.chat.output); return (costIn !== undefined && costOut !== undefined) ? costIn + costOut : undefined; } diff --git a/src/common/stores/llms/store-llms.ts b/src/common/stores/llms/store-llms.ts index 45f27c554..d0365f66e 100644 --- a/src/common/stores/llms/store-llms.ts +++ b/src/common/stores/llms/store-llms.ts @@ -86,6 +86,7 @@ export const useModelsStore = create()(persist( ...(existing.userParameters !== undefined ? { userParameters: { ...existing.userParameters } } : {}), ...(existing.userContextTokens !== undefined ? { userContextTokens: existing.userContextTokens } : {}), ...(existing.userMaxOutputTokens !== undefined ? { userMaxOutputTokens: existing.userMaxOutputTokens } : {}), + ...(existing.userPricing !== undefined ? { userPricing: existing.userPricing } : {}), }; }); } diff --git a/src/modules/aix/client/aix.client.ts b/src/modules/aix/client/aix.client.ts index 4851b3c88..a0ba19a82 100644 --- a/src/modules/aix/client/aix.client.ts +++ b/src/modules/aix/client/aix.client.ts @@ -2,7 +2,7 @@ import { findServiceAccessOrThrow } from '~/modules/llms/vendors/vendor.helpers' import type { DMessage, DMessageGenerator } from '~/common/stores/chat/chat.message'; import type { MaybePromise } from '~/common/types/useful.types'; -import { DLLM, DLLMId, LLM_IF_HOTFIX_NoTemperature, LLM_IF_OAI_Responses, LLM_IF_Outputs_Audio, LLM_IF_Outputs_Image, LLM_IF_Outputs_NoText } from '~/common/stores/llms/llms.types'; +import { DLLM, DLLMId, getLLMPricing, LLM_IF_HOTFIX_NoTemperature, LLM_IF_OAI_Responses, LLM_IF_Outputs_Audio, LLM_IF_Outputs_Image, LLM_IF_Outputs_NoText } from '~/common/stores/llms/llms.types'; import { DMetricsChatGenerate_Lg, metricsChatGenerateLgToMd, metricsComputeChatGenerateCostsMd } from '~/common/stores/metrics/metrics.chatgenerate'; import { DModelParameterValues, getAllModelParameterValues } from '~/common/stores/llms/llms.parameters'; import { apiStream } from '~/common/util/trpc.client'; @@ -535,7 +535,7 @@ function _llToDMessage(src: AixChatGenerateContent_LL, dest: AixChatGenerateCont function _updateGeneratorCostsInPlace(generator: DMessageGenerator, llm: DLLM, debugCostSource: string) { // Compute costs const logLlmRefId = getAllModelParameterValues(llm.initialParameters, llm.userParameters).llmRef || llm.id; - const costs = metricsComputeChatGenerateCostsMd(generator.metrics, llm.pricing?.chat, logLlmRefId); + const costs = metricsComputeChatGenerateCostsMd(generator.metrics, getLLMPricing(llm)?.chat, logLlmRefId); if (!costs) { // FIXME: we shall warn that the costs are missing, as the only way to get pricing is through surfacing missing prices return; diff --git a/src/modules/llms/models-modal/LLMOptionsModal.tsx b/src/modules/llms/models-modal/LLMOptionsModal.tsx index 32425f859..fb00751dc 100644 --- a/src/modules/llms/models-modal/LLMOptionsModal.tsx +++ b/src/modules/llms/models-modal/LLMOptionsModal.tsx @@ -9,7 +9,7 @@ import VisibilityIcon from '@mui/icons-material/Visibility'; import VisibilityOffIcon from '@mui/icons-material/VisibilityOff'; import type { DPricingChatGenerate } from '~/common/stores/llms/llms.pricing'; -import { DLLMId, getLLMContextTokens, getLLMMaxOutputTokens, isLLMVisible } from '~/common/stores/llms/llms.types'; +import { DLLMId, getLLMContextTokens, getLLMMaxOutputTokens, getLLMPricing, isLLMVisible } from '~/common/stores/llms/llms.types'; import { FormLabelStart } from '~/common/components/forms/FormLabelStart'; import { GoodModal } from '~/common/components/modals/GoodModal'; import { ModelDomainsList, ModelDomainsRegistry } from '~/common/stores/llms/model.domains.registry'; @@ -107,6 +107,60 @@ export function LLMOptionsModal(props: { id: DLLMId, onClose: () => void }) { const handleMaxOutputTokensReset = () => updateLLM(llm.id, { userMaxOutputTokens: undefined }); + const handleInputPriceChange = (event: React.ChangeEvent) => { + const value = event.target.value; + const numValue = value ? parseFloat(value) : undefined; + updateLLM(llm.id, { + userPricing: { + chat: { + ...llm.userPricing?.chat, + input: numValue, + output: llm.userPricing?.chat?.output, + }, + }, + }); + }; + + const handleInputPriceReset = () => { + const newPricing = { ...llm.userPricing }; + if (newPricing.chat) { + delete newPricing.chat.input; + // If no other pricing fields are set, clear userPricing entirely + if (!newPricing.chat.output && !newPricing.chat.cache) { + updateLLM(llm.id, { userPricing: undefined }); + } else { + updateLLM(llm.id, { userPricing: newPricing }); + } + } + }; + + const handleOutputPriceChange = (event: React.ChangeEvent) => { + const value = event.target.value; + const numValue = value ? parseFloat(value) : undefined; + updateLLM(llm.id, { + userPricing: { + chat: { + ...llm.userPricing?.chat, + input: llm.userPricing?.chat?.input, + output: numValue, + }, + }, + }); + }; + + const handleOutputPriceReset = () => { + const newPricing = { ...llm.userPricing }; + if (newPricing.chat) { + delete newPricing.chat.output; + // If no other pricing fields are set, clear userPricing entirely + if (!newPricing.chat.input && !newPricing.chat.cache) { + updateLLM(llm.id, { userPricing: undefined }); + } else { + updateLLM(llm.id, { userPricing: newPricing }); + } + } + }; + const handleLlmDelete = () => { removeLLM(llm.id); props.onClose(); @@ -235,7 +289,7 @@ export function LLMOptionsModal(props: { id: DLLMId, onClose: () => void }) { {llm.description} } - {!!llm.pricing?.chat?._isFree && + {!!getLLMPricing(llm)?.chat?._isFree && ๐ŸŽ Free model - note: refresh models to check for updates in pricing } @@ -245,7 +299,7 @@ export function LLMOptionsModal(props: { id: DLLMId, onClose: () => void }) { max output tokens: {getLLMMaxOutputTokens(llm)?.toLocaleString() ?? 'not provided'}
{!!llm.created && <>created:
} {/*ยท tags: {llm.tags.join(', ')}*/} - {!!llm.pricing?.chat && prettyPricingComponent(llm.pricing.chat)} + {!!getLLMPricing(llm)?.chat && prettyPricingComponent(getLLMPricing(llm)!.chat!)} {/*{!!llm.benchmark && <>benchmark: {llm.benchmark.cbaElo?.toLocaleString() || '(unk) '} CBA Elo
}*/} {llm.parameterSpecs?.length > 0 && <>options: {llm.parameterSpecs.map(ps => ps.paramId).join(', ')}
} {Object.keys(llm.initialParameters || {}).length > 0 && <>initial parameters: {JSON.stringify(llm.initialParameters, null, 2)}
} @@ -297,6 +351,57 @@ export function LLMOptionsModal(props: { id: DLLMId, onClose: () => void }) { + {/* Advanced: Pricing Overrides */} + + + + Pricing Override (for hypothetical cost tracking) + + + + + + + Reset + )} + slotProps={{ input: { min: 0, step: 0.01 } }} + sx={{ flex: 1 }} + /> + + + + + + + Reset + )} + sx={{ flex: 1 }} + /> + + + + } diff --git a/src/modules/llms/models-modal/ModelsList.tsx b/src/modules/llms/models-modal/ModelsList.tsx index 2295aaf86..a0e4f5862 100644 --- a/src/modules/llms/models-modal/ModelsList.tsx +++ b/src/modules/llms/models-modal/ModelsList.tsx @@ -8,7 +8,7 @@ import VisibilityOffOutlinedIcon from '@mui/icons-material/VisibilityOffOutlined import VisibilityOutlinedIcon from '@mui/icons-material/VisibilityOutlined'; import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types'; -import { DLLM, DLLMId, getLLMContextTokens, getLLMMaxOutputTokens, isLLMHidden, LLM_IF_ANT_PromptCaching, LLM_IF_GEM_CodeExecution, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn, LLM_IF_OAI_Json, LLM_IF_OAI_PromptCaching, LLM_IF_OAI_Realtime, LLM_IF_OAI_Reasoning, LLM_IF_OAI_Vision, LLM_IF_Outputs_Audio, LLM_IF_Outputs_Image, LLM_IF_Tools_WebSearch } from '~/common/stores/llms/llms.types'; +import { DLLM, DLLMId, getLLMContextTokens, getLLMMaxOutputTokens, getLLMPricing, isLLMHidden, LLM_IF_ANT_PromptCaching, LLM_IF_GEM_CodeExecution, LLM_IF_OAI_Complete, LLM_IF_OAI_Fn, LLM_IF_OAI_Json, LLM_IF_OAI_PromptCaching, LLM_IF_OAI_Realtime, LLM_IF_OAI_Reasoning, LLM_IF_OAI_Vision, LLM_IF_Outputs_Audio, LLM_IF_Outputs_Image, LLM_IF_Tools_WebSearch } from '~/common/stores/llms/llms.types'; import { GoodTooltip } from '~/common/components/GoodTooltip'; import { PhGearSixIcon } from '~/common/components/icons/phosphor/PhGearSixIcon'; import { STAR_EMOJI, StarredToggle, starredToggleStyle } from '~/common/components/StarIcons'; @@ -81,7 +81,7 @@ function ModelItem(props: { // derived const { llm, onModelClicked, onModelSetHidden /*, onModelSetStarred*/ } = props; - const seemsFree = !!llm.pricing?.chat?._isFree; + const seemsFree = !!getLLMPricing(llm)?.chat?._isFree; const isNotSymlink = !llm.label.startsWith('๐Ÿ”—'); diff --git a/src/modules/llms/vendors/openrouter/OpenRouterServiceSetup.tsx b/src/modules/llms/vendors/openrouter/OpenRouterServiceSetup.tsx index b9def18be..df839d2a7 100644 --- a/src/modules/llms/vendors/openrouter/OpenRouterServiceSetup.tsx +++ b/src/modules/llms/vendors/openrouter/OpenRouterServiceSetup.tsx @@ -7,6 +7,7 @@ import VisibilityOutlinedIcon from '@mui/icons-material/VisibilityOutlined'; import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types'; import { AlreadySet } from '~/common/components/AlreadySet'; import { FormInputKey } from '~/common/components/forms/FormInputKey'; +import { getLLMPricing } from '~/common/stores/llms/llms.types'; import { InlineError } from '~/common/components/InlineError'; import { Link } from '~/common/components/Link'; import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton'; @@ -53,7 +54,7 @@ export function OpenRouterServiceSetup(props: { serviceId: DModelsServiceId }) { const { updateLLM } = llmsStoreActions(); llms .filter(llm => llm.sId === props.serviceId) - .filter(llm => llm.pricing?.chat?._isFree === false) + .filter(llm => getLLMPricing(llm)?.chat?._isFree === false) .forEach(llm => updateLLM(llm.id, { userHidden: true })); }; diff --git a/src/modules/llms/vendors/openrouter/openrouter.vendor.ts b/src/modules/llms/vendors/openrouter/openrouter.vendor.ts index 9f92eb2da..20473ac3e 100644 --- a/src/modules/llms/vendors/openrouter/openrouter.vendor.ts +++ b/src/modules/llms/vendors/openrouter/openrouter.vendor.ts @@ -1,6 +1,8 @@ import type { IModelVendor } from '../IModelVendor'; import type { OpenAIAccessSchema } from '../../server/openai/openai.router'; +import { getLLMPricing } from '~/common/stores/llms/llms.types'; + import { ModelVendorOpenAI } from '../openai/openai.vendor'; @@ -51,7 +53,7 @@ export const ModelVendorOpenRouter: IModelVendor { const now = Date.now(); const elapsed = now - nextGenerationTs; - const wait = llm.pricing?.chat?._isFree + const wait = getLLMPricing(llm)?.chat?._isFree ? 5000 + 100 /* 5 seconds for free call, plus some safety margin */ : 100;