zack3d/big-agi
1
0
Fork 0
mirror of https://github.com/enricoros/big-AGI.git synced 2026-05-10 21:50:14 -07:00
Code Issues Packages Projects Releases Wiki Activity

LLMs: auto-inject image output

Browse Source
This commit is contained in:
Enrico Ros
2026-02-04 23:32:10 -08:00
parent 6fdff488a9
commit a10d0dcf5d
5 changed files with 59 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/stores/llms/llms.parameters.ts
+12 -12
View File
@@ -162,7 +162,7 @@ export const DModelParameterRegistry = {
},
},
llmVndAntWebFetch: {
llmVndAntWebFetch: { // implies: LLM_IF_Tools_WebSearch
label: 'Web Fetch',
type: 'enum',
description: 'Enable fetching content from web pages and PDFs',
@@ -170,7 +170,7 @@ export const DModelParameterRegistry = {
// No initialValue - undefined means off (same as 'off')
},
llmVndAntWebSearch: {
llmVndAntWebSearch: { // implies: LLM_IF_Tools_WebSearch
label: 'Web Search',
type: 'enum',
description: 'Enable web search for real-time information',
@@ -186,7 +186,7 @@ export const DModelParameterRegistry = {
// // No initialValue - undefined means off (tool search disabled)
// } as const,
llmVndGeminiAspectRatio: {
llmVndGeminiAspectRatio: { // implies: LLM_IF_Outputs_Image
label: 'Aspect Ratio',
type: 'enum',
description: 'Controls the aspect ratio of generated images',
@@ -211,7 +211,7 @@ export const DModelParameterRegistry = {
// requiredFallback: 'browser', // See `const _requiredParamId: DModelParameterId[]` in llms.parameters.ts for why custom params don't have required values at AIX invocation...
},
llmVndGeminiGoogleSearch: {
llmVndGeminiGoogleSearch: { // implies: LLM_IF_Tools_WebSearch
label: 'Google Search',
type: 'enum',
description: 'Enable Google Search grounding with optional time filter',
@@ -219,7 +219,7 @@ export const DModelParameterRegistry = {
// No initialValue - undefined means off
},
llmVndGeminiImageSize: { // [Gemini, 2025-11-20] Nano Banana launch
llmVndGeminiImageSize: { // implies: LLM_IF_Outputs_Image - [Gemini, 2025-11-20] Nano Banana launch
label: 'Image Size',
type: 'enum',
description: 'Controls the resolution of generated images',
@@ -290,7 +290,7 @@ export const DModelParameterRegistry = {
// No initialValue - undefined means high (thinking enabled, the default for K2.5)
},
llmVndMoonshotWebSearch: {
llmVndMoonshotWebSearch: { // implies: LLM_IF_Tools_WebSearch
label: 'Web Search',
type: 'enum',
description: 'Enable Kimi\'s $web_search builtin function for real-time web search ($0.005 per search)',
@@ -353,7 +353,7 @@ export const DModelParameterRegistry = {
requiredFallback: 'medium',
},
llmVndOaiWebSearchContext: {
llmVndOaiWebSearchContext: { // implies: LLM_IF_Tools_WebSearch
label: 'Search Context Size',
type: 'enum',
description: 'Amount of context retrieved from the web',
@@ -372,7 +372,7 @@ export const DModelParameterRegistry = {
initialValue: false,
},
llmVndOaiImageGeneration: {
llmVndOaiImageGeneration: { // implies: LLM_IF_Outputs_Image
label: 'Image Generation',
type: 'enum',
description: 'Image generation mode and quality',
@@ -401,7 +401,7 @@ export const DModelParameterRegistry = {
// requiredFallback: 'unfiltered',
},
llmVndOrtWebSearch: {
llmVndOrtWebSearch: { // implies: LLM_IF_Tools_WebSearch
label: 'Web Search',
type: 'enum',
description: 'Enable OpenRouter web search (uses native search for OpenAI/Anthropic, Exa for others)',
@@ -409,7 +409,7 @@ export const DModelParameterRegistry = {
// No initialValue - undefined means off
},
llmVndPerplexitySearchMode: {
llmVndPerplexitySearchMode: { // implies: LLM_IF_Tools_WebSearch
label: 'Search Mode',
type: 'enum',
description: 'Type of sources to search',
@@ -435,7 +435,7 @@ export const DModelParameterRegistry = {
// No initialValue - undefined means unfiltered
},
llmVndXaiWebSearch: {
llmVndXaiWebSearch: { // implies: LLM_IF_Tools_WebSearch
label: 'Web Search',
type: 'enum',
description: 'Enable web search for real-time information',
@@ -443,7 +443,7 @@ export const DModelParameterRegistry = {
// No initialValue - undefined means off (same as 'off')
},
llmVndXaiXSearch: {
llmVndXaiXSearch: { // implies: LLM_IF_Tools_WebSearch
label: 'X Search',
type: 'enum',
description: 'Enable X/Twitter search for social media content',
src/modules/llms/server/gemini/gemini.models.ts
+4 -4
View File
@@ -144,7 +144,7 @@ const gemini20FlashLitePricing: ModelDescriptionSchema['chatPrice'] = {
const IF_25 = [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json, LLM_IF_OAI_Reasoning, LLM_IF_GEM_CodeExecution, LLM_IF_OAI_PromptCaching];
const IF_30 = [...IF_25]; // Note: Gemini 3 Developer Guide recommends temperature=1.0, which is now set as the default via initialTemperature
const IF_30_IMG = [...IF_30, LLM_IF_Outputs_Image];
// NOTE: LLM_IF_Outputs_Image is auto-implied by llmsAutoImplyInterfaces() from image parameterSpecs (llmVndGeminiAspectRatio, llmVndGeminiImageSize)
const _knownGeminiModels: ({
@@ -181,7 +181,7 @@ const _knownGeminiModels: ({
labelOverride: 'Nano Banana Pro', // Marketing name for the technical model ID
isPreview: true,
chatPrice: gemini30ProImagePricing,
interfaces: IF_30_IMG,
interfaces: IF_30,
parameterSpecs: [
// { paramId: 'llmVndGeminiShowThoughts' },
{ paramId: 'llmVndGeminiGoogleSearch' },
@@ -197,7 +197,7 @@ const _knownGeminiModels: ({
// copied from symlink
isPreview: true,
chatPrice: gemini30ProImagePricing,
interfaces: IF_30_IMG,
interfaces: IF_30,
parameterSpecs: [
// { paramId: 'llmVndGeminiShowThoughts' },
{ paramId: 'llmVndGeminiGoogleSearch' },
@@ -351,7 +351,7 @@ const _knownGeminiModels: ({
labelOverride: 'Nano Banana',
deprecated: '2026-10-02',
chatPrice: { input: 0.30, output: undefined }, // Per pricing page: $0.30 text/image input, $0.039 per image output, but the text output is not stated
interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json, LLM_IF_Outputs_Image],
interfaces: [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json],
parameterSpecs: [{ paramId: 'llmVndGeminiAspectRatio' }],
benchmark: undefined, // Non-benchmarkable because generates images
},
src/modules/llms/server/listModels.dispatch.ts
+2 -2
View File
@@ -8,7 +8,7 @@ import { createDebugWireLogger } from '~/server/wire';
import { fetchJsonOrTRPCThrow } from '~/server/trpc/trpc.router.fetchers';
import type { ModelDescriptionSchema } from './llm.server.types';
import { llmsAutoInjectWebSearchInterface } from './models.mappings';
import { llmsAutoImplyInterfaces } from './models.mappings';
// protocol: Anthropic
@@ -71,7 +71,7 @@ export async function listModelsRunDispatch(access: AixAPI_Access, signal?: Abor
const dispatch = _listModelsCreateDispatch(access, signal);
const wireModels = await dispatch.fetchModels();
return dispatch.convertToDescriptions(wireModels)
.map(llmsAutoInjectWebSearchInterface); // unified way of auto-injecting cosmetic/derived IFs
.map(llmsAutoImplyInterfaces); // auto-inject implied IFs from parameterSpecs
}
src/modules/llms/server/models.mappings.ts
+40 -25
View File
@@ -1,37 +1,52 @@
import type { DModelInterfaceV1 } from '~/common/stores/llms/llms.types';
import type { DModelParameterId } from '~/common/stores/llms/llms.parameters';
import { LLM_IF_Tools_WebSearch } from '~/common/stores/llms/llms.types';
import { LLM_IF_Outputs_Image, LLM_IF_Tools_WebSearch } from '~/common/stores/llms/llms.types';
import type { ModelDescriptionSchema } from './llm.server.types';
// -- Auto-inject web search interface --
// -- Auto-inject implied model interfaces from parameterSpecs --
const _paramIdToInterface: { paramIds: DModelParameterId[], iface: DModelInterfaceV1 }[] = [
// Web search parameters -> LLM_IF_Tools_WebSearch
{
iface: LLM_IF_Tools_WebSearch,
paramIds: [
'llmVndAntWebFetch',
'llmVndAntWebSearch',
'llmVndGeminiGoogleSearch',
'llmVndMoonshotWebSearch',
'llmVndOaiWebSearchContext',
'llmVndOrtWebSearch',
'llmVndPerplexitySearchMode',
'llmVndXaiWebSearch',
'llmVndXaiXSearch',
],
},
// Image generation parameters -> LLM_IF_Outputs_Image
{
iface: LLM_IF_Outputs_Image,
paramIds: [
'llmVndGeminiAspectRatio',
'llmVndGeminiImageSize',
'llmVndOaiImageGeneration',
],
},
] as const;
/**
* Parameter IDs that imply the model supports web search capabilities.
* When any of these are present in parameterSpecs, LLM_IF_Tools_WebSearch is auto-added to interfaces.
* Auto-injects interfaces (e.g. WebSearch, Outputs_Image) for models whose parameterSpecs
* include parameter IDs that imply those capabilities.
*/
const _webSearchParamIds: string[] = [
'llmVndAntWebFetch',
'llmVndAntWebSearch',
'llmVndGeminiGoogleSearch',
'llmVndMoonshotWebSearch',
'llmVndOaiWebSearchContext',
'llmVndOrtWebSearch',
'llmVndPerplexitySearchMode',
'llmVndXaiWebSearch',
'llmVndXaiXSearch',
] as const satisfies DModelParameterId[];
export function llmsAutoImplyInterfaces(model: ModelDescriptionSchema): ModelDescriptionSchema {
if (!model.parameterSpecs?.length) return model;
/**
* Auto-injects LLM_IF_Tools_WebSearch for models that have web search/fetch parameters.
* Applied centrally in listModelsRunDispatch so individual vendors don't need to manage this.
*/
export function llmsAutoInjectWebSearchInterface(model: ModelDescriptionSchema): ModelDescriptionSchema {
const hasWebParams = model.parameterSpecs?.some(spec => _webSearchParamIds.includes(spec.paramId));
return (hasWebParams && !model.interfaces?.includes(LLM_IF_Tools_WebSearch)) ? {
...model,
interfaces: [...model.interfaces, LLM_IF_Tools_WebSearch],
} : model;
let interfaces = model.interfaces;
for (const { paramIds, iface } of _paramIdToInterface)
if (!interfaces.includes(iface) && model.parameterSpecs.some(spec => paramIds.includes(spec.paramId as DModelParameterId)))
interfaces = [...interfaces, iface];
return interfaces !== model.interfaces ? { ...model, interfaces } : model;
}
src/modules/llms/server/openai/models/openai.models.ts
+1 -1
View File
@@ -59,7 +59,7 @@ const IFS_GPT_AUDIO: DModelInterfaceV1[] = [LLM_IF_OAI_Chat, LLM_IF_Outputs_Audi
const IFS_CHAT_MIN: DModelInterfaceV1[] = [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json] as const;
const IFS_CHAT_CACHE: DModelInterfaceV1[] = [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json, LLM_IF_OAI_PromptCaching] as const;
const IFS_CHAT_CACHE_REASON: DModelInterfaceV1[] = [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Json, LLM_IF_OAI_PromptCaching, LLM_IF_OAI_Reasoning] as const;
// NOTE: 'LLM_IF_Tools_WebSearch' is auto-injected by llmsAutoInjectWebSearchInterface() for models with web search parameterSpecs - no need to add it manually
// NOTE: LLM_IF_Tools_WebSearch and LLM_IF_Outputs_Image are auto-injected by llmsAutoInjectInterfaces() by checking parameter specs, as such we don't need to add them here
// per-type parameter specs
const PS_DEEP_RESEARCH = [{ paramId: 'llmVndOaiWebSearchContext' as const, initialValue: 'medium', hidden: true } as const];