diff --git a/src/common/components/icons/vendors/BedrockIcon.tsx b/src/common/components/icons/vendors/BedrockIcon.tsx new file mode 100644 index 000000000..a65a42c42 --- /dev/null +++ b/src/common/components/icons/vendors/BedrockIcon.tsx @@ -0,0 +1,13 @@ +import * as React from 'react'; + +import { SvgIcon, SvgIconProps } from '@mui/joy'; + +export function BedrockIcon(props: SvgIconProps) { + return + {/* AWS-style smile arrow mark */} + + + + + ; +} diff --git a/src/modules/aix/server/api/aix.wiretypes.ts b/src/modules/aix/server/api/aix.wiretypes.ts index b4e9aaec0..12efa9beb 100644 --- a/src/modules/aix/server/api/aix.wiretypes.ts +++ b/src/modules/aix/server/api/aix.wiretypes.ts @@ -4,6 +4,7 @@ import * as z from 'zod/v4'; import type { DMessageToolResponsePart } from '~/common/stores/chat/chat.fragments'; import { anthropicAccessSchema } from '~/modules/llms/server/anthropic/anthropic.access'; +import { bedrockAccessSchema } from '~/modules/llms/server/bedrock/bedrock.access'; import { geminiAccessSchema } from '~/modules/llms/server/gemini/gemini.access'; import { ollamaAccessSchema } from '~/modules/llms/server/ollama/ollama.access'; import { openAIAccessSchema } from '~/modules/llms/server/openai/openai.access'; @@ -415,6 +416,7 @@ export namespace AixWire_API { export const Access_schema = z.discriminatedUnion('dialect', [ anthropicAccessSchema, + bedrockAccessSchema, geminiAccessSchema, ollamaAccessSchema, openAIAccessSchema, @@ -482,6 +484,9 @@ export namespace AixWire_API { vndAntWebFetch: z.enum(['auto']).optional(), vndAntWebSearch: z.enum(['auto']).optional(), + // Bedrock + vndBedrockInvokeAPI: z.enum(['invoke-anthropic', 'converse']).optional(), + // Gemini vndGeminiAspectRatio: z.enum(['1:1', '2:3', '3:2', '3:4', '4:3', '9:16', '16:9', '21:9']).optional(), vndGeminiCodeExecution: z.enum(['auto']).optional(), diff --git a/src/modules/backend/backend.router.ts b/src/modules/backend/backend.router.ts index e0bb6479d..f1da41a05 100644 --- a/src/modules/backend/backend.router.ts +++ b/src/modules/backend/backend.router.ts @@ -51,6 +51,7 @@ export const backendRouter = createTRPCRouter({ hasLlmAlibaba: !!env.ALIBABA_API_KEY || !!env.ALIBABA_API_HOST, hasLlmAnthropic: !!env.ANTHROPIC_API_KEY, hasLlmAzureOpenAI: !!env.AZURE_OPENAI_API_KEY && !!env.AZURE_OPENAI_API_ENDPOINT, + hasLlmBedrock: !!env.BEDROCK_ACCESS_KEY_ID && !!env.BEDROCK_SECRET_ACCESS_KEY, hasLlmDeepseek: !!env.DEEPSEEK_API_KEY, hasLlmGemini: !!env.GEMINI_API_KEY, hasLlmGroq: !!env.GROQ_API_KEY, diff --git a/src/modules/backend/store-backend-capabilities.ts b/src/modules/backend/store-backend-capabilities.ts index 7bbbf77dd..438afee01 100644 --- a/src/modules/backend/store-backend-capabilities.ts +++ b/src/modules/backend/store-backend-capabilities.ts @@ -11,6 +11,7 @@ export interface BackendCapabilities { hasLlmAlibaba: boolean; hasLlmAnthropic: boolean; hasLlmAzureOpenAI: boolean; + hasLlmBedrock: boolean; hasLlmDeepseek: boolean; hasLlmGemini: boolean; hasLlmGroq: boolean; @@ -51,6 +52,7 @@ const useBackendCapabilitiesStore = create()( // initial values hasLlmAlibaba: false, hasLlmAnthropic: false, + hasLlmBedrock: false, hasLlmAzureOpenAI: false, hasLlmDeepseek: false, hasLlmGemini: false, diff --git a/src/modules/llms/components/LLMVendorIcon.tsx b/src/modules/llms/components/LLMVendorIcon.tsx index 67f384386..7db866e9b 100644 --- a/src/modules/llms/components/LLMVendorIcon.tsx +++ b/src/modules/llms/components/LLMVendorIcon.tsx @@ -12,6 +12,7 @@ import { PhRobot } from '~/common/components/icons/phosphor/PhRobot'; import { AlibabaCloudIcon } from '~/common/components/icons/vendors/AlibabaCloudIcon'; import { AnthropicIcon } from '~/common/components/icons/vendors/AnthropicIcon'; import { AzureIcon } from '~/common/components/icons/vendors/AzureIcon'; +import { BedrockIcon } from '~/common/components/icons/vendors/BedrockIcon'; import { DeepseekIcon } from '~/common/components/icons/vendors/DeepseekIcon'; import { GeminiIcon } from '~/common/components/icons/vendors/GeminiIcon'; import { GroqIcon } from '~/common/components/icons/vendors/GroqIcon'; @@ -38,6 +39,7 @@ const vendorIcons: Record> alibaba: AlibabaCloudIcon, anthropic: AnthropicIcon, azure: AzureIcon, + bedrock: BedrockIcon, deepseek: DeepseekIcon, googleai: GeminiIcon, groq: GroqIcon, diff --git a/src/modules/llms/components/LLMVendorIconSprite.tsx b/src/modules/llms/components/LLMVendorIconSprite.tsx index cabd0fce4..838c2ee5a 100644 --- a/src/modules/llms/components/LLMVendorIconSprite.tsx +++ b/src/modules/llms/components/LLMVendorIconSprite.tsx @@ -15,6 +15,7 @@ const VI: Record = { alibaba: 'vi-alibaba', anthropic: 'vi-anthropic', azure: 'vi-azure', + bedrock: 'vi-bedrock', deepseek: 'vi-deepseek', googleai: 'vi-googleai', groq: 'vi-groq', @@ -66,6 +67,16 @@ export const VendorIconSpriteMemo = React.memo(function VendorIconSprite() { + + + {/* AWS-style smile arrow mark */} + + + + + + + diff --git a/src/modules/llms/components/LLMVendorSetup.tsx b/src/modules/llms/components/LLMVendorSetup.tsx index 62b3c03e1..6d87153e1 100644 --- a/src/modules/llms/components/LLMVendorSetup.tsx +++ b/src/modules/llms/components/LLMVendorSetup.tsx @@ -9,6 +9,7 @@ import { findModelVendor, ModelVendorId } from '../vendors/vendors.registry'; import { AlibabaServiceSetup } from '../vendors/alibaba/AlibabaServiceSetup'; import { AnthropicServiceSetup } from '../vendors/anthropic/AnthropicServiceSetup'; import { AzureServiceSetup } from '../vendors/azure/AzureServiceSetup'; +import { BedrockServiceSetup } from '../vendors/bedrock/BedrockServiceSetup'; import { DeepseekAIServiceSetup } from '../vendors/deepseek/DeepseekAIServiceSetup'; import { GeminiServiceSetup } from '../vendors/gemini/GeminiServiceSetup'; import { GroqServiceSetup } from '../vendors/groq/GroqServiceSetup'; @@ -35,6 +36,7 @@ const vendorSetupComponents: Record m.id === anthropicId); +} + +function _llmBedrockToAnthropicModelId(bedrockBaseId: string): string | undefined { + if (!bedrockBaseId.startsWith('anthropic.')) return undefined; + // e.g. anthropic.claude-opus-4-6-v1 -> claude-opus-4-6; anthropic.claude-opus-4-5-20251101-v1:0 -> claude-opus-4-5-20251101 + return bedrockBaseId.slice('anthropic.'.length).replace(/-v\d+(:\d+)?$/, ''); +} + +// Bedrock supports these interfaces (no Anthropic web tools) +const _BEDROCK_ANT_IF_ALLOWLIST: ReadonlySet = new Set([ + LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_OAI_Reasoning, + LLM_IF_ANT_PromptCaching, +] as const); + +// NOTE: llmVndAntInfSpeed not available on Bedrock, llmVndAntWebFetch/llmVndAntSkills not available +const _BEDROCK_ANT_PARAM_ALLOWLIST: ReadonlySet = new Set([ + // supported + 'llmVndAnt1MContext', + 'llmVndAntEffort', + 'llmVndAntThinkingBudget', + // Not supported by Bedrock + // 'llmVndAntInfSpeed', // Bad Request - speed: Extra inputs are not permitted + // 'llmVndAntSkills', // code execution is not supported: https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool#platform-availability + // 'llmVndAntWebFetch', // Bad Request - tools.0: Input tag 'web_fetch_20250910' found using 'type' does not match any of the expected tags: 'bash_20250124', 'custom', 'text_editor_20250124', 'text_editor_20250429', 'text_editor_20250728', 'web_search_20250305' + // 'llmVndAntWebSearch', // Bedrock should support web search, but we get 'Bad Request' if the 'web_search_20250305' tool is added +] as const satisfies DModelParameterId[]); + +/** Strip unsupported interfaces and params from an Anthropic model for Bedrock */ +export function llmBedrockStripAnthropicMDS(model: ModelDescriptionSchema): ModelDescriptionSchema { + if (!model.parameterSpecs && !model.interfaces.some(i => !_BEDROCK_ANT_IF_ALLOWLIST.has(i))) + return model; // nothing to filter + return { + ...model, + interfaces: model.interfaces.filter(i => _BEDROCK_ANT_IF_ALLOWLIST.has(i)), + ...(model.parameterSpecs ? { + parameterSpecs: model.parameterSpecs.filter(spec => _BEDROCK_ANT_PARAM_ALLOWLIST.has(spec.paramId)), + } : {}), + }; +} + + // -- Anthropic-through-OpenRouter Vendor Lookup -- const _ORT_ANT_IF_ALLOWLIST: ReadonlySet = new Set([ diff --git a/src/modules/llms/server/bedrock/bedrock.access.ts b/src/modules/llms/server/bedrock/bedrock.access.ts new file mode 100644 index 000000000..6a937b3bd --- /dev/null +++ b/src/modules/llms/server/bedrock/bedrock.access.ts @@ -0,0 +1,106 @@ +/** + * Isomorphic AWS Bedrock API access - SigV4 signing via aws4fetch. + * + * This module provides the access schema and signing logic for AWS Bedrock API calls. + * It supports both the `bedrock` control plane (model listing) and `bedrock-runtime` + * data plane (model invocation). + * + * Authentication uses explicit AWS credentials only (no credential chain) for + * Edge Runtime compatibility. aws4fetch auto-detects service and region from URLs. + */ + +import * as z from 'zod/v4'; +import { TRPCError } from '@trpc/server'; + +import { AwsClient } from 'aws4fetch'; + +import { env } from '~/server/env.server'; + + +// --- Schema --- + +export type BedrockAccessSchema = z.infer; +export const bedrockAccessSchema = z.object({ + dialect: z.literal('bedrock'), + bedrockAccessKeyId: z.string().trim(), + bedrockSecretAccessKey: z.string().trim(), + bedrockSessionToken: z.string().trim().nullable(), + bedrockRegion: z.string().trim(), + clientSideFetch: z.boolean(), +}); + + +// --- Credential & Region Resolution --- + +/** + * Resolve all Bedrock access config: credentials (all-or-none) + region. + * If the user provides an access key, all credentials come from user; otherwise all from env. + */ +export function bedrockServerConfig(access: BedrockAccessSchema) { + const userProvided = !!access.bedrockAccessKeyId; + const accessKeyId = userProvided ? access.bedrockAccessKeyId : (env.BEDROCK_ACCESS_KEY_ID || ''); + const secretAccessKey = userProvided ? access.bedrockSecretAccessKey : (env.BEDROCK_SECRET_ACCESS_KEY || ''); + const sessionToken = (userProvided ? access.bedrockSessionToken : env.BEDROCK_SESSION_TOKEN) || undefined; + const region = (userProvided ? access.bedrockRegion : env.BEDROCK_REGION) || 'us-east-1'; + + if (!accessKeyId || !secretAccessKey) + throw new TRPCError({ + code: 'BAD_REQUEST', + message: 'Missing AWS credentials. Add your Access Key ID and Secret Access Key on the UI (Models Setup) or server side (your deployment).', + }); + + return { accessKeyId, secretAccessKey, sessionToken, region }; +} + + +// --- URLs --- + +export function bedrockURLRuntime(region: string, modelId: string, streaming: boolean): string { + const action = streaming ? 'invoke-with-response-stream' : 'invoke'; + return `https://bedrock-runtime.${region}.amazonaws.com/model/${encodeURIComponent(modelId)}/${action}`; +} + +export function bedrockURLControlPlane(region: string, path: string): string { + return `https://bedrock.${region}.amazonaws.com${path}`; +} + + +// --- Bedrock Access (async SigV4) --- + +/** + * Signs a request for AWS Bedrock using SigV4 via aws4fetch. + * Returns { headers, url } like anthropicAccess/geminiAccess, but async due to SigV4 signing. + */ +export async function bedrockAccessAsync( + access: BedrockAccessSchema, + method: 'GET' | 'POST', + url: string, + body?: object, +): Promise<{ headers: HeadersInit; url: string }> { + + const { accessKeyId, secretAccessKey, sessionToken, region } = bedrockServerConfig(access); + + const awsClient = new AwsClient({ + service: 'bedrock', // Bedrock uses 'bedrock' as the SigV4 service name for both control plane and runtime + accessKeyId, + secretAccessKey, + sessionToken, + region, + }); + + // sign the request - uses SubtleCrypto + const signedRequest = await awsClient.sign(url, { + method, + headers: { + 'Accept': 'application/json', + ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + } satisfies RequestInit); + + // extract headers from the signed Request + const headers: Record = {}; + signedRequest.headers.forEach((value, key) => headers[key] = value); + + return { headers, url }; +} diff --git a/src/modules/llms/server/bedrock/bedrock.models.ts b/src/modules/llms/server/bedrock/bedrock.models.ts new file mode 100644 index 000000000..2f09c7e4d --- /dev/null +++ b/src/modules/llms/server/bedrock/bedrock.models.ts @@ -0,0 +1,261 @@ +/** + * Bedrock model definitions and ID mapping. + * + * Bedrock model IDs differ from Anthropic direct API IDs: + * - Anthropic direct: 'claude-opus-4-6' + * - Bedrock foundation: 'anthropic.claude-opus-4-6-v1' + * - Bedrock inference profile: 'us.anthropic.claude-opus-4-6-v1' / 'global.anthropic.claude-opus-4-6-v1' + * + * For non-Anthropic models (via Converse API), models are identified by their + * Bedrock model IDs directly. + */ + +import * as z from 'zod/v4'; + +import type { ModelDescriptionSchema } from '../llm.server.types'; + +import { anthropicInjectVariants, llmBedrockFindAnthropicModel, llmBedrockStripAnthropicMDS } from '../anthropic/anthropic.models'; +import { LLM_IF_ANT_PromptCaching, LLM_IF_OAI_Chat, LLM_IF_OAI_Fn, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types'; + + +// --- Bedrock API Wire Types --- + +export namespace BedrockWire_API_Models_List { + + // ListFoundationModels response + + const _FoundationModel_schema = z.object({ + modelId: z.string(), + modelName: z.string(), + providerName: z.enum(['Amazon', 'Anthropic', 'Cohere', 'DeepSeek', 'Google', 'Luma AI', 'Meta', 'MiniMax', 'Mistral AI', 'Moonshot AI', 'NVIDIA', 'OpenAI', 'Qwen', 'Stability AI', 'Z.AI']).or(z.string()), + inputModalities: z.array(z.enum(['TEXT', 'IMAGE', 'EMBEDDING', 'VIDEO', 'SPEECH']).or(z.string())), + outputModalities: z.array(z.enum(['TEXT', 'IMAGE', 'EMBEDDING', 'VIDEO', 'SPEECH']).or(z.string())), + responseStreamingSupported: z.boolean().nullable().optional(), + inferenceTypesSupported: z.array(z.enum(['ON_DEMAND', 'INFERENCE_PROFILE', 'PROVISIONED']).or(z.string())).optional(), + modelLifecycle: z.object({ + status: z.enum(['ACTIVE', 'LEGACY']).or(z.string()), + }).optional(), + // Converse API metadata (present on newer models, null on legacy) + converse: z.object({ + maxTokensMaximum: z.number().nullable().optional(), + reasoningSupported: z.object({ + embedded: z.boolean(), + }).nullable().optional(), + systemRoleSupported: z.boolean().optional(), + userImageTypesSupported: z.array(z.enum(['png', 'jpeg', 'gif', 'webp']).or(z.string())).optional(), + }).nullable().optional(), + }); + + export const FoundationModelsResponse_schema = z.object({ + modelSummaries: z.array(_FoundationModel_schema), + }); + + // ListInferenceProfiles response + + const _InferenceProfile_schema = z.object({ + inferenceProfileId: z.string(), + inferenceProfileName: z.string(), + description: z.string().optional(), + type: z.enum(['SYSTEM_DEFINED', 'APPLICATION']).or(z.string()), + status: z.enum(['ACTIVE', 'LEGACY']).or(z.string()).optional(), + models: z.array(z.object({ + modelArn: z.string().optional(), + })).optional(), + }); + + export const InferenceProfilesResponse_schema = z.object({ + inferenceProfileSummaries: z.array(_InferenceProfile_schema), + nextToken: z.string().optional().nullable(), + }); + +} + + +// --- Model ID Helpers --- + +const _REGION_PREFIX_RE = /^(us|eu|global|jp|apac)\./; + +/** Strip region prefix (us., global., etc.) from a Bedrock model ID, returning a bedrockBaseId */ +function _stripRegionPrefix(bedrockModelId: string): string { + return bedrockModelId.replace(_REGION_PREFIX_RE, ''); +} + +/** Extract region prefix (us, global, etc.) or undefined if none */ +function _extractRegionPrefix(bedrockModelId: string): string | undefined { + return _REGION_PREFIX_RE.exec(bedrockModelId)?.[1]; +} + +/** Check if a Bedrock model ID is an Anthropic model */ +function _seemsAnthropicBedrockModel(bedrockModelId: string): boolean { + return _stripRegionPrefix(bedrockModelId).startsWith('anthropic.'); +} + + +// --- Model Description Building --- + +/** + * Convert Bedrock model listings into ModelDescriptionSchema[]. + * Enriches known Anthropic models with hardcoded definitions. + * Non-Anthropic models get basic descriptions with Converse API marker. + */ +export function bedrockModelsToDescriptions( + foundationModels: z.infer, + inferenceProfiles: z.infer, +): ModelDescriptionSchema[] { + + // Collect unique model IDs from both sources + const modelMap = new Map(); + + // Foundation Models + for (const fm of foundationModels.modelSummaries) { + // exclude legacy models + if (fm.modelLifecycle?.status === 'LEGACY') continue; + + // excludes embedding, image gen, video gen, speech-only + if (!fm.inputModalities?.includes('TEXT') || !fm.outputModalities?.includes('TEXT')) continue; + + // denylist '..match..' + if (['rerank'].some(match => fm.modelId.includes(match))) continue; + + modelMap.set(fm.modelId, { + id: fm.modelId, + label: fm.modelName, + provider: fm.providerName, + isInferenceProfile: false, + streaming: fm.responseStreamingSupported ?? true, + converseMaxTokens: fm.converse?.maxTokensMaximum ?? null, + converseImageTypes: fm.converse?.userImageTypesSupported ?? [], + }); + } + + // Inference Profiles + for (const ip of inferenceProfiles.inferenceProfileSummaries) { + // exclude legacy models + if (ip.status && ip.status !== 'ACTIVE') continue; + + // denylist 'start..' + const baseId = _stripRegionPrefix(ip.inferenceProfileId); + if (['stability.'].some(start => baseId.startsWith(start))) continue; + + // check if there's a matching foundation model (not anthropic, we map them differently) + const foundationMeta = modelMap.get(baseId); + // if (!_seemsAnthropicBedrockModel(ip.inferenceProfileId) && !foundationMeta) + // console.log('[Bedrock] No matching foundation model for inference profile', ip.inferenceProfileId); + + modelMap.set(ip.inferenceProfileId, { + id: ip.inferenceProfileId, + label: ip.inferenceProfileName, + provider: _extractProvider(ip.inferenceProfileId), + isInferenceProfile: true, + streaming: foundationMeta?.streaming ?? true, + converseMaxTokens: foundationMeta?.converseMaxTokens ?? null, + converseImageTypes: foundationMeta?.converseImageTypes ?? [], + }); + } + + // -> ModelDescriptionSchema[], with Anthropic thinking variants injected inline + const descriptions: ModelDescriptionSchema[] = []; + for (const [modelId, meta] of modelMap) { + + // Known Anthropic models: enrich with hardcoded definitions + inject thinking variants + const antModel = llmBedrockFindAnthropicModel(_stripRegionPrefix(modelId)); + if (antModel) { + const isProfile = !!_extractRegionPrefix(modelId); + // Inject variants (returns [variant, base] or [base] if no variant) + const withVariants = anthropicInjectVariants([], antModel); + for (const variant of withVariants) { + const label = isProfile ? _profileLabel(variant.label, modelId) : variant.label; + descriptions.push({ ...variant, id: modelId, label }); + } + continue; + } + + // Unknown models - these will NOT be accessible, hence the '🚧'. We show them just in case, but maybe we shall not + const isAnthropic = _seemsAnthropicBedrockModel(modelId); + const hasVision = meta.converseImageTypes.length > 0; + descriptions.push({ + id: modelId, + label: '🚧 ' + (meta.isInferenceProfile ? _profileLabel(meta.label, modelId) : meta.label), + description: `${meta.provider} model on AWS Bedrock${isAnthropic ? '' : ' (Converse API)'}`, + contextWindow: isAnthropic ? 200000 : (meta.converseMaxTokens ? meta.converseMaxTokens * 2 : 32768), + maxCompletionTokens: isAnthropic ? 64000 : (meta.converseMaxTokens ?? 4096), + interfaces: + isAnthropic ? [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision, LLM_IF_OAI_Fn, LLM_IF_ANT_PromptCaching] + : hasVision ? [LLM_IF_OAI_Chat, LLM_IF_OAI_Vision] + : [LLM_IF_OAI_Chat], + hidden: true, // not in our known models DB — hide until verified usable + }); + } + + // Filter interfaces and params to Bedrock-supported subset, then sort + const filtered = descriptions.map(llmBedrockStripAnthropicMDS); + filtered.sort(_bedrockModelSort); + return filtered; +} + + +// --- Helpers --- + +/** Build a profile label: strip redundant region prefix from name, append `· Region` suffix (omit for global) */ +function _profileLabel(name: string, modelId: string): string { + const prefix = _extractRegionPrefix(modelId) ?? 'regional'; + // Strip leading "US ", "GLOBAL ", etc. from the AWS-provided name + const cleanName = name.replace(/^(US|EU|GLOBAL|JP|APAC)\s+/i, ''); + // if (prefix === 'global') return cleanName; + // Display-friendly casing: US, EU, Global, etc. + const displayPrefix = prefix === 'global' ? 'Global' : prefix.toUpperCase(); + // return `${cleanName} [${displayPrefix}]`; + return `${cleanName} · ${displayPrefix}`; +} + +/** Extract provider name from inference profile ID */ +function _extractProvider(profileId: string): string { + // Format: region.provider.model-name -> Provider + const parts = _stripRegionPrefix(profileId).split('.'); + return parts[0] ? parts[0].charAt(0).toUpperCase() + parts[0].slice(1) : 'Unknown'; +} + +/** Sort: Anthropic first, then family > class > variant (thinking before plain) > region */ +function _bedrockModelSort(a: ModelDescriptionSchema, b: ModelDescriptionSchema): number { + const aIsAnt = _seemsAnthropicBedrockModel(a.id); + const bIsAnt = _seemsAnthropicBedrockModel(b.id); + if (aIsAnt && !bIsAnt) return -1; + if (!aIsAnt && bIsAnt) return 1; + + // Within Anthropic: sort by family precedence + const familyPrecedence = ['-4-7-', '-4-6', '-4-5-', '-4-1-', '-4-', '-3-7-', '-3-5-', '-3-']; + const classPrecedence = ['-opus-', '-sonnet-', '-haiku-']; + + const getFamilyIdx = (id: string) => familyPrecedence.findIndex(f => id.includes(f)); + const getClassIdx = (id: string) => classPrecedence.findIndex(c => id.includes(c)); + + const familyA = getFamilyIdx(a.id); + const familyB = getFamilyIdx(b.id); + if (familyA !== familyB) return (familyA === -1 ? 999 : familyA) - (familyB === -1 ? 999 : familyB); + + const classA = getClassIdx(a.id); + const classB = getClassIdx(b.id); + if (classA !== classB) return (classA === -1 ? 999 : classA) - (classB === -1 ? 999 : classB); + + // Thinking/adaptive variants before plain (idVariant present = variant) + const aIsVariant = !!a.idVariant; + const bIsVariant = !!b.idVariant; + if (aIsVariant && !bIsVariant) return -1; + if (!aIsVariant && bIsVariant) return 1; + + // Prefer global > us > eu > regional + const prefixOrder = ['global', 'us', 'eu', 'jp', 'apac']; + const getPrefixIdx = (id: string) => { + const prefix = _extractRegionPrefix(id); + return prefix ? prefixOrder.indexOf(prefix) : 999; + }; + return getPrefixIdx(a.id) - getPrefixIdx(b.id); +} diff --git a/src/modules/llms/server/bedrock/bedrock.router.ts b/src/modules/llms/server/bedrock/bedrock.router.ts new file mode 100644 index 000000000..637f59ca5 --- /dev/null +++ b/src/modules/llms/server/bedrock/bedrock.router.ts @@ -0,0 +1,33 @@ +import * as z from 'zod/v4'; + +import { createTRPCRouter, edgeProcedure } from '~/server/trpc/trpc.server'; + +import { ListModelsResponse_schema } from '../llm.server.types'; +import { listModelsRunDispatch } from '../listModels.dispatch'; + +import { bedrockAccessSchema } from './bedrock.access'; + + +// Input Schemas + +const _listModelsInputSchema = z.object({ + access: bedrockAccessSchema, +}); + + +// Router + +export const llmBedrockRouter = createTRPCRouter({ + + /* [Bedrock] list models - fetches from ListFoundationModels + ListInferenceProfiles */ + listModels: edgeProcedure + .input(_listModelsInputSchema) + .output(ListModelsResponse_schema) + .query(async ({ input: { access }, signal }) => { + + const models = await listModelsRunDispatch(access, signal); + + return { models }; + }), + +}); diff --git a/src/modules/llms/server/listModels.dispatch.ts b/src/modules/llms/server/listModels.dispatch.ts index 5671ecbe4..d1ffbef8c 100644 --- a/src/modules/llms/server/listModels.dispatch.ts +++ b/src/modules/llms/server/listModels.dispatch.ts @@ -15,6 +15,10 @@ import { llmDevValidateParameterSpecs_DEV, llmsAutoImplyInterfaces } from './mod import { anthropicInjectVariants, anthropicValidateModelDefs_DEV, AnthropicWire_API_Models_List, hardcodedAnthropicModels, llmsAntCreatePlaceholderModel } from './anthropic/anthropic.models'; import { ANTHROPIC_API_PATHS, anthropicAccess } from './anthropic/anthropic.access'; +// protocol: Bedrock +import { bedrockAccessAsync, bedrockURLControlPlane, bedrockServerConfig } from './bedrock/bedrock.access'; +import { bedrockModelsToDescriptions, BedrockWire_API_Models_List } from './bedrock/bedrock.models'; + // protocol: Gemini import { GeminiWire_API_Models_List } from '~/modules/aix/server/dispatch/wiretypes/gemini.wiretypes'; import { geminiAccess } from './gemini/gemini.access'; @@ -160,6 +164,46 @@ function _listModelsCreateDispatch(access: AixAPI_Access, signal?: AbortSignal): }); } + case 'bedrock': { + return createListModelsDispatch({ + fetchModels: async () => { + + // construct URLs by region + const { region } = bedrockServerConfig(access); + const fmUrl = bedrockURLControlPlane(region, '/foundation-models?byInferenceType=ON_DEMAND'); + const ipUrl = bedrockURLControlPlane(region, '/inference-profiles?typeEquals=SYSTEM_DEFINED&maxResults=1000'); + + // sign and fetch both lists in parallel - and unbreak on failure + const [fmResponse, ipResponse] = await Promise.all([ + // Foundation Models + bedrockAccessAsync(access, 'GET', fmUrl, undefined) + .then(fmAccess => fetchJsonOrTRPCThrow({ ...fmAccess, signal, name: 'Bedrock/FM' })) + .catch(error => { + console.warn('[Bedrock] Foundation Models list failed:', (error as Error)?.message || error); + return { modelSummaries: [] }; + }), + // Inference Profiles + bedrockAccessAsync(access, 'GET', ipUrl, undefined) + .then(ipAccess => fetchJsonOrTRPCThrow({ ...ipAccess, signal, name: 'Bedrock/IP' })) + .catch(error => { + console.warn('[Bedrock] Inference Profiles list failed:', (error as Error)?.message || error); + return { inferenceProfileSummaries: [] }; + }), + ]); + + _wire?.logResponse(fmResponse); + _wire?.logResponse(ipResponse); + + return { + foundationModels: BedrockWire_API_Models_List.FoundationModelsResponse_schema.parse(fmResponse), + inferenceProfiles: BedrockWire_API_Models_List.InferenceProfilesResponse_schema.parse(ipResponse), + }; + }, + convertToDescriptions: ({ foundationModels, inferenceProfiles }) => + bedrockModelsToDescriptions(foundationModels, inferenceProfiles), + }); + } + case 'gemini': { return createListModelsDispatch({ fetchModels: async () => { diff --git a/src/modules/llms/vendors/bedrock/BedrockRegionAutocomplete.tsx b/src/modules/llms/vendors/bedrock/BedrockRegionAutocomplete.tsx new file mode 100644 index 000000000..fb197db84 --- /dev/null +++ b/src/modules/llms/vendors/bedrock/BedrockRegionAutocomplete.tsx @@ -0,0 +1,113 @@ +import * as React from 'react'; + +import { Autocomplete, AutocompleteOption, Box, FormControl, FormHelperText, FormLabel, Typography } from '@mui/joy'; + +import { Link } from '~/common/components/Link'; + + +interface BedrockRegion { + id: string; + label: string; +} + +// Bedrock-supported regions enabled by default (no opt-in required). +// Users can type any region (freeSolo) for opt-in regions. +// Sources: https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html +// https://docs.aws.amazon.com/global-infrastructure/latest/regions/aws-regions.html#regions-opt-in-status +const BEDROCK_REGIONS: BedrockRegion[] = [ + // US + { id: 'us-east-1', label: 'N. Virginia' }, + { id: 'us-east-2', label: 'Ohio' }, + { id: 'us-west-1', label: 'N. California' }, + { id: 'us-west-2', label: 'Oregon' }, + // Canada + { id: 'ca-central-1', label: 'Canada' }, + // South America + { id: 'sa-east-1', label: 'São Paulo' }, + // Europe + { id: 'eu-central-1', label: 'Frankfurt' }, + { id: 'eu-north-1', label: 'Stockholm' }, + { id: 'eu-west-1', label: 'Ireland' }, + { id: 'eu-west-2', label: 'London' }, + { id: 'eu-west-3', label: 'Paris' }, + // Asia Pacific + { id: 'ap-northeast-1', label: 'Tokyo' }, + { id: 'ap-northeast-2', label: 'Seoul' }, + { id: 'ap-northeast-3', label: 'Osaka' }, + { id: 'ap-south-1', label: 'Mumbai' }, + { id: 'ap-southeast-1', label: 'Singapore' }, + { id: 'ap-southeast-2', label: 'Sydney' }, +]; + +const DEFAULT_REGION = 'us-west-2'; + + +export function BedrockRegionAutocomplete(props: { + value: string; + onChange: (value: string) => void; +}) { + + // local input state for freeSolo + const [inputValue, setInputValue] = React.useState(props.value || DEFAULT_REGION); + + // sync input when value prop changes externally + React.useEffect(() => { + setInputValue(props.value || DEFAULT_REGION); + }, [props.value]); + + // handlers + const handleChange = React.useCallback((_event: unknown, newValue: string | BedrockRegion | null) => { + if (newValue === null) + props.onChange(DEFAULT_REGION); + else if (typeof newValue === 'string') + props.onChange(newValue); + else + props.onChange(newValue.id); + }, [props]); + + const handleInputChange = React.useCallback((_event: unknown, newInputValue: string, reason: string) => { + if (reason !== 'input') + return; + setInputValue(newInputValue); + props.onChange(newInputValue); + }, [props]); + + return ( + + + + AWS Region + + + see regions + + + + freeSolo + openOnFocus + clearOnEscape + placeholder='us-west-2' + options={BEDROCK_REGIONS} + getOptionKey={(option) => typeof option === 'string' ? option : option.id} + getOptionLabel={(option) => typeof option === 'string' ? option : option.id} + isOptionEqualToValue={(option, val) => option.id === (typeof val === 'string' ? val : val.id)} + value={BEDROCK_REGIONS.find(r => r.id === props.value) ?? (props.value || null)} + onChange={handleChange} + inputValue={inputValue} + onInputChange={handleInputChange} + renderOption={(optionProps, option) => { + const { key, ...rest } = optionProps as any; + return ( + + {option.id} + {option.label} + + ); + }} + slotProps={{ + root: { sx: { boxShadow: 'none' } }, + }} + /> + + ); +} \ No newline at end of file diff --git a/src/modules/llms/vendors/bedrock/BedrockServiceSetup.tsx b/src/modules/llms/vendors/bedrock/BedrockServiceSetup.tsx new file mode 100644 index 000000000..c2ed8e74a --- /dev/null +++ b/src/modules/llms/vendors/bedrock/BedrockServiceSetup.tsx @@ -0,0 +1,108 @@ +import * as React from 'react'; + +import { Box, IconButton, Typography } from '@mui/joy'; +import UnfoldLessIcon from '@mui/icons-material/UnfoldLess'; +import UnfoldMoreIcon from '@mui/icons-material/UnfoldMore'; + +import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types'; +import { ClaudeCrabIcon } from '~/common/components/icons/vendors/ClaudeCrabIcon'; +import { FormInputKey } from '~/common/components/forms/FormInputKey'; +import { InlineError } from '~/common/components/InlineError'; +import { Link } from '~/common/components/Link'; +import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton'; + +import { ApproximateCosts } from '../ApproximateCosts'; +import { useLlmUpdateModels } from '../../llm.client.hooks'; +import { useServiceSetup } from '../useServiceSetup'; + +import { BedrockRegionAutocomplete } from './BedrockRegionAutocomplete'; +import { isValidBedrockAccessKeyId, isValidBedrockSecretAccessKey, ModelVendorBedrock } from './bedrock.vendor'; + + +export function BedrockServiceSetup(props: { serviceId: DModelsServiceId }) { + + // external state + const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, updateSettings } = + useServiceSetup(props.serviceId, ModelVendorBedrock); + + // derived state + const { bedrockAccessKeyId, bedrockSecretAccessKey, bedrockSessionToken, bedrockRegion } = serviceAccess; + const needsUserKey = !serviceHasCloudTenantConfig; + + // advanced mode + // const advanced = useToggleableBoolean(!!bedrockSessionToken); + const [showSetupInstructions, setShowSetupInstructions] = React.useState(false); + + const accessKeyValid = isValidBedrockAccessKeyId(bedrockAccessKeyId); + const secretKeyValid = isValidBedrockSecretAccessKey(bedrockSecretAccessKey); + const accessKeyError = (!!bedrockAccessKeyId) && !accessKeyValid; + const secretKeyError = (!!bedrockSecretAccessKey) && !secretKeyValid; + const shallFetchSucceed = bedrockAccessKeyId ? (accessKeyValid && secretKeyValid) : !needsUserKey; + + // fetch models + const { isFetching, refetch, isError, error } = + useLlmUpdateModels(!serviceHasLLMs && shallFetchSucceed, service); + + return <> + + + + + + Access Claude models through your AWS Bedrock account. + Requires an IAM Access Key with Bedrock permissions. + {showSetupInstructions && ( + +
  • Open the AWS IAM Console
    • +
  • Go to Users -> Create user (e.g. big-agi-bedrock, no console access)
    • +
  • Attach the AmazonBedrockFullAccess policy
    • +
  • Open the user -> Security credentials -> Create access key
    • +
  • Select "Application running outside AWS" -> Create
    • +
  • Copy the Access Key ID and Secret Access Key (shown only once)
    • +
  • Enter them below with your AWS Region
    • +     + )} +     + setShowSetupInstructions(on => !on)}> + {showSetupInstructions ? : } + +     +     + + updateSettings({ bedrockAccessKeyId: value })} + required={needsUserKey} isError={accessKeyError} + placeholder='AKIA...' + /> + + updateSettings({ bedrockSecretAccessKey: value })} + required={needsUserKey} isError={secretKeyError} + placeholder='wJalr...' + /> + + updateSettings({ bedrockRegion: value })} + /> + + {/*{advanced.on && updateSettings({ bedrockSessionToken: text })}*/} + {/*/>}*/} + + + {/**/} + + {isError && } + + ; +} diff --git a/src/modules/llms/vendors/bedrock/bedrock.vendor.ts b/src/modules/llms/vendors/bedrock/bedrock.vendor.ts new file mode 100644 index 000000000..74a469dda --- /dev/null +++ b/src/modules/llms/vendors/bedrock/bedrock.vendor.ts @@ -0,0 +1,57 @@ +import { apiAsync } from '~/common/util/trpc.client'; + +import type { BedrockAccessSchema } from '../../server/bedrock/bedrock.access'; +import type { IModelVendor } from '../IModelVendor'; + + +// validation +export const isValidBedrockAccessKeyId = (key?: string) => !!key && key.length >= 16; +export const isValidBedrockSecretAccessKey = (key?: string) => !!key && key.length >= 16; + +export interface DBedrockServiceSettings { + bedrockAccessKeyId: string; + bedrockSecretAccessKey: string; + bedrockSessionToken: string; + bedrockRegion: string; + csf?: boolean; +} + +export const ModelVendorBedrock: IModelVendor = { + id: 'bedrock', + name: 'AWS Bedrock', + displayRank: 14, + displayGroup: 'cloud', + location: 'cloud', + brandColor: '#FF9900', // AWS orange + instanceLimit: 1, + hasServerConfigKey: 'hasLlmBedrock', + + // NOTE: CSF not supported: Bedrock has no CORS headers + csfAvailable: _csfBedrockAvailable, + + // functions + initializeSetup: () => ({ + bedrockAccessKeyId: '', + bedrockSecretAccessKey: '', + bedrockSessionToken: '', + bedrockRegion: 'us-west-2', + csf: false, + }), + + getTransportAccess: (partialSetup): BedrockAccessSchema => ({ + dialect: 'bedrock', + bedrockAccessKeyId: partialSetup?.bedrockAccessKeyId || '', + bedrockSecretAccessKey: partialSetup?.bedrockSecretAccessKey || '', + bedrockSessionToken: partialSetup?.bedrockSessionToken || null, + bedrockRegion: partialSetup?.bedrockRegion || 'us-west-2', + clientSideFetch: _csfBedrockAvailable(partialSetup) && !!partialSetup?.csf, + }), + + // List Models + rpcUpdateModelsOrThrow: async (access) => await apiAsync.llmBedrock.listModels.query({ access }), + +}; + +function _csfBedrockAvailable(s?: Partial) { + return !!s?.bedrockAccessKeyId && !!s?.bedrockSecretAccessKey; +} diff --git a/src/modules/llms/vendors/vendors.registry.ts b/src/modules/llms/vendors/vendors.registry.ts index 998fb2565..7d733586a 100644 --- a/src/modules/llms/vendors/vendors.registry.ts +++ b/src/modules/llms/vendors/vendors.registry.ts @@ -3,6 +3,7 @@ import type { AixAPI_Access } from '~/modules/aix/server/api/aix.wiretypes'; import { ModelVendorAlibaba } from './alibaba/alibaba.vendor'; import { ModelVendorAnthropic } from './anthropic/anthropic.vendor'; import { ModelVendorAzure } from './azure/azure.vendor'; +import { ModelVendorBedrock } from './bedrock/bedrock.vendor'; import { ModelVendorDeepseek } from './deepseek/deepseekai.vendor'; import { ModelVendorGemini } from './gemini/gemini.vendor'; import { ModelVendorGroq } from './groq/groq.vendor'; @@ -26,6 +27,7 @@ export type ModelVendorId = | 'alibaba' | 'anthropic' | 'azure' + | 'bedrock' | 'deepseek' | 'googleai' | 'groq' @@ -48,6 +50,7 @@ const MODEL_VENDOR_REGISTRY: Record = { alibaba: ModelVendorAlibaba, anthropic: ModelVendorAnthropic, azure: ModelVendorAzure, + bedrock: ModelVendorBedrock, deepseek: ModelVendorDeepseek, googleai: ModelVendorGemini, groq: ModelVendorGroq, diff --git a/src/server/env.server.ts b/src/server/env.server.ts index fcf3f4d8f..3c66d3e4a 100644 --- a/src/server/env.server.ts +++ b/src/server/env.server.ts @@ -53,6 +53,12 @@ export const env = createEnv({ ANTHROPIC_API_KEY: z.string().optional(), ANTHROPIC_API_HOST: z.url().optional(), + // LLM: AWS Bedrock (using 2 or 3 auth values) + BEDROCK_ACCESS_KEY_ID: z.string().optional(), + BEDROCK_SECRET_ACCESS_KEY: z.string().optional(), + BEDROCK_SESSION_TOKEN: z.string().optional(), // required with the other 2 on corporate accounts sometimes + BEDROCK_REGION: z.string().optional(), + // LLM: Deepseek AI DEEPSEEK_API_KEY: z.string().optional(), diff --git a/src/server/trpc/trpc.router-edge.ts b/src/server/trpc/trpc.router-edge.ts index 4a29d1f5a..fb5182277 100644 --- a/src/server/trpc/trpc.router-edge.ts +++ b/src/server/trpc/trpc.router-edge.ts @@ -5,6 +5,7 @@ import { aixRouter } from '~/modules/aix/server/api/aix.router'; import { backendRouter } from '~/modules/backend/backend.router'; import { googleSearchRouter } from '~/modules/google/search.router'; import { llmAnthropicRouter } from '~/modules/llms/server/anthropic/anthropic.router'; +import { llmBedrockRouter } from '~/modules/llms/server/bedrock/bedrock.router'; import { llmGeminiRouter } from '~/modules/llms/server/gemini/gemini.router'; import { llmOllamaRouter } from '~/modules/llms/server/ollama/ollama.router'; import { llmOpenAIRouter } from '~/modules/llms/server/openai/openai.router'; @@ -19,6 +20,7 @@ export const appRouterEdge = createTRPCRouter({ backend: backendRouter, googleSearch: googleSearchRouter, llmAnthropic: llmAnthropicRouter, + llmBedrock: llmBedrockRouter, llmGemini: llmGeminiRouter, llmOllama: llmOllamaRouter, llmOpenAI: llmOpenAIRouter,