From 5cc48d24ec64c2a6461d1f6517684e5e903979ce Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Tue, 7 Apr 2026 04:23:31 -0700 Subject: [PATCH] AIX: Anthropic: Download Files (AIX hosted resource support) --- .../BlockPartHostedResource.tsx | 165 ++++++++++++++++++ .../fragments-content/ContentFragments.tsx | 11 ++ .../fragments-void/BlockPartPlaceholder.tsx | 2 +- src/common/stores/chat/chat.fragments.ts | 22 +++ src/common/stores/chat/chat.message.ts | 1 + src/common/stores/chat/chat.tokens.ts | 1 + src/modules/aix/client/ContentReassembler.ts | 35 +++- .../client/aix.client.chatGenerateRequest.ts | 14 ++ src/modules/aix/client/aix.client.ts | 1 + src/modules/aix/server/api/aix.wiretypes.ts | 10 ++ .../chatGenerate/ChatGenerateTransmitter.ts | 6 + .../parsers/IParticleTransmitter.ts | 4 + .../chatGenerate/parsers/anthropic.parser.ts | 19 +- .../llms/server/anthropic/anthropic.router.ts | 4 +- tools/develop/llm-parameter-sweep/sweep.ts | 1 + 15 files changed, 275 insertions(+), 21 deletions(-) create mode 100644 src/apps/chat/components/message/fragments-content/BlockPartHostedResource.tsx diff --git a/src/apps/chat/components/message/fragments-content/BlockPartHostedResource.tsx b/src/apps/chat/components/message/fragments-content/BlockPartHostedResource.tsx new file mode 100644 index 000000000..573f91035 --- /dev/null +++ b/src/apps/chat/components/message/fragments-content/BlockPartHostedResource.tsx @@ -0,0 +1,165 @@ +import * as React from 'react'; + +import TimeAgo from 'react-timeago'; + +import { Box, CircularProgress, IconButton, Sheet, Typography } from '@mui/joy'; +import AttachFileRoundedIcon from '@mui/icons-material/AttachFileRounded'; +import DownloadIcon from '@mui/icons-material/Download'; + +import type { AnthropicAccessSchema } from '~/modules/llms/server/anthropic/anthropic.access'; +import { findModelVendor } from '~/modules/llms/vendors/vendors.registry'; + +import type { ContentScaling } from '~/common/app.theme'; +import type { DLLMId } from '~/common/stores/llms/llms.types'; +import type { DMessageHostedResourcePart } from '~/common/stores/chat/chat.fragments'; +import { apiAsync, apiQuery } from '~/common/util/trpc.client'; +import { downloadBlob } from '~/common/util/downloadUtils'; +import { findModelsServiceOrNull, llmsStoreState } from '~/common/stores/llms/store-llms'; +import { humanReadableBytes } from '~/common/util/textUtils'; + + +/** + * Resolve Anthropic access credentials, preferring the generator's specific service + * (the one that created the file) and falling back to the first available Anthropic service. + */ +function _resolveAnthropicAccess(generatorLlmId?: DLLMId): AnthropicAccessSchema | null { + const vendor = findModelVendor('anthropic'); + if (!vendor) return null; + + const { llms, sources } = llmsStoreState(); + + // prefer the generator's service (the one that created the file) + if (generatorLlmId) { + const llm = llms.find(m => m.id === generatorLlmId); + if (llm) { + const service = findModelsServiceOrNull(llm.sId); + if (service?.vId === 'anthropic') + return vendor.getTransportAccess(service.setup); + } + } + + // fall back to the first available Anthropic service + const anthropicService = sources.find(s => s.vId === 'anthropic'); + if (!anthropicService) return null; + return vendor.getTransportAccess(anthropicService.setup); +} + + +function NoAccessChip(props: { fileId: string }) { + return ( + + + + {props.fileId} (no credentials) + + + ); +} + + +function AnthropicFileChip(props: { + access: AnthropicAccessSchema, + fileId: string, + contentScaling: ContentScaling, +}) { + + // state + const [downloading, setDownloading] = React.useState(false); + const [downloadError, setDownloadError] = React.useState(null); + + // props + const { access, fileId } = props; + + // external state + const { data: metadata, isLoading: metaLoading, error: metaError } = apiQuery.llmAnthropic.getFileMetadata.useQuery( + // metadata query - cached by React Query, staleTime: Infinity (file metadata is immutable) + { access, fileId }, + { staleTime: Infinity }, + ); + + + // derive display info from typed metadata + const fileName = metadata?.filename || fileId; + const displayName = fileName.length > 40 ? fileName.slice(0, 20) + '...' + fileName.slice(-15) : fileName; + + const handleDownload = React.useCallback(async () => { + setDownloading(true); + setDownloadError(null); + try { + const { base64Data, mimeType } = await apiAsync.llmAnthropic.downloadFile.query({ access, fileId }); + const binary = atob(base64Data); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) + bytes[i] = binary.charCodeAt(i); + const blob = new Blob([bytes], { type: mimeType }); + downloadBlob(blob, fileName); + } catch (error: any) { + setDownloadError(error?.message || 'Download failed'); + } finally { + setDownloading(false); + } + }, [access, fileId, fileName]); + + const hasError = !!metaError || !!downloadError; + + return ( + + + + + {metaLoading ? 'Loading...' : hasError ? `${displayName} - ${downloadError || 'Could not load file info'}` : displayName} + + {metadata && ( + + {humanReadableBytes(metadata.size_bytes)} - - {metadata.mime_type} + + )} + + + {downloading ? : } + + + ); +} + + +export function BlockPartHostedResource(props: { + hostedResourcePart: DMessageHostedResourcePart, + messageGeneratorLlmId?: string | null, + contentScaling: ContentScaling, +}) { + + const { resource } = props.hostedResourcePart; + + // memo state + const access = React.useMemo(() => { + return resource.via === 'anthropic' ? _resolveAnthropicAccess(props.messageGeneratorLlmId ?? undefined) : null; + }, [resource.via, props.messageGeneratorLlmId]); + + // only support Anthropic files for now + if (resource.via !== 'anthropic' || !access) + return ; + + return ( + + ); +} diff --git a/src/apps/chat/components/message/fragments-content/ContentFragments.tsx b/src/apps/chat/components/message/fragments-content/ContentFragments.tsx index e3e4b94e0..7bf1cb997 100644 --- a/src/apps/chat/components/message/fragments-content/ContentFragments.tsx +++ b/src/apps/chat/components/message/fragments-content/ContentFragments.tsx @@ -14,6 +14,7 @@ import type { ChatMessageTextPartEditState } from '../ChatMessage'; import { BlockEdit_TextFragment } from './BlockEdit_TextFragment'; import { BlockOpEmpty } from './BlockOpEmpty'; import { BlockPartError } from './BlockPartError'; +import { BlockPartHostedResource } from './BlockPartHostedResource'; import { BlockPartImageRef } from './BlockPartImageRef'; import { BlockPartModelAux } from '../fragments-void/BlockPartModelAux'; import { BlockPartPlaceholder } from '../fragments-void/BlockPartPlaceholder'; @@ -360,6 +361,16 @@ export function ContentFragments(props: { /> ); + case 'hosted_resource': + return ( + + ); + case '_pt_sentinel': return null; diff --git a/src/apps/chat/components/message/fragments-void/BlockPartPlaceholder.tsx b/src/apps/chat/components/message/fragments-void/BlockPartPlaceholder.tsx index 4c0841ab7..8cf028fb0 100644 --- a/src/apps/chat/components/message/fragments-void/BlockPartPlaceholder.tsx +++ b/src/apps/chat/components/message/fragments-void/BlockPartPlaceholder.tsx @@ -326,7 +326,7 @@ function ModelOperationChip(props: { {!!iTexts?.length && !!oTexts?.length && } {!!oTexts?.length && oTexts.map((t, i) => ( - + {i > 0 && '\n'}{t} ))} diff --git a/src/common/stores/chat/chat.fragments.ts b/src/common/stores/chat/chat.fragments.ts index acb6c490a..038c71d9a 100644 --- a/src/common/stores/chat/chat.fragments.ts +++ b/src/common/stores/chat/chat.fragments.ts @@ -35,6 +35,7 @@ export type DMessageContentFragment = _DMessageFragmentWrapper<'content', | DMessageImageRefPart // large image | DMessageToolInvocationPart // shown to dev only, signature of the llm function call | DMessageToolResponsePart // shown to dev only, response of the llm + | DMessageHostedResourcePart // provider-hosted resource (e.g. Anthropic file from Skills) with download affordance | DMessageErrorPart // red message, e.g. non-content application issues | _SentinelPart >; @@ -231,6 +232,15 @@ export type DMessageToolResponsePart = { type DMessageToolEnvironment = 'upstream' | 'server' | 'client'; type DMessageToolCodeExecutor = 'gemini_auto_inline' | 'code_interpreter'; + +/** Hosted resource - a provider-hosted resource (e.g. Anthropic container file from Skills/code execution). */ +export type DMessageHostedResourcePart = { + pt: 'hosted_resource'; + resource: + | { via: 'anthropic', fileId: string, containerId?: string }; +}; + + type DVoidModelAnnotationsPart = { pt: 'annotations', annotations: readonly DVoidWebCitation[], @@ -390,6 +400,10 @@ export function isToolResponseFunctionCallPart(part: DMessageContentFragment['pa return part.pt === 'tool_response' && part.response.type === 'function_call'; } +export function isHostedResourcePart(part: DMessageContentFragment['part']): part is DMessageHostedResourcePart { + return part.pt === 'hosted_resource'; +} + export function isAnnotationsPart(part: DMessageVoidFragment['part']) { return part.pt === 'annotations'; } @@ -433,6 +447,10 @@ export function create_CodeExecutionResponse_ContentFragment(id: string, error: return _createContentFragment(_create_CodeExecutionResponse_Part(id, error, result, executor, environment)); } +export function createHostedResourceContentFragment(resource: DMessageHostedResourcePart['resource']): DMessageContentFragment { + return _createContentFragment({ pt: 'hosted_resource', resource }); +} + function _createContentFragment(part: DMessageContentFragment['part']): DMessageContentFragment { return { ft: 'content', fId: agiId('chat-dfragment' /* -content */), part }; } @@ -677,6 +695,9 @@ function _duplicate_Part): void { + + // Break text accumulation, as we will display this as it happens (parting text, if needed) + this.S._textFragmentIndex = null; + + switch (op.kind) { + + case 'vnd.ant.file': + this._pushFragment(createHostedResourceContentFragment({ + via: 'anthropic', + fileId: op.fileId, + ...(op.containerId ? { containerId: op.containerId } : {}), + })); + break; + + default: + const _exhaustiveCheck: never = op.kind; + console.warn('[ContentReassembler] onAppendHostedResource: unrecognized hosted resource kind', { op }); + break; + } + } + private onAddUrlCitation(urlc: Extract): void { const { title, url, num: refNumber, from: startIndex, to: endIndex, text: textSnippet, pubTs } = urlc; diff --git a/src/modules/aix/client/aix.client.chatGenerateRequest.ts b/src/modules/aix/client/aix.client.chatGenerateRequest.ts index 1e1b82c0d..0518bf310 100644 --- a/src/modules/aix/client/aix.client.chatGenerateRequest.ts +++ b/src/modules/aix/client/aix.client.chatGenerateRequest.ts @@ -102,6 +102,7 @@ export async function aixCGR_SystemMessage_FromDMessageOrThrow( case 'image_ref': case 'tool_invocation': case 'tool_response': + case 'hosted_resource': case 'error': case '_pt_sentinel': console.warn('[DEV] aixCGR_systemMessageFromInstruction: unexpected System Content fragment', { sFragment }); @@ -362,6 +363,7 @@ export async function aixCGR_ChatSequence_FromDMessagesOrThrow( case 'error': case 'tool_invocation': case 'tool_response': + case 'hosted_resource': console.warn('aixCGR_FromDMessages: unexpected Non-User fragment part type', (uFragment.part as any).pt); break; @@ -522,6 +524,18 @@ export async function aixCGR_ChatSequence_FromDMessagesOrThrow( modelMessage.parts.push(_vnd ? { ...aPart, _vnd } : aPart); break; + case 'hosted_resource': + // Hosted resources are download-only artifacts - emit a text placeholder for model context + // NOTE: disabled for now - we don't know how usefult this hinting it, and we're clashing with proprietary Anthropic prompts + // modelMessage.parts.push({ + // pt: 'text', + // text: `[Output file: ${aPart.resource.via === 'anthropic' ? aPart.resource.fileId : 'unknown'}]`, + // // ...(aPart.resource.via === 'anthropic' && { + // // _vnd: { anthropic: { containerUpload: { fileId: aPart.resource.fileId, ...(aPart.resource.containerId && { containerId: aPart.resource.containerId }) } } }, + // // }), + // }); + break; + default: const _exhaustiveCheck: never = aPart; console.warn('aixCGR_FromDMessages: unexpected Assistant fragment part', aPart); diff --git a/src/modules/aix/client/aix.client.ts b/src/modules/aix/client/aix.client.ts index bd28ef795..af7502fae 100644 --- a/src/modules/aix/client/aix.client.ts +++ b/src/modules/aix/client/aix.client.ts @@ -432,6 +432,7 @@ function _llToL2Simple({ fragments, generator }: AixChatGenerateContent_LL, dest case 'ph': // placeholder - ignored case 'reference': // impossible case 'image_ref': // impossible + case 'hosted_resource': // impossible - download-only artifact case 'tool_response': // impossible - stopped at the invocation already case '_pt_sentinel': // impossible break; diff --git a/src/modules/aix/server/api/aix.wiretypes.ts b/src/modules/aix/server/api/aix.wiretypes.ts index 1c432b979..1eeebfd5a 100644 --- a/src/modules/aix/server/api/aix.wiretypes.ts +++ b/src/modules/aix/server/api/aix.wiretypes.ts @@ -101,6 +101,13 @@ export namespace AixWire_Parts { gemini: z.object({ thoughtSignature: z.string().optional(), }).optional(), + // NOTE: we do NOT use this mechanism for per-vendor customization/ALT for parts + // anthropic: z.object({ + // containerUpload: z.object({ + // fileId: z.string(), + // containerId: z.string().optional(), + // }).optional(), + // }).optional(), }).optional(), // _vnd: z.record(z.string(), z.unknown()).optional(), @@ -749,6 +756,9 @@ export namespace AixWire_Particles { */ | { p: /*'mo'*/ 'vp', opId: string, text: string, mot: 'search-web' | 'gen-image' | 'code-exec', state?: 'done' | 'error', parentOpId?: string, iTexts?: string[], oTexts?: string[] } | { p: 'urlc', title: string, url: string, num?: number, from?: number, to?: number, text?: string, pubTs?: number } // url citation - pubTs: publication timestamp + | { p: 'hres' } & ( // hosted resource - provider-hosted resource + | { kind: 'vnd.ant.file', fileId: string, containerId?: string } + ) | { p: 'svs' } & ( // set vendor state - vendor-specific opaque protocol state | { vendor: 'anthropic', state: { container: { id: string; expiresAt: string } } } // message-level | { vendor: 'gemini', state: { thoughtSignature: string } } // fragment-level diff --git a/src/modules/aix/server/dispatch/chatGenerate/ChatGenerateTransmitter.ts b/src/modules/aix/server/dispatch/chatGenerate/ChatGenerateTransmitter.ts index edeb21ba9..5f249c26f 100644 --- a/src/modules/aix/server/dispatch/chatGenerate/ChatGenerateTransmitter.ts +++ b/src/modules/aix/server/dispatch/chatGenerate/ChatGenerateTransmitter.ts @@ -347,6 +347,12 @@ export class ChatGenerateTransmitter implements IParticleTransmitter { this._queueParticleS(); } + /** Appends a hosted resource (e.g. Anthropic container file) - inline content between text fragments */ + appendHostedResource(hres: Extract) { + this.endMessagePart(); + this.transmissionQueue.push(hres); + } + /** * Undocumented, internal, as the IPartTransmitter callers will call setDialectTerminatingIssue instead diff --git a/src/modules/aix/server/dispatch/chatGenerate/parsers/IParticleTransmitter.ts b/src/modules/aix/server/dispatch/chatGenerate/parsers/IParticleTransmitter.ts index 75e35910a..fdee6e9ba 100644 --- a/src/modules/aix/server/dispatch/chatGenerate/parsers/IParticleTransmitter.ts +++ b/src/modules/aix/server/dispatch/chatGenerate/parsers/IParticleTransmitter.ts @@ -45,6 +45,9 @@ export interface IParticleTransmitter { /** Appends an image generated by the model */ appendImageInline(mimeType: string, base64Data: string, label: string, generator: string, prompt: string, hintSkipResize?: boolean): void; + /** Appends a hosted resource (e.g. Anthropic container file) - inline content between text fragments */ + appendHostedResource(hres: Extract): void; + /** * Creates a FC part, flushing the previous one if needed, and starts adding data to it * @param id if null [Gemini], a new id will be generated to keep it linked to future tool responses @@ -66,6 +69,7 @@ export interface IParticleTransmitter { /** Adds a URL citation part */ appendUrlCitation(title: string, url: string, citationNumber?: number, startIndex?: number, endIndex?: number, textSnippet?: string, pubTs?: number): void; + // Special // /** Sends control particles right away, such as aix-info/aix-retry-reset/... control particles */ diff --git a/src/modules/aix/server/dispatch/chatGenerate/parsers/anthropic.parser.ts b/src/modules/aix/server/dispatch/chatGenerate/parsers/anthropic.parser.ts index db42f8d2a..4982b9ba8 100644 --- a/src/modules/aix/server/dispatch/chatGenerate/parsers/anthropic.parser.ts +++ b/src/modules/aix/server/dispatch/chatGenerate/parsers/anthropic.parser.ts @@ -887,14 +887,11 @@ function _handleCBS_CodeExecutionToolResult(pt: IParticleTransmitter, block: Ext const codeExecFailed = block.content.return_code !== 0; pt.sendOperationState('code-exec', codeExecFailed ? `Code executed` /* was: failed */ : 'Code executed', { opId, state: codeExecFailed ? 'error' : 'done', ...oTexts.length ? { oTexts } : undefined }); - // add text if there are generated files in content array (e.g. generated from a skill) - const fileIds: string[] = []; + // emit structured hosted resource references for generated files (e.g. from a skill) if (Array.isArray(block.content?.content)) for (const ob of block.content.content) if (ob.type === 'code_execution_output' && ob.file_id) - fileIds.push(ob.file_id); - if (fileIds.length > 0) - pt.appendText(`\n\n⚡ Code executed by Skill\n${fileIds.map(id => `\n📎 File: \`${id}\``).join('')}\n`); + pt.appendHostedResource({ p: 'hres', kind: 'vnd.ant.file', fileId: ob.file_id }); break; } @@ -922,14 +919,11 @@ function _handleCBS_BashCodeExecutionToolResult(pt: IParticleTransmitter, block: const bashFailed = block.content.return_code !== 0; pt.sendOperationState('code-exec', bashFailed ? `Bash executed` /* was: failed */ : 'Bash executed', { opId, state: bashFailed ? 'error' : 'done', ...oTexts.length ? { oTexts } : undefined }); - // add text if there are generated files in content array - const fileIds: string[] = []; + // emit structured hosted resource references for generated files if (Array.isArray(block.content.content)) for (const ob of block.content.content) if (ob.type === 'bash_code_execution_output' && ob.file_id) - fileIds.push(ob.file_id); - if (fileIds.length > 0) - pt.appendText(`\n\n⚡ Bash executed by Skill\n${fileIds.map(id => `\n📎 File: \`${id}\``).join('')}\n`); + pt.appendHostedResource({ p: 'hres', kind: 'vnd.ant.file', fileId: ob.file_id }); break; case 'bash_code_execution_tool_result_error': @@ -972,9 +966,8 @@ function _handleCBS_TextEditorCodeExecutionToolResult(pt: IParticleTransmitter, } function _handleCBS_ContainerUpload(pt: IParticleTransmitter, block: Extract<_ContentBlock, { type: 'container_upload' }>, containerId: string | undefined): void { - // Container upload - this is when a Skill has generated a file - file_id can be used with the Files API to download the file - pt.appendText(`\n\n⚡ File uploaded to container (${containerId || 'none'})\n\n📎 File: \`${block.file_id}\`\n\n`); - // TODO: Future enhancement - could trigger automatic file download here using the Files API with content_block.file_id, or offer an UI way to do so through a dedicated part/block? + // Container upload - Skill has generated a file, emit as a downloadable hosted resource + pt.appendHostedResource({ p: 'hres', kind: 'vnd.ant.file', fileId: block.file_id, ...(containerId ? { containerId } : {}) }); } function _handleCBS_ToolSearchToolResult(pt: IParticleTransmitter, block: Extract<_ContentBlock, { type: 'tool_search_tool_result' }>): void { diff --git a/src/modules/llms/server/anthropic/anthropic.router.ts b/src/modules/llms/server/anthropic/anthropic.router.ts index 0ad0ba754..e11e57dd0 100644 --- a/src/modules/llms/server/anthropic/anthropic.router.ts +++ b/src/modules/llms/server/anthropic/anthropic.router.ts @@ -72,7 +72,7 @@ export const llmAnthropicRouter = createTRPCRouter({ downloadable: z.boolean().optional(), })) .query(async ({ input: { access, fileId } }) => { - return await anthropicGETOrThrow(access, `${ANTHROPIC_API_PATHS.files}/${fileId}`, { enableSkills: true }); + return await anthropicGETOrThrow(access, `${ANTHROPIC_API_PATHS.files}/${fileId}`, { enableSkills: true, enableCodeExecution: true }); }), /* [Anthropic] download file content - for Skills-generated files */ @@ -82,7 +82,7 @@ export const llmAnthropicRouter = createTRPCRouter({ fileId: z.string(), })) .query(async ({ input: { access, fileId } }) => { - const { headers, url } = anthropicAccess(access, `${ANTHROPIC_API_PATHS.files}/${fileId}/content`, { enableSkills: true }); + const { headers, url } = anthropicAccess(access, `${ANTHROPIC_API_PATHS.files}/${fileId}/content`, { enableSkills: true, enableCodeExecution: true }); const response = await fetchResponseOrTRPCThrow({ url, headers, name: 'Anthropic' }); return { base64Data: Buffer.from(await response.arrayBuffer()).toString('base64'), diff --git a/tools/develop/llm-parameter-sweep/sweep.ts b/tools/develop/llm-parameter-sweep/sweep.ts index 60947b262..77de2841e 100644 --- a/tools/develop/llm-parameter-sweep/sweep.ts +++ b/tools/develop/llm-parameter-sweep/sweep.ts @@ -324,6 +324,7 @@ class SweepCollectorTransmitter implements IParticleTransmitter { appendAutoText_weak(textChunk: string): void { this.text += textChunk; } appendAudioInline(_mimeType: string, _base64Data: string, _label: string, _generator: string, _durationMs: number): void { /* no-op */ } appendImageInline(_mimeType: string, _base64Data: string, _label: string, _generator: string, _prompt: string): void { /* no-op */ } + appendHostedResource(_hres: any): void { /* no-op */ } startFunctionCallInvocation(_id: string | null, _functionName: string, _expectedArgsFmt: 'incr_str' | 'json_object', _args: string | object | null): void { /* no-op */ } appendFunctionCallInvocationArgs(_id: string | null, _argsJsonChunk: string): void { /* no-op */ } addCodeExecutionInvocation(_id: string | null, _language: string, _code: string, _author: 'gemini_auto_inline' | 'code_interpreter'): void { /* no-op */ }