AIX: broaden Docs support

src/modules/aix/client/aix.client.chatGenerateRequest.ts

@@ -2,7 +2,7 @@ import { getImageAsset } from '~/modules/dblobs/dblobs.images';
 
 import { DLLM, LLM_IF_HOTFIX_NoStream, LLM_IF_HOTFIX_StripImages, LLM_IF_HOTFIX_Sys0ToUsr0 } from '~/common/stores/llms/llms.types';
 import { DMessage, DMessageRole, DMetaReferenceItem, MESSAGE_FLAG_AIX_SKIP, MESSAGE_FLAG_VND_ANT_CACHE_AUTO, MESSAGE_FLAG_VND_ANT_CACHE_USER, messageHasUserFlag } from '~/common/stores/chat/chat.message';
-import { DMessageFragment, DMessageImageRefPart, isContentOrAttachmentFragment, isTextContentFragment, isToolResponseFunctionCallPart } from '~/common/stores/chat/chat.fragments';
+import { DMessageFragment, DMessageImageRefPart, isAttachmentFragment, isContentOrAttachmentFragment, isDocPart, isTextContentFragment, isToolResponseFunctionCallPart } from '~/common/stores/chat/chat.fragments';
 import { Is } from '~/common/util/pwaUtils';
 import { LLMImageResizeMode, resizeBase64ImageIfNeeded } from '~/common/util/imageUtils';
 
@@ -80,9 +80,9 @@ export async function aixCGR_SystemMessage_FromDMessageOrThrow(
   for (const fragment of systemInstruction.fragments) {
     if (isTextContentFragment(fragment)) {
       sm.parts.push(fragment.part);
-    }
-    // TODO: handle other types of fragments if needed, such as the 'doc' type
-    else {
+    } else if (isAttachmentFragment(fragment) && isDocPart(fragment.part)) {
+      sm.parts.push(fragment.part);
+    } else {
       if (process.env.NODE_ENV === 'development')
         throw new Error('[DEV] aixCGR_systemMessageFromInstruction: unexpected system fragment');
       console.warn('[DEV] aixCGR_systemMessageFromInstruction: unexpected system fragment:', fragment);

src/modules/aix/server/api/aix.wiretypes.ts

@@ -224,6 +224,7 @@ export namespace AixWire_Content {
   export const SystemInstruction_schema = z.object({
     parts: z.array(z.discriminatedUnion('pt', [
       AixWire_Parts.TextPart_schema,
+      AixWire_Parts.DocPart_schema, // Jan 10, 2025: added support for Docs in AIX system
       AixWire_Parts.MetaCacheControl_schema,
     ])),
   });

src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts

@@ -1,6 +1,6 @@
 import type { OpenAIDialects } from '~/modules/llms/server/openai/openai.router';
 
-import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
+import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixParts_DocPart, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
 import { OpenAIWire_API_Chat_Completions, OpenAIWire_ContentParts, OpenAIWire_Messages } from '../../wiretypes/openai.wiretypes';
 
 
@@ -21,6 +21,7 @@ const hotFixOnlySupportN1 = true;
 const hotFixPreferArrayUserContent = true;
 const hotFixForceImageContentPartOpenAIDetail: 'auto' | 'low' | 'high' = 'high';
 const hotFixSquashTextSeparator = '\n\n\n---\n\n\n';
+const approxSystemMessageJoiner = '\n\n---\n\n';
 
 
 type TRequest = OpenAIWire_API_Chat_Completions.Request;
@@ -198,17 +199,35 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | null, chat
   // Transform the chat messages into OpenAI's format (an array of 'system', 'user', 'assistant', and 'tool' messages)
   const chatMessages: TRequestMessages = [];
 
-  // Convert the system message
+  // Convert the system message - single-part stay as-is and multi-part (text or doc) are flattened to a string
+  const msg0TextParts: OpenAIWire_ContentParts.TextContentPart[] = [];
   systemMessage?.parts.forEach((part) => {
-    if (part.pt === 'meta_cache_control') {
-      // ignore this hint - openai doesn't support this yet
-    } else
-      chatMessages.push({
-        role: !hotFixOpenAIo1Family ? 'system' : 'developer', // NOTE: o1Family in this case is not o1-preview as it's sporting the Sys0ToUsr0 hotfix
-        content: part.text, /*, name: _optionalParticipantName */
-      });
+    switch (part.pt) {
+      case 'text':
+        msg0TextParts.push(OpenAIWire_ContentParts.TextContentPart(part.text));
+        break;
+
+      case 'doc':
+        msg0TextParts.push(_toApproximateOpenAIDocPart(part));
+        break;
+
+      case 'meta_cache_control':
+        // ignore this hint - openai doesn't support this yet
+        break;
+
+      default:
+        throw new Error(`Unsupported part type in System message: ${(part as any).pt}`);
+    }
   });
 
+  // Add the system message
+  if (msg0TextParts.length)
+    chatMessages.push({
+      role: !hotFixOpenAIo1Family ? 'system' : 'developer', // NOTE: o1Family in this case is not o1-preview as it's sporting the Sys0ToUsr0 hotfix
+      content: _toApproximateOpanAIFlattenSystemMessage(msg0TextParts),
+    });
+
+
   // Convert the messages
   for (const { parts, role } of chatSequence) {
     switch (role) {
@@ -218,18 +237,8 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | null, chat
           const currentMessage = chatMessages[chatMessages.length - 1];
           switch (part.pt) {
 
-            case 'doc':
             case 'text':
-              // Implementation notes:
-              // - doc is rendered as a simple text part, but enclosed in a markdow block
-              // - TODO: consider better representation - we use the 'legacy' markdown encoding here,
-              //    but we may as well support different ones (e.g. XML) in the future
-              const textContentString =
-                part.pt === 'text' ? part.text
-                  : /* doc */ part.data.text.startsWith('```') ? part.data.text
-                    : `\`\`\`${part.ref || ''}\n${part.data.text}\n\`\`\`\n`;
-
-              const textContentPart = OpenAIWire_ContentParts.TextContentPart(textContentString);
+              const textContentPart = OpenAIWire_ContentParts.TextContentPart(part.text);
 
               // Append to existing content[], or new message
               if (currentMessage?.role === 'user' && Array.isArray(currentMessage.content))
@@ -238,6 +247,16 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | null, chat
                 chatMessages.push({ role: 'user', content: hotFixPreferArrayUserContent ? [textContentPart] : textContentPart.text });
               break;
 
+            case 'doc':
+              const docContentPart = _toApproximateOpenAIDocPart(part);
+
+              // Append to existing content[], or new message
+              if (currentMessage?.role === 'user' && Array.isArray(currentMessage.content))
+                currentMessage.content.push(docContentPart);
+              else
+                chatMessages.push({ role: 'user', content: hotFixPreferArrayUserContent ? [docContentPart] : docContentPart.text });
+              break;
+
             case 'inline_image':
               // create a new OpenAIWire_ImageContentPart
               const { mimeType, base64 } = part;
@@ -431,3 +450,25 @@ function _toOpenAIInReferenceToText(irt: AixParts_MetaInReferenceToPart): string
   return `CONTEXT: The user is referring to these ${items.length} in particular:\n\n${
     items.map((text, index) => formatItem(text, index)).join(allShort ? '\n' : '\n\n')}`;
 }
+
+
+// Approximate conversions
+
+function _toApproximateOpanAIFlattenSystemMessage(texts: OpenAIWire_ContentParts.TextContentPart[]): string {
+  return texts.map(text => text.text).join(approxSystemMessageJoiner);
+}
+
+function _toApproximateOpenAIDocPart({ data, ref }: AixParts_DocPart): OpenAIWire_ContentParts.TextContentPart {
+
+  // Corner case, low probability: if the content is already enclosed in triple-backticks, return it as-is
+  if (data.text.startsWith('```'))
+    return OpenAIWire_ContentParts.TextContentPart(data.text);
+
+  // TODO: consider a better representation here - we use the 'legacy' markdown encoding
+  //       but we may as well support different ones in the future, such as:
+  //       - '<doc id='ref' title='title' version='version'>\n...\n</doc>'
+  //       - ```doc id='ref' title='title' version='version'\n...\n```
+  //       - etc.
+
+  return OpenAIWire_ContentParts.TextContentPart(`\`\`\`${ref || ''}\n${data.text}\n\`\`\`\n`);
+}

src/modules/aix/server/dispatch/wiretypes/openai.wiretypes.ts

@@ -19,6 +19,7 @@ export namespace OpenAIWire_ContentParts {
 
   /// Content parts - Input
 
+  export type TextContentPart = z.infer<typeof TextContentPart_schema>;
   const TextContentPart_schema = z.object({
     type: z.literal('text'),
     text: z.string(),