diff --git a/src/modules/aix/client/aix.client.chatGenerateRequest.ts b/src/modules/aix/client/aix.client.chatGenerateRequest.ts
index 76aa8611f..37af02b7a 100644
--- a/src/modules/aix/client/aix.client.chatGenerateRequest.ts
+++ b/src/modules/aix/client/aix.client.chatGenerateRequest.ts
@@ -2,7 +2,7 @@ import { getImageAsset } from '~/modules/dblobs/dblobs.images';
import { DLLM, LLM_IF_HOTFIX_NoStream, LLM_IF_HOTFIX_StripImages, LLM_IF_HOTFIX_Sys0ToUsr0 } from '~/common/stores/llms/llms.types';
import { DMessage, DMessageRole, DMetaReferenceItem, MESSAGE_FLAG_AIX_SKIP, MESSAGE_FLAG_VND_ANT_CACHE_AUTO, MESSAGE_FLAG_VND_ANT_CACHE_USER, messageHasUserFlag } from '~/common/stores/chat/chat.message';
-import { DMessageFragment, DMessageImageRefPart, isContentOrAttachmentFragment, isTextContentFragment, isToolResponseFunctionCallPart } from '~/common/stores/chat/chat.fragments';
+import { DMessageFragment, DMessageImageRefPart, isAttachmentFragment, isContentOrAttachmentFragment, isDocPart, isTextContentFragment, isToolResponseFunctionCallPart } from '~/common/stores/chat/chat.fragments';
import { Is } from '~/common/util/pwaUtils';
import { LLMImageResizeMode, resizeBase64ImageIfNeeded } from '~/common/util/imageUtils';
@@ -80,9 +80,9 @@ export async function aixCGR_SystemMessage_FromDMessageOrThrow(
for (const fragment of systemInstruction.fragments) {
if (isTextContentFragment(fragment)) {
sm.parts.push(fragment.part);
- }
- // TODO: handle other types of fragments if needed, such as the 'doc' type
- else {
+ } else if (isAttachmentFragment(fragment) && isDocPart(fragment.part)) {
+ sm.parts.push(fragment.part);
+ } else {
if (process.env.NODE_ENV === 'development')
throw new Error('[DEV] aixCGR_systemMessageFromInstruction: unexpected system fragment');
console.warn('[DEV] aixCGR_systemMessageFromInstruction: unexpected system fragment:', fragment);
diff --git a/src/modules/aix/server/api/aix.wiretypes.ts b/src/modules/aix/server/api/aix.wiretypes.ts
index 3717b6440..d1358c249 100644
--- a/src/modules/aix/server/api/aix.wiretypes.ts
+++ b/src/modules/aix/server/api/aix.wiretypes.ts
@@ -224,6 +224,7 @@ export namespace AixWire_Content {
export const SystemInstruction_schema = z.object({
parts: z.array(z.discriminatedUnion('pt', [
AixWire_Parts.TextPart_schema,
+ AixWire_Parts.DocPart_schema, // Jan 10, 2025: added support for Docs in AIX system
AixWire_Parts.MetaCacheControl_schema,
])),
});
diff --git a/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts b/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts
index 70e93d7a4..0eba2eec1 100644
--- a/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts
+++ b/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts
@@ -1,6 +1,6 @@
import type { OpenAIDialects } from '~/modules/llms/server/openai/openai.router';
-import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
+import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixParts_DocPart, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
import { OpenAIWire_API_Chat_Completions, OpenAIWire_ContentParts, OpenAIWire_Messages } from '../../wiretypes/openai.wiretypes';
@@ -21,6 +21,7 @@ const hotFixOnlySupportN1 = true;
const hotFixPreferArrayUserContent = true;
const hotFixForceImageContentPartOpenAIDetail: 'auto' | 'low' | 'high' = 'high';
const hotFixSquashTextSeparator = '\n\n\n---\n\n\n';
+const approxSystemMessageJoiner = '\n\n---\n\n';
type TRequest = OpenAIWire_API_Chat_Completions.Request;
@@ -198,17 +199,35 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | null, chat
// Transform the chat messages into OpenAI's format (an array of 'system', 'user', 'assistant', and 'tool' messages)
const chatMessages: TRequestMessages = [];
- // Convert the system message
+ // Convert the system message - single-part stay as-is and multi-part (text or doc) are flattened to a string
+ const msg0TextParts: OpenAIWire_ContentParts.TextContentPart[] = [];
systemMessage?.parts.forEach((part) => {
- if (part.pt === 'meta_cache_control') {
- // ignore this hint - openai doesn't support this yet
- } else
- chatMessages.push({
- role: !hotFixOpenAIo1Family ? 'system' : 'developer', // NOTE: o1Family in this case is not o1-preview as it's sporting the Sys0ToUsr0 hotfix
- content: part.text, /*, name: _optionalParticipantName */
- });
+ switch (part.pt) {
+ case 'text':
+ msg0TextParts.push(OpenAIWire_ContentParts.TextContentPart(part.text));
+ break;
+
+ case 'doc':
+ msg0TextParts.push(_toApproximateOpenAIDocPart(part));
+ break;
+
+ case 'meta_cache_control':
+ // ignore this hint - openai doesn't support this yet
+ break;
+
+ default:
+ throw new Error(`Unsupported part type in System message: ${(part as any).pt}`);
+ }
});
+ // Add the system message
+ if (msg0TextParts.length)
+ chatMessages.push({
+ role: !hotFixOpenAIo1Family ? 'system' : 'developer', // NOTE: o1Family in this case is not o1-preview as it's sporting the Sys0ToUsr0 hotfix
+ content: _toApproximateOpanAIFlattenSystemMessage(msg0TextParts),
+ });
+
+
// Convert the messages
for (const { parts, role } of chatSequence) {
switch (role) {
@@ -218,18 +237,8 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | null, chat
const currentMessage = chatMessages[chatMessages.length - 1];
switch (part.pt) {
- case 'doc':
case 'text':
- // Implementation notes:
- // - doc is rendered as a simple text part, but enclosed in a markdow block
- // - TODO: consider better representation - we use the 'legacy' markdown encoding here,
- // but we may as well support different ones (e.g. XML) in the future
- const textContentString =
- part.pt === 'text' ? part.text
- : /* doc */ part.data.text.startsWith('```') ? part.data.text
- : `\`\`\`${part.ref || ''}\n${part.data.text}\n\`\`\`\n`;
-
- const textContentPart = OpenAIWire_ContentParts.TextContentPart(textContentString);
+ const textContentPart = OpenAIWire_ContentParts.TextContentPart(part.text);
// Append to existing content[], or new message
if (currentMessage?.role === 'user' && Array.isArray(currentMessage.content))
@@ -238,6 +247,16 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | null, chat
chatMessages.push({ role: 'user', content: hotFixPreferArrayUserContent ? [textContentPart] : textContentPart.text });
break;
+ case 'doc':
+ const docContentPart = _toApproximateOpenAIDocPart(part);
+
+ // Append to existing content[], or new message
+ if (currentMessage?.role === 'user' && Array.isArray(currentMessage.content))
+ currentMessage.content.push(docContentPart);
+ else
+ chatMessages.push({ role: 'user', content: hotFixPreferArrayUserContent ? [docContentPart] : docContentPart.text });
+ break;
+
case 'inline_image':
// create a new OpenAIWire_ImageContentPart
const { mimeType, base64 } = part;
@@ -431,3 +450,25 @@ function _toOpenAIInReferenceToText(irt: AixParts_MetaInReferenceToPart): string
return `CONTEXT: The user is referring to these ${items.length} in particular:\n\n${
items.map((text, index) => formatItem(text, index)).join(allShort ? '\n' : '\n\n')}`;
}
+
+
+// Approximate conversions
+
+function _toApproximateOpanAIFlattenSystemMessage(texts: OpenAIWire_ContentParts.TextContentPart[]): string {
+ return texts.map(text => text.text).join(approxSystemMessageJoiner);
+}
+
+function _toApproximateOpenAIDocPart({ data, ref }: AixParts_DocPart): OpenAIWire_ContentParts.TextContentPart {
+
+ // Corner case, low probability: if the content is already enclosed in triple-backticks, return it as-is
+ if (data.text.startsWith('```'))
+ return OpenAIWire_ContentParts.TextContentPart(data.text);
+
+ // TODO: consider a better representation here - we use the 'legacy' markdown encoding
+ // but we may as well support different ones in the future, such as:
+ // - '\n...\n'
+ // - ```doc id='ref' title='title' version='version'\n...\n```
+ // - etc.
+
+ return OpenAIWire_ContentParts.TextContentPart(`\`\`\`${ref || ''}\n${data.text}\n\`\`\`\n`);
+}
diff --git a/src/modules/aix/server/dispatch/wiretypes/openai.wiretypes.ts b/src/modules/aix/server/dispatch/wiretypes/openai.wiretypes.ts
index 81dadbf06..635ced6d8 100644
--- a/src/modules/aix/server/dispatch/wiretypes/openai.wiretypes.ts
+++ b/src/modules/aix/server/dispatch/wiretypes/openai.wiretypes.ts
@@ -19,6 +19,7 @@ export namespace OpenAIWire_ContentParts {
/// Content parts - Input
+ export type TextContentPart = z.infer;
const TextContentPart_schema = z.object({
type: z.literal('text'),
text: z.string(),