MP: adapt ego attachment, messageSingleTextOrThrow--

src/apps/chat/components/composer/Composer.tsx

@@ -29,7 +29,7 @@ import { SpeechResult, useSpeechRecognition } from '~/common/components/useSpeec
 import { animationEnterBelow } from '~/common/util/animUtils';
 import { conversationTitle, DConversationId } from '~/common/stores/chat/chat.conversation';
 import { copyToClipboard, supportsClipboardRead } from '~/common/util/clipboardUtils';
-import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, DMessageMetadata, messageSingleTextOrThrow } from '~/common/stores/chat/chat.message';
+import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, DMessageMetadata, duplicateDMessageFragments, isContentFragment, messageFragmentsReduceText } from '~/common/stores/chat/chat.message';
 import { estimateTextTokens, glueForMessageTokens } from '~/common/stores/chat/chat.tokens';
 import { getConversation, useChatStore } from '~/common/stores/chat/store-chats';
 import { isMacUser } from '~/common/util/pwaUtils';
@@ -155,7 +155,7 @@ export function Composer(props: {
   // attachments-overlay: comes from the attachments slice of the conversation overlay
   const {
     attachmentDrafts,
-    attachAppendClipboardItems, attachAppendDataTransfer, attachAppendEgoMessage, attachAppendFile,
+    attachAppendClipboardItems, attachAppendDataTransfer, attachAppendEgoContent, attachAppendFile,
     attachmentsClear, attachmentsTakeAllFragments, attachmentsTakeTextFragments,
   } = useAttachmentDrafts(conversationOverlayStore, enableLoadURLsInComposer);
 
@@ -321,19 +321,22 @@ export function Composer(props: {
     }
   }, [props.composerTextAreaRef, setComposeText]);
 
-  const onActileMessageAttach = React.useCallback((item: StarredMessageItem) => {
+  const onActileMessageAttach = React.useCallback(async (item: StarredMessageItem) => {
     // get the message
     const conversation = getConversation(item.conversationId);
     const messageToAttach = conversation?.messages.find(m => m.id === item.messageId);
-    const messageText = messageToAttach ? messageSingleTextOrThrow(messageToAttach) : null;
-    if (conversation && messageToAttach && messageText) {
-      // Testing with this serialization for LLM. Note it will still be within a multi-part message,
-      // this could be in a titled markdown block. Don't know yet how this fares with different LLMs.
-      const chatTitle = conversationTitle(conversation);
-      const textPlain = `---\nitem id: ${messageToAttach.id}\ncontext title: ${chatTitle}\n---\n${messageText.trim()}\n`;
-      void attachAppendEgoMessage('context-item', textPlain, `${chatTitle} > ${messageText.slice(0, 10)}...`);
+    if (conversation && messageToAttach) {
+      const contentToAttach = duplicateDMessageFragments(messageToAttach.fragments)
+        .filter(isContentFragment);
+      if (contentToAttach.length) {
+        const chatTitle = conversationTitle(conversation);
+        const messageText = messageFragmentsReduceText(contentToAttach);
+        const refLabel = `${chatTitle} > ${messageText.slice(0, 10)}...`;
+        const refId = `${item.messageId} - ${chatTitle}`;
+        await attachAppendEgoContent(refLabel, refId, contentToAttach);
+      }
     }
-  }, [attachAppendEgoMessage]);
+  }, [attachAppendEgoContent]);
 
   const actileProviders = React.useMemo(() => {
     return [providerCommands(onActileCommandPaste), providerStarredMessage(onActileMessageAttach)];

src/apps/chat/components/composer/llmattachments/LLMAttachmentItem.tsx

@@ -80,7 +80,7 @@ const converterTypeToIconMap: { [key in AttachmentDraftConverterType]: React.Com
   'image-resized-low': PhotoSizeSelectSmallOutlinedIcon,
   'image-to-default': ImageOutlinedIcon,
   'image-ocr': AbcIcon,
-  'ego-message-md': TelegramIcon,
+  'ego-contents-inlined': TelegramIcon,
   'unhandled': TextureIcon,
 };
 

src/common/attachment-drafts/attachment.pipeline.ts

@@ -4,7 +4,7 @@ import { createBase64UuidV4 } from '~/common/util/textUtils';
 import { htmlTableToMarkdown } from '~/common/util/htmlTableToMarkdown';
 import { pdfToImageDataURLs, pdfToText } from '~/common/util/pdfUtils';
 
-import { createTextAttachmentFragment, DMessageAttachmentFragment } from '~/common/stores/chat/chat.message';
+import { createAttachmentFragment, createTextAttachmentFragment, DMessageAttachmentFragment, DMessageContentFragment } from '~/common/stores/chat/chat.message';
 
 import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftInput, AttachmentDraftSource } from './attachment.types';
 import type { AttachmentsDraftsStore } from './store-attachment-drafts-slice';
@@ -198,11 +198,11 @@ export async function attachmentLoadInputAsync(source: Readonly<AttachmentDraftS
     case 'ego':
       edit({
         label: source.label,
-        ref: source.blockTitle,
+        ref: source.refId,
         input: {
-          mimeType: 'ego/message',
-          data: source.textPlain,
-          dataSize: source.textPlain.length,
+          mimeType: 'ego/contents',
+          data: source.contents,
+          dataSize: source.contents.length,
         },
       });
       break;
@@ -262,8 +262,8 @@ export function attachmentDefineConverters(sourceType: AttachmentDraftSource['me
       break;
 
     // EGO
-    case input.mimeType === 'ego/message':
-      converters.push({ id: 'ego-message-md', name: 'Message' });
+    case input.mimeType === 'ego/contents':
+      converters.push({ id: 'ego-contents-inlined', name: 'Message' });
       break;
 
     // catch-all
@@ -438,8 +438,17 @@ export async function attachmentPerformConversion(
 
 
     // self: message
-    case 'ego-message-md':
-      newFragments.push(createTextAttachmentFragment(inputDataToString(input.data), ref));
+    case 'ego-contents-inlined':
+      if (!Array.isArray(input.data)) {
+        console.log('Expected DMessageContentFragment[] for ego-contents-inlined, got:', typeof input.data);
+        break;
+      }
+      for (const contentFragment of input.data) {
+        if (contentFragment.part.pt === 'text' || contentFragment.part.pt === 'image_ref')
+          newFragments.push(createAttachmentFragment(source.media === 'ego' ? source.refId : 'Message', contentFragment.part));
+        else
+          console.log('Unhandled ego-contents-inlined part:', contentFragment.part.pt);
+      }
       break;
 
     case 'unhandled':
@@ -455,10 +464,11 @@ export async function attachmentPerformConversion(
 }
 
 
-function inputDataToString(data: string | ArrayBuffer | null | undefined): string {
+function inputDataToString(data: string | ArrayBuffer | DMessageContentFragment[] | null | undefined): string {
   if (typeof data === 'string')
     return data;
   if (data instanceof ArrayBuffer)
     return new TextDecoder().decode(data);
+  console.log('attachment.inputDataToString: expected string or ArrayBuffer, got:', typeof data);
   return '';
 }

src/common/attachment-drafts/attachment.types.ts

@@ -1,6 +1,6 @@
 import type { FileWithHandle } from 'browser-fs-access';
 
-import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.message';
+import type { DMessageAttachmentFragment, DMessageContentFragment } from '~/common/stores/chat/chat.message';
 
 
 // Attachment Draft
@@ -51,10 +51,10 @@ export type AttachmentDraftSource = {
 } | {
   // special type for attachments thar are references to self (ego, application) objects
   media: 'ego';
-  method: 'ego-message';
+  method: 'ego-contents';
+  contents: DMessageContentFragment[];
   label: string;
-  blockTitle: string;
-  textPlain: string;
+  refId: string; // message ID where the context came from (unused..)
 };
 
 export type AttachmentDraftSourceOriginFile = 'camera' | 'screencapture' | 'file-open' | 'clipboard-read' | AttachmentDraftSourceOriginDTO;
@@ -66,7 +66,7 @@ export type AttachmentDraftSourceOriginDTO = 'drop' | 'paste';
 
 export type AttachmentDraftInput = {
   mimeType: string; // Original MIME type of the file
-  data: string | ArrayBuffer; // The original data of the attachment
+  data: string | ArrayBuffer | DMessageContentFragment[]; // The original data of the attachment
   dataSize: number; // Size of the original data in bytes
   altMimeType?: string; // Alternative MIME type for the input
   altData?: string; // Alternative data for the input
@@ -92,7 +92,7 @@ export type AttachmentDraftConverterType =
   | 'text' | 'rich-text' | 'rich-text-table'
   | 'pdf-text' | 'pdf-images'
   | 'image-original' | 'image-resized-high' | 'image-resized-low' | 'image-ocr' | 'image-to-default'
-  | 'ego-message-md'
+  | 'ego-contents-inlined'
   | 'unhandled';
 
 

src/common/attachment-drafts/useAttachmentDrafts.tsx

@@ -7,7 +7,7 @@ import { asValidURL } from '~/common/util/urlUtils';
 import { extractFilePathsWithCommonRadix } from '~/common/util/dropTextUtils';
 import { getClipboardItems } from '~/common/util/clipboardUtils';
 
-import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.message';
+import type { DMessageAttachmentFragment, DMessageContentFragment } from '~/common/stores/chat/chat.message';
 import { attachmentWrapText, TextAttachmentWrapFormat } from '~/common/stores/chat/chat.tokens';
 import { useChatAttachmentsStore } from '~/common/chats/store-chat-overlay';
 
@@ -117,12 +117,12 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
   }, [attachAppendFile, _createAttachmentDraft, enableLoadURLs]);
 
 
-  const attachAppendEgoMessage = React.useCallback((blockTitle: string, textPlain: string, attachmentLabel: string) => {
+  const attachAppendEgoContent = React.useCallback((label: string, refId: string, contents: DMessageContentFragment[]) => {
     if (ATTACHMENTS_DEBUG_INTAKE)
-      console.log('attachAppendEgo', { blockTitle, textPlain, attachmentLabel });
+      console.log('attachAppendEgoContent', label, refId, contents);
 
     return _createAttachmentDraft({
-      media: 'ego', method: 'ego-message', label: attachmentLabel, blockTitle: blockTitle, textPlain: textPlain,
+      media: 'ego', method: 'ego-contents', label, refId, contents,
     });
   }, [_createAttachmentDraft]);
 
@@ -205,7 +205,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
     // create drafts
     attachAppendClipboardItems,
     attachAppendDataTransfer,
-    attachAppendEgoMessage,
+    attachAppendEgoContent,
     attachAppendFile,
 
     // manage attachments

src/common/stores/chat/chat.message.ts

@@ -216,7 +216,7 @@ export function duplicateDMessage(message: Readonly<DMessage>): DMessage {
     id: createBase64UuidV4(),
 
     role: message.role,
-    fragments: message.fragments.map(_duplicateFragment),
+    fragments: duplicateDMessageFragments(message.fragments),
 
     ...(message.pendingIncomplete ? { pendingIncomplete: true } : {}),
     ...(message.pendingPlaceholderText ? { pendingPlaceholderText: message.pendingPlaceholderText } : {}),
@@ -236,6 +236,10 @@ export function duplicateDMessage(message: Readonly<DMessage>): DMessage {
   };
 }
 
+export function duplicateDMessageFragments(fragments: Readonly<DMessageFragment[]>): DMessageFragment[] {
+  return fragments.map(_duplicateFragment);
+}
+
 function _duplicateFragment(fragment: DMessageFragment): DMessageFragment {
   switch (fragment.ft) {
     case 'content':
@@ -284,7 +288,10 @@ function _duplicateReference(ref: DMessageDataRef): DMessageDataRef {
 
 // helpers during the transition from V3
 
-// specialize output type with 'is'
+export function isContentFragment(fragment: DMessageFragment): fragment is DMessageContentFragment {
+  return fragment.ft === 'content';
+}
+
 export function isContentOrAttachmentFragment(fragment: DMessageFragment): fragment is DMessageContentFragment | DMessageAttachmentFragment {
   return fragment.ft === 'content' || fragment.ft === 'attachment';
 }