Attachments: MultiPart-ready. Closes #251 for this stage.

src/apps/chat/AppChat.tsx

@@ -21,6 +21,7 @@ import { createDMessage, DConversationId, DMessage, getConversation, useConversa
 import { openLayoutLLMOptions, useLayoutPluggable } from '~/common/layout/store-applayout';
 import { useUXLabsStore } from '~/common/state/store-ux-labs';
 
+import type { ComposerOutputMultiPart } from './components/composer/composer.types';
 import { ChatDrawerItemsMemo } from './components/applayout/ChatDrawerItems';
 import { ChatDropdowns } from './components/applayout/ChatDropdowns';
 import { ChatMenuItems } from './components/applayout/ChatMenuItems';
@@ -182,13 +183,33 @@ export function AppChat() {
     setMessages(conversationId, history);
   }, [focusedSystemPurposeId, setMessages]);
 
-  const handleComposerNewMessage = async (chatModeId: ChatModeId, conversationId: DConversationId, userText: string) => {
+  const handleComposerAction = (chatModeId: ChatModeId, conversationId: DConversationId, multiPartMessage: ComposerOutputMultiPart): boolean => {
+
+    // validate inputs
+    if (multiPartMessage.length !== 1 || multiPartMessage[0].type !== 'text-block') {
+      addSnackbar({
+        key: 'chat-composer-action-invalid',
+        message: 'Only a single text part is supported for now.',
+        type: 'issue',
+        overrides: {
+          autoHideDuration: 2000,
+        },
+      });
+      return false;
+    }
+    const userText = multiPartMessage[0].text;
+
+    // find conversation
     const conversation = getConversation(conversationId);
-    if (conversation)
-      return await _handleExecute(chatModeId, conversationId, [
-        ...conversation.messages,
-        createDMessage('user', userText),
-      ]);
+    if (!conversation)
+      return false;
+
+    // start execution (async)
+    void _handleExecute(chatModeId, conversationId, [
+      ...conversation.messages,
+      createDMessage('user', userText),
+    ]);
+    return true;
   };
 
   const handleConversationExecuteHistory = async (conversationId: DConversationId, history: DMessage[]) =>
@@ -409,7 +430,7 @@ export function AppChat() {
       composerTextAreaRef={composerTextAreaRef}
       conversationId={focusedConversationId}
       isDeveloperMode={focusedSystemPurposeId === 'Developer'}
-      onNewMessage={handleComposerNewMessage}
+      onAction={handleComposerAction}
       sx={{
         zIndex: 21, // position: 'sticky', bottom: 0,
         backgroundColor: 'background.surface',

src/apps/chat/components/composer/Composer.tsx

@@ -35,9 +35,10 @@ import { useUXLabsStore } from '~/common/state/store-ux-labs';
 
 import type { AttachmentId } from './attachments/store-attachments';
 import { Attachments } from './attachments/Attachments';
+import { getTextBlockText, useLLMAttachments } from './attachments/useLLMAttachments';
 import { useAttachments } from './attachments/useAttachments';
-import { useLLMAttachments } from './attachments/useLLMAttachments';
 
+import type { ComposerOutputMultiPart } from './composer.types';
 import { ButtonAttachCamera } from './ButtonAttachCamera';
 import { ButtonAttachClipboard } from './ButtonAttachClipboard';
 import { ButtonAttachFile } from './ButtonAttachFile';
@@ -64,21 +65,14 @@ const animationStopEnter = keyframes`
 
 
 /**
- * A React component for composing and sending messages in a chat-like interface.
- *
- * Note: Useful bash trick to generate code from a list of files:
- *       $ for F in *.ts; do echo; echo "\`\`\`$F"; cat $F; echo; echo "\`\`\`"; done | clip
- *
- * @param {boolean} props.disableSend - Flag to disable the send button.
- * @param {(text: string, conversationId: string | null) => void} props.sendMessage - Function to send the message. conversationId is null for the Active conversation
- * @param {() => void} props.stopGeneration - Function to stop response generation
+ * A React component for composing messages, with attachments and different modes.
  */
 export function Composer(props: {
   chatLLM: DLLM | null;
   composerTextAreaRef: React.RefObject<HTMLTextAreaElement>;
   conversationId: DConversationId | null;
   isDeveloperMode: boolean;
-  onNewMessage: (chatModeId: ChatModeId, conversationId: DConversationId, text: string) => void;
+  onAction: (chatModeId: ChatModeId, conversationId: DConversationId, multiPartMessage: ComposerOutputMultiPart) => boolean;
   sx?: SxProps;
 }) {
 
@@ -147,42 +141,48 @@ export function Composer(props: {
 
   // Primary button
 
-  const handleSendClicked = (_chatModeId: ChatModeId) => {
-    let text = (composeText || '').trim();
-    // inline the text attachments and clear if any string
-    if (!_chatModeId.startsWith('draw-')) {
-      const inlineTextAttachments = llmAttachments.inlineTextAttachments();
-      if (inlineTextAttachments !== null) {
-        if (text.length)
-          text += inlineTextAttachments;
-        else
-          text = inlineTextAttachments.trim();
-        clearAttachments();
+  const { conversationId, onAction } = props;
+
+  const handleSendAction = React.useCallback((_chatModeId: ChatModeId, composerText: string): boolean => {
+    if (!conversationId)
+      return false;
+
+    // get attachments
+    const multiPartMessage = llmAttachments.getAttachmentsOutputs(composerText || null);
+    if (!multiPartMessage.length)
+      return false;
+
+    // send the message
+    const enqueued = onAction(_chatModeId, conversationId, multiPartMessage);
+    if (enqueued) {
+      clearAttachments();
+      setComposeText('');
+    }
+
+    return enqueued;
+  }, [clearAttachments, conversationId, llmAttachments, onAction, setComposeText]);
+
+  const handleTextareaKeyDown = React.useCallback((e: React.KeyboardEvent) => {
+    if (e.key === 'Enter') {
+
+      // Alt: append the message instead
+      if (e.altKey) {
+        handleSendAction('write-user', composeText);
+        return e.preventDefault();
+      }
+
+      // Shift: toggles the 'enter is newline'
+      if (enterIsNewline ? e.shiftKey : !e.shiftKey) {
+        if (!assistantTyping)
+          handleSendAction(chatModeId, composeText);
+        return e.preventDefault();
       }
     }
-    if (text.length && props.conversationId && chatLLMId) {
-      setComposeText('');
-      props.onNewMessage(_chatModeId, props.conversationId, text);
-    }
-  };
+  }, [assistantTyping, chatModeId, composeText, enterIsNewline, handleSendAction]);
 
-  const handleTextareaKeyDown = (e: React.KeyboardEvent) => {
-    if (e.key !== 'Enter')
-      return;
+  const handleSendClicked = () => handleSendAction(chatModeId, composeText);
 
-    // Alt: append the message
-    if (e.altKey) {
-      handleSendClicked('write-user');
-      return e.preventDefault();
-    }
-
-    // Shift: toggles the 'enter is newline'
-    if (enterIsNewline ? e.shiftKey : !e.shiftKey) {
-      if (!assistantTyping)
-        handleSendClicked(chatModeId);
-      return e.preventDefault();
-    }
-  };
+  const handleStopClicked = () => props.conversationId && stopTyping(props.conversationId);
 
 
   // Secondary buttons
@@ -204,36 +204,38 @@ export function Composer(props: {
     setChatModeId(_chatModeId);
   };
 
-  const handleStopClicked = () => props.conversationId && stopTyping(props.conversationId);
-
 
   // Mic typing & continuation mode
 
   const onSpeechResultCallback = React.useCallback((result: SpeechResult) => {
-    setSpeechInterimResult(result.done ? null : { ...result });
-    if (result.done) {
-      // append the transcript
-      const transcript = result.transcript.trim();
-      let newText = (composeText || '').trim();
-      newText = newText ? newText + ' ' + transcript : transcript;
-
-      // auto-send if requested
-      const autoSend = micContinuation && newText.length >= 1 && !!props.conversationId; //&& assistantTyping;
-      if (autoSend) {
-        props.onNewMessage(chatModeId, props.conversationId!, newText);
-        if (result.doneReason !== 'manual')
-          playSoundUrl('/sounds/mic-off-mid.mp3');
-      } else {
-        if (newText)
-          props.composerTextAreaRef.current?.focus();
-        if (!micContinuation && result.doneReason !== 'manual')
-          playSoundUrl('/sounds/mic-off-mid.mp3');
-      }
-
-      // set the text (or clear if auto-sent)
-      setComposeText(autoSend ? '' : newText);
+    // not done: show interim
+    if (!result.done) {
+      setSpeechInterimResult({ ...result });
+      return;
     }
-  }, [chatModeId, composeText, micContinuation, props, setComposeText]);
+
+    // done
+    setSpeechInterimResult(null);
+    const transcript = result.transcript.trim();
+    let nextText = (composeText || '').trim();
+    nextText = nextText ? nextText + ' ' + transcript : transcript;
+
+    // auto-send (mic continuation mode) if requested
+    const autoSend = micContinuation && nextText.length >= 1 && !!props.conversationId; //&& assistantTyping;
+    const notUserStop = result.doneReason !== 'manual';
+    if (autoSend) {
+      if (notUserStop)
+        playSoundUrl('/sounds/mic-off-mid.mp3');
+      handleSendAction(chatModeId, nextText);
+    } else {
+      if (!micContinuation && notUserStop)
+        playSoundUrl('/sounds/mic-off-mid.mp3');
+      if (nextText) {
+        props.composerTextAreaRef.current?.focus();
+        setComposeText(nextText);
+      }
+    }
+  }, [chatModeId, composeText, handleSendAction, micContinuation, props.composerTextAreaRef, props.conversationId, setComposeText]);
 
   const { isSpeechEnabled, isSpeechError, isRecordingAudio, isRecordingSpeech, toggleRecording } =
     useSpeechRecognition(onSpeechResultCallback, chatMicTimeoutMs || 2000, 'm');
@@ -283,20 +285,20 @@ export function Composer(props: {
   useGlobalShortcut(supportsClipboardRead ? 'v' : false, true, true, false, attachAppendClipboardItems);
 
   const handleAttachmentInlineText = React.useCallback((attachmentId: AttachmentId) => {
-    setComposeText(text => {
-      const inlinedText = llmAttachments.inlineTextAttachment(attachmentId);
-      if (inlinedText !== null)
-        removeAttachment(attachmentId);
-      return inlinedText ? text + inlinedText : text;
+    setComposeText(currentText => {
+      const attachmentOutputs = llmAttachments.getAttachmentOutputs(currentText, attachmentId);
+      const inlinedText = getTextBlockText(attachmentOutputs) || '';
+      removeAttachment(attachmentId);
+      return inlinedText;
     });
   }, [llmAttachments, removeAttachment, setComposeText]);
 
-  const handleAttachmentsInline = React.useCallback(() => {
-    setComposeText(text => {
-      const inlinedText = llmAttachments.inlineTextAttachments();
-      if (inlinedText !== null)
-        clearAttachments();
-      return inlinedText ? text + inlinedText : text;
+  const handleAttachmentsInlineText = React.useCallback(() => {
+    setComposeText(currentText => {
+      const attachmentsOutputs = llmAttachments.getAttachmentsOutputs(currentText);
+      const inlinedText = getTextBlockText(attachmentsOutputs) || '';
+      clearAttachments();
+      return inlinedText;
     });
   }, [clearAttachments, llmAttachments, setComposeText]);
 
@@ -529,7 +531,7 @@ export function Composer(props: {
               llmAttachments={llmAttachments}
               onAttachmentInlineText={handleAttachmentInlineText}
               onAttachmentsClear={clearAttachments}
-              onAttachmentsInlineText={handleAttachmentsInline}
+              onAttachmentsInlineText={handleAttachmentsInlineText}
             />
 
           </Box>
@@ -564,7 +566,7 @@ export function Composer(props: {
                   <Button
                     key='composer-act'
                     fullWidth disabled={!props.conversationId || !chatLLMId || !llmAttachments.isOutputAttacheable}
-                    onClick={() => handleSendClicked(chatModeId)}
+                    onClick={handleSendClicked}
                     endDecorator={micContinuation ? <AutoModeIcon /> : isWriteUser ? <SendIcon sx={{ fontSize: 18 }} /> : isReAct ? <PsychologyIcon /> : <TelegramIcon />}
                   >
                     {micContinuation && 'Voice '}

src/apps/chat/components/composer/attachments/AttachmentMenu.tsx

@@ -10,7 +10,7 @@ import VerticalAlignBottomIcon from '@mui/icons-material/VerticalAlignBottom';
 import { CloseableMenu } from '~/common/components/CloseableMenu';
 import { copyToClipboard } from '~/common/util/clipboardUtils';
 
-import { attachmentCollapseOutputs, LLMAttachment } from './useLLMAttachments';
+import type { LLMAttachment } from './useLLMAttachments';
 import { useAttachmentsStore } from './store-attachments';
 
 
@@ -33,6 +33,7 @@ export function AttachmentMenu(props: {
 
   const {
     attachment,
+    attachmentOutputs,
     isUnconvertible,
     isOutputMissing,
     isOutputTextInlineable,
@@ -79,9 +80,8 @@ export function AttachmentMenu(props: {
   // }, [aId, onAttachmentSummarizeText]);
 
   const handleCopyOutputToClipboard = React.useCallback(() => {
-    const outputs = attachmentCollapseOutputs(aOutputs);
-    if (outputs.length >= 1) {
-      const concat = outputs.map(output => {
+    if (attachmentOutputs.length >= 1) {
+      const concat = attachmentOutputs.map(output => {
         if (output.type === 'text-block')
           return output.text;
         else if (output.type === 'image-part')
@@ -91,7 +91,7 @@ export function AttachmentMenu(props: {
       }).join('\n\n---\n\n');
       copyToClipboard(concat.trim(), 'Converted attachment');
     }
-  }, [aOutputs]);
+  }, [attachmentOutputs]);
 
 
   return (

src/apps/chat/components/composer/attachments/useLLMAttachments.ts

@@ -10,8 +10,8 @@ import type { ComposerOutputMultiPart, ComposerOutputPartType } from '../compose
 
 export interface LLMAttachments {
   attachments: LLMAttachment[];
-  inlineTextAttachment: (attachmentId: AttachmentId) => string | null;
-  inlineTextAttachments: () => string | null;
+  getAttachmentOutputs: (initialTextBlockText: string | null, attachmentId: AttachmentId) => ComposerOutputMultiPart;
+  getAttachmentsOutputs: (initialTextBlockText: string | null) => ComposerOutputMultiPart;
   isOutputAttacheable: boolean;
   isOutputTextInlineable: boolean;
   tokenCountApprox: number;
@@ -19,6 +19,7 @@ export interface LLMAttachments {
 
 export interface LLMAttachment {
   attachment: Attachment;
+  attachmentOutputs: ComposerOutputMultiPart;
   isUnconvertible: boolean;
   isOutputMissing: boolean;
   isOutputAttachable: boolean;
@@ -36,23 +37,22 @@ export function useLLMAttachments(attachments: Attachment[], chatLLMId: DLLMId |
 
     const llmAttachments = attachments.map(attachment => toLLMAttachment(attachment, supportedOutputPartTypes, chatLLMId));
 
-    const inlineTextAttachment = (attachmentId: AttachmentId): string | null => {
+    const getAttachmentOutputs = (initialTextBlockText: string | null, attachmentId: AttachmentId): ComposerOutputMultiPart => {
+      // get outputs of a specific attachment
       const outputs = attachments.find(a => a.id === attachmentId)?.outputs || [];
-      const collapsedTextOutputs = attachmentCollapseOutputs(outputs.filter(part => part.type === 'text-block'));
-      return (collapsedTextOutputs.length === 1 && collapsedTextOutputs[0].type === 'text-block') ? collapsedTextOutputs[0].text : null;
+      return attachmentCollapseOutputs(initialTextBlockText, outputs);
     };
 
-    const inlineTextAttachments = (): string | null => {
+    const getAttachmentsOutputs = (initialTextBlockText: string | null): ComposerOutputMultiPart => {
       // accumulate all outputs of all attachments
-      const outputs = llmAttachments.reduce((acc, a) => acc.concat(a.attachment.outputs), [] as ComposerOutputMultiPart);
-      const collapsedTextOutputs = attachmentCollapseOutputs(outputs.filter(part => part.type === 'text-block'));
-      return (collapsedTextOutputs.length === 1 && collapsedTextOutputs[0].type === 'text-block') ? collapsedTextOutputs[0].text : null;
+      const allOutputs = llmAttachments.reduce((acc, a) => acc.concat(a.attachment.outputs), [] as ComposerOutputMultiPart);
+      return attachmentCollapseOutputs(initialTextBlockText, allOutputs);
     };
 
     return {
       attachments: llmAttachments,
-      inlineTextAttachment,
-      inlineTextAttachments,
+      getAttachmentOutputs,
+      getAttachmentsOutputs,
       isOutputAttacheable: llmAttachments.every(a => a.isOutputAttachable),
       isOutputTextInlineable: llmAttachments.every(a => a.isOutputTextInlineable),
       tokenCountApprox: llmAttachments.reduce((acc, a) => acc + (a.tokenCountApprox || 0), 0),
@@ -60,6 +60,11 @@ export function useLLMAttachments(attachments: Attachment[], chatLLMId: DLLMId |
   }, [attachments, chatLLMId]);
 }
 
+export function getTextBlockText(outputs: ComposerOutputMultiPart): string | null {
+  const textOutputs = outputs.filter(part => part.type === 'text-block');
+  return (textOutputs.length === 1 && textOutputs[0].type === 'text-block') ? textOutputs[0].text : null;
+}
+
 
 function toLLMAttachment(attachment: Attachment, supportedOutputPartTypes: ComposerOutputPartType[], llmForTokenCount: DLLMId | null): LLMAttachment {
   const { converters, outputs } = attachment;
@@ -69,9 +74,9 @@ function toLLMAttachment(attachment: Attachment, supportedOutputPartTypes: Compo
   const isOutputAttachable = areAllOutputsSupported(outputs, supportedOutputPartTypes);
   const isOutputTextInlineable = areAllOutputsSupported(outputs, supportedOutputPartTypes.filter(pt => pt === 'text-block'));
 
-  const _collapsedOutputs = attachmentCollapseOutputs(outputs);
-  const approxTokenCount = llmForTokenCount
-    ? _collapsedOutputs.reduce((acc, output) => {
+  const attachmentOutputs = attachmentCollapseOutputs(null, outputs);
+  const tokenCountApprox = llmForTokenCount
+    ? attachmentOutputs.reduce((acc, output) => {
       if (output.type === 'text-block')
         return acc + countModelTokens(output.text, llmForTokenCount, 'attachments tokens count');
       console.warn('Unhandled token preview for output type:', output.type);
@@ -81,24 +86,34 @@ function toLLMAttachment(attachment: Attachment, supportedOutputPartTypes: Compo
 
   return {
     attachment,
+    attachmentOutputs,
     isUnconvertible,
     isOutputMissing,
     isOutputAttachable,
     isOutputTextInlineable,
-    tokenCountApprox: approxTokenCount,
+    tokenCountApprox,
   };
 }
 
-
 function areAllOutputsSupported(outputs: ComposerOutputMultiPart, supportedOutputPartTypes: ComposerOutputPartType[]) {
   return outputs.length
     ? outputs.every(output => supportedOutputPartTypes.includes(output.type))
     : false;
 }
 
-export function attachmentCollapseOutputs(outputs: ComposerOutputMultiPart): ComposerOutputMultiPart {
+function attachmentCollapseOutputs(initialTextBlockText: string | null, outputs: ComposerOutputMultiPart): ComposerOutputMultiPart {
   const accumulatedOutputs: ComposerOutputMultiPart = [];
 
+  // if there's initial text, make it a collapsible default (unquited) text block
+  if (initialTextBlockText !== null) {
+    accumulatedOutputs.push({
+      type: 'text-block',
+      text: initialTextBlockText,
+      title: null,
+      collapsible: true,
+    });
+  }
+
   // Accumulate attachment outputs of the same type and 'collapsible' into a single object of that type.
   for (const output of outputs) {
     const last = accumulatedOutputs[accumulatedOutputs.length - 1];