InReferenceTo: multi-sentence, multi-role

src/apps/chat/components/ChatMessageList.tsx

@@ -11,7 +11,7 @@ import type { DConversationId } from '~/common/stores/chat/chat.conversation';
 import type { DMessageFragment, DMessageFragmentId } from '~/common/stores/chat/chat.fragments';
 import { InlineError } from '~/common/components/InlineError';
 import { ShortcutKey, useGlobalShortcuts } from '~/common/components/shortcuts/useGlobalShortcuts';
-import { createDMessageTextContent, DMessageId, DMessageUserFlag, messageToggleUserFlag } from '~/common/stores/chat/chat.message';
+import { createDMessageTextContent, DMessageId, DMessageUserFlag, DMetaReferenceItem, messageToggleUserFlag } from '~/common/stores/chat/chat.message';
 import { getConversation, useChatStore } from '~/common/stores/chat/store-chats';
 import { optimaOpenPreferences } from '~/common/layout/optima/useOptima';
 import { useBrowserTranslationWarning } from '~/common/components/useIsBrowserTranslating';
@@ -24,6 +24,7 @@ import { CleanerMessage, MessagesSelectionHeader } from './message/CleanerMessag
 import { Ephemerals } from './Ephemerals';
 import { PersonaSelector } from './persona-selector/PersonaSelector';
 import { useChatAutoSuggestHTMLUI, useChatShowSystemMessages } from '../store-app-chat';
+import { useChatComposerOverlayStore } from '~/common/chat-overlay/store-chat-overlay';
 
 
 /**
@@ -68,7 +69,7 @@ export function ChatMessageList(props: {
 
   // derived state
   const { conversationHandler, conversationId, capabilityHasT2I, onConversationBranch, onConversationExecuteHistory, onTextDiagram, onTextImagine, onTextSpeak } = props;
-
+  const composeCanAddReplyTo = useChatComposerOverlayStore(conversationHandler?.getOverlayStore() ?? null, state => state.inReferenceTo?.length < 5);
 
   // text actions
 
@@ -150,8 +151,8 @@ export function ChatMessageList(props: {
     }), false, false);
   }, [props.conversationHandler]);
 
-  const handleReplyTo = React.useCallback((_messageId: DMessageId, text: string) => {
-    props.conversationHandler?.getOverlayStore().getState().setReplyToText(text);
+  const handleAddInReferenceTo = React.useCallback((item: DMetaReferenceItem) => {
+    props.conversationHandler?.getOverlayStore().getState().addInReferenceTo(item);
   }, [props.conversationHandler]);
 
   const handleTextDiagram = React.useCallback(async (messageId: DMessageId, text: string) => {
@@ -294,6 +295,7 @@ export function ChatMessageList(props: {
               isImagining={isImagining}
               isSpeaking={isSpeaking}
               showUnsafeHtml={danger_experimentalHtmlWebUi}
+              onAddInReferenceTo={!composeCanAddReplyTo ? undefined : handleAddInReferenceTo}
               onMessageAssistantFrom={handleMessageAssistantFrom}
               onMessageBeam={handleMessageBeam}
               onMessageBranch={handleMessageBranch}
@@ -304,7 +306,6 @@ export function ChatMessageList(props: {
               onMessageFragmentReplace={handleMessageReplaceFragment}
               onMessageToggleUserFlag={handleMessageToggleUserFlag}
               onMessageTruncate={handleMessageTruncate}
-              onReplyTo={handleReplyTo}
               onTextDiagram={handleTextDiagram}
               onTextImagine={capabilityHasT2I ? handleTextImagine : undefined}
               onTextSpeak={isSpeakable ? handleTextSpeak : undefined}

src/apps/chat/components/composer/Composer.tsx

@@ -26,7 +26,7 @@ import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
 import { ButtonAttachFilesMemo } from '~/common/components/ButtonAttachFiles';
 import { ChatBeamIcon } from '~/common/components/icons/ChatBeamIcon';
 import { ConversationsManager } from '~/common/chat-overlay/ConversationsManager';
-import { DMessageMetadata, messageFragmentsReduceText } from '~/common/stores/chat/chat.message';
+import { DMessageMetadata, DMetaReferenceItem, messageFragmentsReduceText } from '~/common/stores/chat/chat.message';
 import { ShortcutKey, ShortcutObject, useGlobalShortcuts } from '~/common/components/shortcuts/useGlobalShortcuts';
 import { animationEnterBelow } from '~/common/util/animUtils';
 import { browserSpeechRecognitionCapability, SpeechResult, useSpeechRecognition } from '~/common/components/useSpeechRecognition';
@@ -42,7 +42,7 @@ import { optimaOpenPreferences } from '~/common/layout/optima/useOptima';
 import { platformAwareKeystrokes } from '~/common/components/KeyStroke';
 import { supportsScreenCapture } from '~/common/util/screenCaptureUtils';
 import { useAppStateStore } from '~/common/state/store-appstate';
-import { useChatOverlayStore } from '~/common/chat-overlay/store-chat-overlay';
+import { useChatComposerOverlayStore } from '~/common/chat-overlay/store-chat-overlay';
 import { useDebouncer } from '~/common/components/useDebouncer';
 import { useUICounter, useUIPreferencesStore } from '~/common/state/store-ui';
 import { useUXLabsStore } from '~/common/state/store-ux-labs';
@@ -141,10 +141,9 @@ export function Composer(props: {
     ? ConversationsManager.getHandler(props.targetConversationId)?.getOverlayStore() || null
     : null;
 
-  // composer-overlay: for the reply-to state, comes from the conversation overlay
-  const { replyToGenerateText } = useChatOverlayStore(conversationOverlayStore, useShallow(store => ({
-    replyToGenerateText: (chatExecuteMode === 'generate-content' || chatExecuteMode === 'generate-text-v1') ? store.replyToText?.trim() ?? undefined : undefined,
-  })));
+  // composer-overlay: for the in-reference-to state, comes from the conversation overlay
+  const allowInReferenceTo = chatExecuteMode === 'generate-content' || chatExecuteMode === 'generate-text-v1';
+  const inReferenceTo = useChatComposerOverlayStore(conversationOverlayStore, store => allowInReferenceTo ? store.inReferenceTo : null);
 
   // don't load URLs if the user is typing a command or there's no capability
   const enableLoadURLsInComposer = useBrowseCapability().inComposer && !composeText.startsWith('/');
@@ -206,14 +205,18 @@ export function Composer(props: {
 
   // Overlay actions
 
-  const handleReplyToClear = React.useCallback(() => {
-    conversationOverlayStore?.getState().setReplyToText(null);
+  const handleRemoveInReferenceTo = React.useCallback((item: DMetaReferenceItem) => {
+    conversationOverlayStore?.getState().removeInReferenceTo(item);
+  }, [conversationOverlayStore]);
+
+  const handleInReferenceToClear = React.useCallback(() => {
+    conversationOverlayStore?.getState().clearInReferenceTo();
   }, [conversationOverlayStore]);
 
   React.useEffect(() => {
-    if (replyToGenerateText)
+    if (inReferenceTo?.length)
       setTimeout(() => composerTextAreaRef.current?.focus(), 1 /* prevent focus theft */);
-  }, [composerTextAreaRef, replyToGenerateText]);
+  }, [composerTextAreaRef, inReferenceTo]);
 
 
   // Primary button
@@ -221,8 +224,8 @@ export function Composer(props: {
   const handleClear = React.useCallback(() => {
     setComposeText('');
     attachmentsRemoveAll();
-    handleReplyToClear();
-  }, [attachmentsRemoveAll, handleReplyToClear, setComposeText]);
+    handleInReferenceToClear();
+  }, [attachmentsRemoveAll, handleInReferenceToClear, setComposeText]);
 
 
   const handleSendAction = React.useCallback(async (_chatExecuteMode: ChatExecuteMode, composerText: string): Promise<boolean> => {
@@ -250,13 +253,15 @@ export function Composer(props: {
       return false;
     }
 
+    // prepare the metadata
+    const metadata = inReferenceTo?.length ? { inReferenceTo: inReferenceTo } : undefined;
+
     // send the message - NOTE: if successful, the ownership of the fragments is transferred to the receiver, so we just clear them
-    const metadata = replyToGenerateText ? { inReplyToText: replyToGenerateText } : undefined;
     const enqueued = onAction(targetConversationId, _chatExecuteMode, fragments, metadata);
     if (enqueued)
       handleClear();
     return enqueued;
-  }, [attachmentsTakeAllFragments, handleClear, onAction, replyToGenerateText, targetConversationId]);
+  }, [attachmentsTakeAllFragments, handleClear, inReferenceTo, onAction, targetConversationId]);
 
 
   const handleAppendTextAndSend = React.useCallback(async (appendText: string) => {
@@ -518,8 +523,8 @@ export function Composer(props: {
   const isReAct = chatExecuteMode === 'react-content';
   const isDraw = chatExecuteMode === 'generate-image';
 
-  const showChatReplyTo = !!replyToGenerateText;
-  const showChatExtras = isText && !showChatReplyTo;
+  const showChatInReferenceTo = !!inReferenceTo?.length;
+  const showChatExtras = isText && !showChatInReferenceTo;
 
   const sendButtonVariant: VariantProp = (isAppend || (isMobile && isTextBeam)) ? 'outlined' : 'solid';
 
@@ -539,7 +544,7 @@ export function Composer(props: {
     isDraw ? 'Describe an idea or a drawing...'
       : isReAct ? 'Multi-step reasoning question...'
         : isTextBeam ? 'Beam: combine the smarts of models...'
-          : showChatReplyTo ? 'Chat about this'
+          : showChatInReferenceTo ? 'Chat about this'
             : props.isDeveloperMode ? 'Chat with me' + (isDesktop ? ' · drop source' : '') + ' · attach code...'
               : props.capabilityHasT2I ? 'Chat · /beam · /draw · drop files...'
                 : 'Chat · /react · drop files...';
@@ -659,7 +664,7 @@ export function Composer(props: {
                     variant='outlined'
                     color={isDraw ? 'warning' : isReAct ? 'success' : undefined}
                     autoFocus
-                    minRows={isMobile ? 4 : showChatReplyTo ? 4 : 5}
+                    minRows={isMobile ? 4 : showChatInReferenceTo ? 4 : 5}
                     maxRows={isMobile ? 8 : 10}
                     placeholder={textPlaceholder}
                     value={composeText}
@@ -672,9 +677,9 @@ export function Composer(props: {
                       <ComposerTextAreaActions
                         agiAttachmentButton={agiAttachmentPromptsComponent}
                         agiAttachmentPrompts={agiAttachmentPrompts}
+                        inReferenceTo={inReferenceTo}
                         onAppendAndSend={handleAppendTextAndSend}
-                        onReplyToClear={handleReplyToClear}
-                        replyToText={replyToGenerateText}
+                        onRemoveReferenceTo={handleRemoveInReferenceTo}
                       />
                     }
                     slotProps={{
@@ -689,15 +694,15 @@ export function Composer(props: {
                     }}
                     sx={{
                       backgroundColor: 'background.level1',
-                      '&:focus-within': { backgroundColor: 'background.popup', '.reply-to-bubble': { backgroundColor: 'background.popup' } },
+                      '&:focus-within': { backgroundColor: 'background.popup', '.in-reference-to-bubble': { backgroundColor: 'background.popup' } },
                       lineHeight: lineHeightTextareaMd,
                     }} />
 
-                  {!showChatReplyTo && tokenLimit > 0 && (tokensComposer > 0 || (tokensHistory + tokensReponseMax) > 0) && (
+                  {!showChatInReferenceTo && tokenLimit > 0 && (tokensComposer > 0 || (tokensHistory + tokensReponseMax) > 0) && (
                     <TokenProgressbarMemo direct={tokensComposer} history={tokensHistory} responseMax={tokensReponseMax} limit={tokenLimit} tokenPriceIn={tokenPriceIn} tokenPriceOut={tokenPriceOut} />
                   )}
 
-                  {!showChatReplyTo && tokenLimit > 0 && (
+                  {!showChatInReferenceTo && tokenLimit > 0 && (
                     <TokenBadgeMemo direct={tokensComposer} history={tokensHistory} responseMax={tokensReponseMax} limit={tokenLimit} tokenPriceIn={tokenPriceIn} tokenPriceOut={tokenPriceOut} showCost={labsShowCost} enableHover={!isMobile} showExcess absoluteBottomRight />
                   )}
 

src/apps/chat/components/composer/ComposerTextAreaActions.tsx

@@ -2,19 +2,21 @@ import * as React from 'react';
 
 import { Box, Sheet } from '@mui/joy';
 
-import { ReplyToBubble } from '../message/ReplyToBubble';
+import type { DMetaReferenceItem } from '~/common/stores/chat/chat.message';
+
+import { InReferenceToBubble } from '../message/InReferenceToBubble';
 
 
 export function ComposerTextAreaActions(props: {
   agiAttachmentButton?: React.ReactNode,
   agiAttachmentPrompts?: string[],
-  replyToText?: string,
+  inReferenceTo?: DMetaReferenceItem[] | null
   onAppendAndSend: (appendText: string) => Promise<void>,
-  onReplyToClear: () => void,
+  onRemoveReferenceTo: (item: DMetaReferenceItem) => void,
 }) {
 
   // skip the component if there's nothing to show
-  if (!props.agiAttachmentPrompts?.length && !props.agiAttachmentButton && props.replyToText === undefined)
+  if (!props.agiAttachmentPrompts?.length && !props.agiAttachmentButton && !props.inReferenceTo?.length)
     return null;
 
   return (
@@ -38,14 +40,15 @@ export function ComposerTextAreaActions(props: {
       },
     }}>
 
-      {/* Reply-To bubble */}
-      {props.replyToText !== undefined && (
-        <ReplyToBubble
-          replyToText={props.replyToText}
-          onClear={props.onReplyToClear}
-          className='reply-to-bubble'
+      {/* In-Reference-To bubbles */}
+      {props.inReferenceTo?.map((item, index) => (
+        <InReferenceToBubble
+          key={index}
+          item={item}
+          onRemove={props.onRemoveReferenceTo}
+          className='in-reference-to-bubble'
         />
-      )}
+      ))}
 
       {/* Auto-Prompts from attachments */}
       {!!props.agiAttachmentPrompts?.length && (

src/apps/chat/components/message/ChatMessage.tsx

@@ -25,7 +25,7 @@ import VerticalAlignBottomIcon from '@mui/icons-material/VerticalAlignBottom';
 
 import { ChatBeamIcon } from '~/common/components/icons/ChatBeamIcon';
 import { CloseableMenu } from '~/common/components/CloseableMenu';
-import { DMessage, DMessageId, DMessageUserFlag, messageFragmentsReduceText, messageHasUserFlag } from '~/common/stores/chat/chat.message';
+import { DMessage, DMessageId, DMessageUserFlag, DMetaReferenceItem, messageFragmentsReduceText, messageHasUserFlag } from '~/common/stores/chat/chat.message';
 import { KeyStroke } from '~/common/components/KeyStroke';
 import { adjustContentScaling, themeScalingMap, themeZIndexPageBar } from '~/common/app.theme';
 import { animationColorRainbow } from '~/common/util/animUtils';
@@ -38,7 +38,7 @@ import { ContentFragmentsWithInlineEdit } from './fragments-content/ContentFragm
 import { ContinueFragment } from './ContinueFragment';
 import { DocumentAttachmentFragments } from './fragments-attachment-doc/DocumentAttachmentFragments';
 import { ImageAttachmentFragments } from './fragments-attachment-image/ImageAttachmentFragments';
-import { ReplyToBubble } from './ReplyToBubble';
+import { InReferenceToBubble } from './InReferenceToBubble';
 import { avatarIconSx, makeMessageAvatarIcon, messageAsideColumnSx, messageBackground, messageZenAsideColumnSx } from './messageUtils';
 import { useChatShowTextDiff } from '../../store-app-chat';
 
@@ -77,6 +77,7 @@ export function ChatMessage(props: {
   showUnsafeHtml?: boolean,
   adjustContentScaling?: number,
   topDecorator?: React.ReactNode,
+  onAddInReferenceTo?: (item: DMetaReferenceItem) => void,
   onMessageAssistantFrom?: (messageId: string, offset: number) => Promise<void>,
   onMessageBeam?: (messageId: string) => Promise<void>,
   onMessageBranch?: (messageId: string) => void,
@@ -87,7 +88,6 @@ export function ChatMessage(props: {
   onMessageFragmentReplace?: (messageId: DMessageId, fragmentId: DMessageFragmentId, newFragment: DMessageFragment) => void,
   onMessageToggleUserFlag?: (messageId: string, flag: DMessageUserFlag) => void,
   onMessageTruncate?: (messageId: string) => void,
-  onReplyTo?: (messageId: string, selectedText: string) => void,
   onTextDiagram?: (messageId: string, text: string) => Promise<void>,
   onTextImagine?: (text: string) => Promise<void>,
   onTextSpeak?: (text: string) => Promise<void>,
@@ -203,7 +203,7 @@ export function ChatMessage(props: {
 
   // Message Operations Menu
 
-  const { onMessageToggleUserFlag } = props;
+  const { onAddInReferenceTo, onMessageToggleUserFlag } = props;
 
   const handleOpsMenuToggle = React.useCallback((event: React.MouseEvent<HTMLElement>) => {
     event.preventDefault(); // added for the Right mouse click (to prevent the menu)
@@ -273,10 +273,10 @@ export function ChatMessage(props: {
     }
   };
 
-  const handleOpsReplyTo = (e: React.MouseEvent) => {
+  const handleOpsAddInReferenceTo = (e: React.MouseEvent) => {
     e.preventDefault();
-    if (props.onReplyTo && textSel.trim().length >= BUBBLE_MIN_TEXT_LENGTH) {
-      props.onReplyTo(messageId, textSel.trim());
+    if (onAddInReferenceTo && textSel.trim().length >= BUBBLE_MIN_TEXT_LENGTH) {
+      onAddInReferenceTo({ mrt: 'dmsg', mText: textSel.trim(), mRole: messageRole /*, messageId*/ });
       handleCloseOpsMenu();
       closeContextMenu();
       closeBubble();
@@ -573,14 +573,15 @@ export function ChatMessage(props: {
             </Typography>
           )}
 
-          {/* Reply-To Bubble */}
-          {!!messageMetadata?.inReplyToText && (
-            <ReplyToBubble
-              inlineUserMessage
-              replyToText={messageMetadata.inReplyToText}
-              className='reply-to-bubble'
+          {/* In-Reference-To Bubble */}
+          {messageMetadata?.inReferenceTo?.map((item, index) => (
+            <InReferenceToBubble
+              key={'irt-' + index}
+              item={item}
+              bubbleVariant='message'
+              className='in-reference-to-bubble'
             />
-          )}
+          ))}
 
           {/* Image Attachment Fragments - just for a prettier display on top of the message */}
           {imageAttachments.length >= 1 && !isEditingText && (
@@ -829,8 +830,8 @@ export function ChatMessage(props: {
                 },
               }}
             >
-              {!!props.onReplyTo && <Tooltip disableInteractive arrow placement='top' title={fromAssistant ? 'Reply' : 'Refer To'}>
-                <IconButton color='primary' onClick={handleOpsReplyTo}>
+              {!!onAddInReferenceTo && <Tooltip disableInteractive arrow placement='top' title={fromAssistant ? 'Reply' : 'Refer To'}>
+                <IconButton color='primary' onClick={handleOpsAddInReferenceTo}>
                   <ReplyRoundedIcon sx={{ fontSize: 'xl' }} />
                 </IconButton>
               </Tooltip>}
@@ -839,7 +840,7 @@ export function ChatMessage(props: {
               {/*    <ChatBeamIcon sx={{ fontSize: 'xl' }} />*/}
               {/*  </IconButton>*/}
               {/*</Tooltip>}*/}
-              {!!props.onReplyTo && fromAssistant && <MoreVertIcon sx={{ color: 'neutral.outlinedBorder', fontSize: 'md' }} />}
+              {!!onAddInReferenceTo && fromAssistant && <MoreVertIcon sx={{ color: 'neutral.outlinedBorder', fontSize: 'md' }} />}
               <Tooltip disableInteractive arrow placement='top' title='Copy'>
                 <IconButton onClick={handleOpsCopy}>
                   <ContentCopyIcon />

src/apps/chat/components/message/InReferenceToBubble.tsx

@@ -5,6 +5,8 @@ import { Box, IconButton, Tooltip, Typography } from '@mui/joy';
 import CloseRoundedIcon from '@mui/icons-material/CloseRounded';
 import ReplyRoundedIcon from '@mui/icons-material/ReplyRounded';
 
+import type { DMetaReferenceItem } from '~/common/stores/chat/chat.message';
+
 
 // configuration
 const INLINE_COLOR = 'primary';
@@ -52,38 +54,52 @@ export const inlineMessageBubbleSx: SxProps = {
 };
 
 
-export function ReplyToBubble(props: {
-  replyToText?: string,
-  inlineUserMessage?: boolean
-  onClear?: () => void,
+export function InReferenceToBubble(props: {
+  item: DMetaReferenceItem,
+  onRemove?: (item: DMetaReferenceItem) => void,
   className?: string,
+  bubbleVariant?: 'message',
 }) {
+
+  // derived state
+
+  const variantMessage = props.bubbleVariant === 'message';
+
+  // handlers
+
+  const { onRemove } = props;
+
+  const handleRemoveClicked = React.useCallback(() => {
+    onRemove?.(props.item);
+  }, [onRemove, props.item]);
+
   return (
-    <Box className={props.className} sx={!props.inlineUserMessage ? bubbleComposerSx : inlineMessageBubbleSx}>
+    <Box className={props.className} sx={!variantMessage ? bubbleComposerSx : inlineMessageBubbleSx}>
 
       <Tooltip disableInteractive arrow title='Referring to this assistant text' placement='top'>
         <ReplyRoundedIcon sx={{
-          color: props.inlineUserMessage ? `${INLINE_COLOR}.outlinedColor` : 'primary.solidBg',
+          color: variantMessage ? `${INLINE_COLOR}.outlinedColor` : 'primary.solidBg',
           fontSize: 'xl',
           mt: 0.125,
+          transform: props.item.mRole === 'assistant' ? undefined : 'rotate(105deg)',
         }} />
       </Tooltip>
 
       <Typography level='body-sm' sx={{
         flex: 1,
         ml: 1,
-        mr: props.inlineUserMessage ? 1 : 0.5,
+        mr: variantMessage ? 1 : 0.5,
         overflow: 'auto',
-        maxHeight: props.inlineUserMessage ? '8rem' : '5.75rem',
+        maxHeight: variantMessage ? '8rem' : '5.75rem',
         lineHeight: 'xl',
-        color: props.inlineUserMessage ? 'primary.softActiveColor' : 'text.secondary',
+        color: variantMessage ? 'primary.softActiveColor' : 'text.secondary',
         whiteSpace: 'break-spaces', // 'balance'
       }}>
-        {props.replyToText}
+        {props.item.mText}
       </Typography>
 
-      {!!props.onClear && (
-        <IconButton size='sm' onClick={props.onClear} sx={{ my: -0.5, background: 'none' }}>
+      {!!props.onRemove && (
+        <IconButton size='sm' onClick={handleRemoveClicked} sx={{ my: -0.5, background: 'none' }}>
           <CloseRoundedIcon />
         </IconButton>
       )}

src/common/chat-overlay/ConversationHandler.ts

@@ -40,7 +40,7 @@ export class ConversationHandler {
 
   inlineUpdatePurposeInHistory(history: DMessage[], assistantLlmId: DLLMId | undefined): DMessage[] {
     const purposeId = getConversationSystemPurposeId(this.conversationId);
-    // TODO: HACK: find the persona identiy separately from the "first system message", as e.g. right now would take the reply-to and promote as system
+    // TODO: HACK: find the persona identiy separately from the "first system message"
     const systemMessageIndex = history.findIndex(m => m.role === 'system');
 
     let systemMessage: DMessage = systemMessageIndex >= 0

src/common/chat-overlay/store-chat-overlay.ts

@@ -1,8 +1,8 @@
-import { StoreApi, useStore } from 'zustand';
+import { useStore } from 'zustand';
 import { createStore as createVanillaStore } from 'zustand/vanilla';
 
 import { AttachmentDraftsStoreApi, AttachmentsDraftsStore, createAttachmentDraftsStoreSlice } from '~/common/attachment-drafts/store-attachment-drafts-slice';
-import { ComposerOverlayStore, createComposerOverlayStoreSlice } from './store-composeroverlay-slice';
+import { ComposerOverlayStore, ComposerOverlayStoreApi, createComposerOverlayStoreSlice } from './store-composeroverlay-slice';
 
 
 // Note: at this time there are numerous overlay stores, including beam (vanilla), ephemerals (EventTarget), and this one.
@@ -19,7 +19,10 @@ export const createPerChatVanillaStore = () => createVanillaStore<PerChatOverlay
 
 const fallbackStoreApi = createPerChatVanillaStore();
 
-export const useChatOverlayStore = <T, >(vanillaStore: Readonly<StoreApi<PerChatOverlayStore>> | null, selector: (store: PerChatOverlayStore) => T): T =>
+// export const useChatOverlayStore = <T, >(vanillaStore: Readonly<StoreApi<PerChatOverlayStore>> | null, selector: (store: PerChatOverlayStore) => T): T =>
+//   useStore(vanillaStore || fallbackStoreApi, selector);
+
+export const useChatComposerOverlayStore = <T, >(vanillaStore: Readonly<ComposerOverlayStoreApi> | null, selector: (store: ComposerOverlayStore) => T): T =>
   useStore(vanillaStore || fallbackStoreApi, selector);
 
 export const useChatAttachmentsStore = <T, >(vanillaStore: Readonly<AttachmentDraftsStoreApi> | null, selector: (store: AttachmentsDraftsStore) => T): T =>

src/common/chat-overlay/store-composeroverlay-slice.ts

@@ -1,34 +1,49 @@
 import type { StateCreator } from 'zustand/vanilla';
 
+import type { DMetaReferenceItem } from '~/common/stores/chat/chat.message';
+import type { StoreApi } from 'zustand';
+
 
 /// Chat Overlay Store: per-chat overlay state ///
 
 interface ComposerOverlayState {
 
-  // if set, this is the 'reply to' mode text
-  replyToText: string | null;
+  // list of all the references that the composer is holding to, before sending them out in the next message
+  inReferenceTo: DMetaReferenceItem[];
 
 }
 
 export interface ComposerOverlayStore extends ComposerOverlayState {
 
-  setReplyToText: (text: string | null) => void;
+  addInReferenceTo: (item: DMetaReferenceItem) => void;
+  removeInReferenceTo: (item: DMetaReferenceItem) => void;
+  clearInReferenceTo: () => void;
 
 }
 
+export type ComposerOverlayStoreApi = StoreApi<ComposerOverlayStore>;
+
 /**
  * NOTE: the Composer state is managed primarily by the component, however there's some state that's:
- *  - associated with the chat (e.g. reply-to text)
+ *  - associated with the chat (e.g. in-reference-to text)
  *  - persisted across chats
  *
- * This slice manages the reply-to text state, but there's also a sister slice that manages the attachment drafts.
+ * This slice manages the in-reference-to text state, but there's also a sister slice that manages the attachment drafts.
  */
 export const createComposerOverlayStoreSlice: StateCreator<ComposerOverlayStore, [], [], ComposerOverlayStore> = (_set, _get) => ({
 
   // init state
-  replyToText: null,
+  inReferenceTo: [],
 
   // actions
-  setReplyToText: (text: string | null) => _set({ replyToText: text }),
+  addInReferenceTo: (item) => _set(state => ({
+    inReferenceTo: [...state.inReferenceTo, item],
+  })),
 
-});
+  removeInReferenceTo: (item) => _set(state => ({
+    inReferenceTo: state.inReferenceTo.filter((i) => i !== item),
+  })),
+
+  clearInReferenceTo: () => _set({ inReferenceTo: [] }),
+
+});

src/common/stores/chat/chat.message.ts

@@ -51,8 +51,15 @@ export type DMessageGenerator = {
 // Metadata
 
 export interface DMessageMetadata {
-  inReplyToText?: string;           // text this was in reply to
-  ranOutOfTokens?: true;            // if the message was cut off due to token limit
+  inReferenceTo?: DMetaReferenceItem[]; // text this was in reply to
+  ranOutOfTokens?: true;                // if the message was cut off due to token limit
+}
+
+export interface DMetaReferenceItem {
+  mrt: 'dmsg';                          // for future type discrimination
+  mText: string;
+  mRole: DMessageRole;
+  // messageId?: string;
 }
 
 

src/common/stores/chat/chats.converters.ts

@@ -54,6 +54,9 @@ export namespace V4ToHeadConverters {
   }
 
   export function dev_inMemHeadUpgradeDMessage(m: DMessage): void {
+    // TODO: remove insides for 2.0.0
+
+    // uplevel fragments
     for (const fragment of m.fragments) {
 
       // Result of a rename of DMessageDocPart.type -> .vdt
@@ -64,7 +67,15 @@ export namespace V4ToHeadConverters {
           delete docPart.type;
         }
       }
+    }
 
+    // uplevel metadata: if (metadata?.inReplyToText) cm.metadata = { inReferenceTo: [{ mrt: 'dmsg', mText: metadata.inReplyToText, mRole: 'assistant' }] };
+    if (m.metadata && 'inReplyToText' in m.metadata && m.metadata.inReplyToText) {
+      const replyToText = m.metadata.inReplyToText as string;
+      delete m.metadata.inReplyToText;
+      m.metadata.inReferenceTo = [{
+        mrt: 'dmsg', mText: replyToText, mRole: 'assistant',
+      }];
     }
   }
 
@@ -231,7 +242,10 @@ export namespace V3StoreDataToHead {
       if (id) cm.id = id;
       if (purposeId) cm.purposeId = purposeId;
       if (originLLM) cm.originLLM = originLLM;
-      if (metadata) cm.metadata = metadata;
+      if (metadata?.inReplyToText) {
+        if (!cm.metadata) cm.metadata = {};
+        cm.metadata.inReferenceTo = [{ mrt: 'dmsg', mText: metadata.inReplyToText, mRole: 'assistant' }];
+      }
       if (userFlags) cm.userFlags = userFlags;
       cm.tokenCount = tokenCount || 0;
       if (created) cm.created = created;
@@ -268,9 +282,9 @@ export namespace V3StoreDataToHead {
     originLLM?: string;               // only assistant - model that generated this message, goes beyond known models
 
     metadata?: {                      // metadata, mainly at creation and for UI
-      inReplyToText?: string;         // text this was in reply to
+      inReplyToText?: string;         // V3: text this was in reply to
     };
-    userFlags?: ('starred')[];             // (UI) user-set per-message flags
+    userFlags?: ('starred')[];        // (UI) user-set per-message flags
 
     tokenCount: number;               // cache for token count, using the current Conversation model (0 = not yet calculated)
 

src/modules/aix/client/aix.client.fromDMessages.api.ts

@@ -1,11 +1,11 @@
 import { getImageAsset } from '~/modules/dblobs/dblobs.images';
 
-import type { DMessage } from '~/common/stores/chat/chat.message';
+import type { DMessage, DMetaReferenceItem } from '~/common/stores/chat/chat.message';
 import { DMessageImageRefPart, isContentFragment, isContentOrAttachmentFragment, isTextPart } from '~/common/stores/chat/chat.fragments';
 import { LLMImageResizeMode, resizeBase64ImageIfNeeded } from '~/common/util/imageUtils';
 
 // NOTE: pay particular attention to the "import type", as this is importing from the server-side Zod definitions
-import type { AixAPIChatGenerate_Request, AixMessages_ModelMessage, AixMessages_UserMessage, AixParts_InlineImagePart, AixParts_MetaReplyToPart } from '../server/api/aix.wiretypes';
+import type { AixAPIChatGenerate_Request, AixMessages_ModelMessage, AixMessages_UserMessage, AixParts_InlineImagePart, AixParts_MetaInReferenceToPart } from '../server/api/aix.wiretypes';
 
 // TODO: remove console messages to zero, or replace with throws or something
 
@@ -78,12 +78,12 @@ export async function aixChatGenerateRequestFromDMessages(messageSequence: Reado
         return uMsg;
       }, Promise.resolve({ role: 'user', parts: [] } as AixMessages_UserMessage));
 
-      // handle reply-to metadata, adding a part right after the user text (or at the beginning)
-      if (m.metadata?.inReplyToText) {
+      // handle in-reference-to metadata, adding a part right after the user text (or at the beginning)
+      if (m.metadata?.inReferenceTo?.length) {
         // find the index of the tast text part
         const lastTextPartIndex = aixChatMessageUser.parts.findLastIndex(p => p.pt === 'text');
         // insert the meta part after the last text part (and before the first attachment)
-        aixChatMessageUser.parts.splice(lastTextPartIndex + 1, 0, _clientCreateAixMetaReplyToPart(m.metadata.inReplyToText));
+        aixChatMessageUser.parts.splice(lastTextPartIndex + 1, 0, _clientCreateAixMetaInReferenceToPart(m.metadata.inReferenceTo));
       }
 
       acc.chatSequence.push(aixChatMessageUser);
@@ -179,6 +179,6 @@ function _clientCreateAixInlineImagePart(base64: string, mimeType: string): AixP
   return { pt: 'inline_image', mimeType: (mimeType || 'image/png') as AixParts_InlineImagePart['mimeType'], base64 };
 }
 
-function _clientCreateAixMetaReplyToPart(replyTo: string): AixParts_MetaReplyToPart {
-  return { pt: 'meta_reply_to', replyTo };
+function _clientCreateAixMetaInReferenceToPart(items: DMetaReferenceItem[]): AixParts_MetaInReferenceToPart {
+  return { pt: 'meta_in_reference_to', referTo: items };
 }

src/modules/aix/server/api/aix.wiretypes.ts

@@ -19,7 +19,7 @@ import { openAIAccessSchema } from '~/modules/llms/server/openai/openai.router';
 // Export types
 export type AixParts_DocPart = z.infer<typeof AixWire_Parts.DocPart_schema>;
 export type AixParts_InlineImagePart = z.infer<typeof AixWire_Parts.InlineImagePart_schema>;
-export type AixParts_MetaReplyToPart = z.infer<typeof AixWire_Parts.MetaReplyToPart_schema>;
+export type AixParts_MetaInReferenceToPart = z.infer<typeof AixWire_Parts.MetaInReferenceToPart_schema>;
 
 export type AixMessages_SystemMessage = z.infer<typeof AixWire_Content.SystemInstruction_schema>;
 export type AixMessages_UserMessage = z.infer<typeof AixWire_Content.UserMessage_schema>;
@@ -196,9 +196,13 @@ export namespace AixWire_Parts {
 
   // Metas
 
-  export const MetaReplyToPart_schema = z.object({
-    pt: z.literal('meta_reply_to'),
-    replyTo: z.string(),
+  export const MetaInReferenceToPart_schema = z.object({
+    pt: z.literal('meta_in_reference_to'),
+    referTo: z.array(z.object({
+      mrt: z.literal('dmsg'),
+      mText: z.string(),
+      mRole: z.string(),
+    })),
   });
 
 }
@@ -219,7 +223,7 @@ export namespace AixWire_Content {
       AixWire_Parts.TextPart_schema,
       AixWire_Parts.InlineImagePart_schema,
       AixWire_Parts.DocPart_schema,
-      AixWire_Parts.MetaReplyToPart_schema,
+      AixWire_Parts.MetaInReferenceToPart_schema,
     ])),
   });
 

src/modules/aix/server/dispatch/chatGenerate/adapters/anthropic.messageCreate.ts

@@ -1,4 +1,6 @@
-import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
+import { escapeXml } from '~/server/wire';
+
+import { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
 import { AnthropicWire_API_Message_Create, AnthropicWire_Blocks } from '../../wiretypes/anthropic.wiretypes';
 
 
@@ -98,8 +100,10 @@ function* _generateAnthropicMessagesContentBlocks({ parts, role }: AixMessages_C
             yield { role: 'user', content: AnthropicWire_Blocks.TextBlock('```' + (part.ref || '') + '\n' + part.data.text + '\n```\n') };
             break;
 
-          case 'meta_reply_to':
-            yield { role: 'user', content: AnthropicWire_Blocks.TextBlock(`<context>The user is referring to this in particular: ${part.replyTo}</context>`) };
+          case 'meta_in_reference_to':
+            const irtXMLString = inReferenceTo_To_XMLString(part);
+            if (irtXMLString)
+              yield { role: 'user', content: AnthropicWire_Blocks.TextBlock(irtXMLString) };
             break;
 
           default:
@@ -206,3 +210,12 @@ function _toAnthropicToolChoice(itp: AixTools_ToolsPolicy): NonNullable<TRequest
       return { type: 'tool' as const, name: itp.function_call.name };
   }
 }
+
+export function inReferenceTo_To_XMLString(irt: AixParts_MetaInReferenceToPart): string | null {
+  const refs = irt.referTo.map(r => escapeXml(r.mText));
+  if (!refs.length)
+    return null; // `<context>User provides no specific references</context>`;
+  return refs.length === 1
+    ? `<context>User refers to this in particular:<ref>${refs[0]}</ref></context>`
+    : `<context>User refers to ${refs.length} items:<ref>${refs.join('</ref><ref>')}</ref></context>`;
+}

src/modules/aix/server/dispatch/chatGenerate/adapters/gemini.generateContent.ts

@@ -1,6 +1,8 @@
 import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixParts_DocPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
 import { GeminiWire_API_Generate_Content, GeminiWire_ContentParts, GeminiWire_Messages, GeminiWire_Safety } from '../../wiretypes/gemini.wiretypes';
 
+import { inReferenceTo_To_XMLString } from './anthropic.messageCreate';
+
 
 // configuration
 const hotFixImagePartsFirst = true;
@@ -90,8 +92,10 @@ function _toGeminiContents(chatSequence: AixMessages_ChatMessage[]): GeminiWire_
           parts.push(_toApproximateGeminiDocPart(part));
           break;
 
-        case 'meta_reply_to':
-          parts.push(_toApproximateGeminiReplyTo(part.replyTo));
+        case 'meta_in_reference_to':
+          const irtXMLString = inReferenceTo_To_XMLString(part);
+          if (irtXMLString)
+            parts.push(GeminiWire_ContentParts.TextPart(irtXMLString));
           break;
 
         case 'tool_invocation':
@@ -242,7 +246,3 @@ function _toGeminiSafetySettings(threshold: GeminiWire_Safety.HarmBlockThreshold
 function _toApproximateGeminiDocPart(aixPartsDocPart: AixParts_DocPart): GeminiWire_ContentParts.ContentPart {
   return GeminiWire_ContentParts.TextPart(`\`\`\`${aixPartsDocPart.ref || ''}\n${aixPartsDocPart.data.text}\n\`\`\`\n`);
 }
-
-function _toApproximateGeminiReplyTo(replyTo: string): GeminiWire_ContentParts.ContentPart {
-  return GeminiWire_ContentParts.TextPart(`<context>The user is referring to this in particular: ${replyTo}</context>`);
-}

src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts

@@ -1,6 +1,6 @@
 import type { OpenAIDialects } from '~/modules/llms/server/openai/openai.router';
 
-import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
+import { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes';
 import { OpenAIWire_API_Chat_Completions, OpenAIWire_ContentParts, OpenAIWire_Messages } from '../../wiretypes/openai.wiretypes';
 
 
@@ -209,9 +209,8 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | undefined,
                 chatMessages.push({ role: 'user', content: [imageContentPart] });
               break;
 
-            case 'meta_reply_to':
-              const context = `The user is referring to this in particular:\n{{ReplyToText}}`.replace('{{ReplyToText}}', part.replyTo);
-              chatMessages.push({ role: 'system', content: context });
+            case 'meta_in_reference_to':
+              chatMessages.push({ role: 'system', content: _toOpenAIInReferenceToText(part) });
               break;
 
             default:
@@ -353,3 +352,29 @@ function _toOpenAIToolChoice(openAIDialect: OpenAIDialects, itp: AixTools_ToolsP
       return { type: 'function' as const, function: { name: itp.function_call.name } };
   }
 }
+
+function _toOpenAIInReferenceToText(irt: AixParts_MetaInReferenceToPart): string {
+  // Get the item texts without roles
+  const items = irt.referTo.map(r => r.mText);
+  if (items.length === 0)
+    return 'CONTEXT: The user provides no specific references.';
+
+  const isShortItem = (text: string): boolean =>
+    text.split('\n').length <= 3 && text.length <= 200;
+
+  const formatItem = (text: string, index?: number): string => {
+    if (isShortItem(text)) {
+      const formatted = text.replace(/\n/g, ' ').replace(/\s+/g, ' ');
+      return index !== undefined ? `${index + 1}. "${formatted}"` : `"${formatted}"`;
+    }
+    return `${index !== undefined ? `ITEM ${index + 1}:\n` : ''}---\n${text}\n---`;
+  };
+
+  // Formely: `The user is referring to this in particular:\n{{ReplyToText}}`.replace('{{ReplyToText}}', part.replyTo);
+  if (items.length === 1)
+    return `CONTEXT: The user is referring to this in particular:\n${formatItem(items[0])}`;
+
+  const allShort = items.every(isShortItem);
+  return `CONTEXT: The user is referring to these ${items.length} in particular:\n\n${
+    items.map((text, index) => formatItem(text, index)).join(allShort ? '\n' : '\n\n')}`;
+}

src/server/wire.ts

@@ -120,3 +120,24 @@ export class ServerDebugWireEvents {
 }
 
 export const createServerDebugWireEvents = () => SERVER_DEBUG_WIRE ? new ServerDebugWireEvents() : null;
+
+
+/** Utility to escape XML, for example to avoid XSS attacks. */
+export function escapeXml(unsafe: string): string {
+  return unsafe.replace(/[&<>"']/g, (match) => {
+    switch (match) {
+      case '&':
+        return '&amp;';
+      case '<':
+        return '&lt;';
+      case '>':
+        return '&gt;';
+      case '"':
+        return '&quot;';
+      case '\'':
+        return '&#39;';
+      default:
+        return match;
+    }
+  });
+}