From 30ffd1a7ee080df06b65cb6b562ac3b9520b6606 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Sun, 4 Aug 2024 05:20:50 -0700 Subject: [PATCH] InReferenceTo: multi-sentence, multi-role --- src/apps/chat/components/ChatMessageList.tsx | 11 ++-- .../chat/components/composer/Composer.tsx | 51 ++++++++++--------- .../composer/ComposerTextAreaActions.tsx | 25 +++++---- .../chat/components/message/ChatMessage.tsx | 35 ++++++------- ...lyToBubble.tsx => InReferenceToBubble.tsx} | 40 ++++++++++----- .../chat-overlay/ConversationHandler.ts | 2 +- src/common/chat-overlay/store-chat-overlay.ts | 9 ++-- .../store-composeroverlay-slice.ts | 31 ++++++++--- src/common/stores/chat/chat.message.ts | 11 +++- src/common/stores/chat/chats.converters.ts | 20 ++++++-- .../client/aix.client.fromDMessages.api.ts | 14 ++--- src/modules/aix/server/api/aix.wiretypes.ts | 14 +++-- .../adapters/anthropic.messageCreate.ts | 19 +++++-- .../adapters/gemini.generateContent.ts | 12 ++--- .../adapters/openai.chatCompletions.ts | 33 ++++++++++-- src/server/wire.ts | 21 ++++++++ 16 files changed, 238 insertions(+), 110 deletions(-) rename src/apps/chat/components/message/{ReplyToBubble.tsx => InReferenceToBubble.tsx} (66%) diff --git a/src/apps/chat/components/ChatMessageList.tsx b/src/apps/chat/components/ChatMessageList.tsx index aca3fc5da..570e337c5 100644 --- a/src/apps/chat/components/ChatMessageList.tsx +++ b/src/apps/chat/components/ChatMessageList.tsx @@ -11,7 +11,7 @@ import type { DConversationId } from '~/common/stores/chat/chat.conversation'; import type { DMessageFragment, DMessageFragmentId } from '~/common/stores/chat/chat.fragments'; import { InlineError } from '~/common/components/InlineError'; import { ShortcutKey, useGlobalShortcuts } from '~/common/components/shortcuts/useGlobalShortcuts'; -import { createDMessageTextContent, DMessageId, DMessageUserFlag, messageToggleUserFlag } from '~/common/stores/chat/chat.message'; +import { createDMessageTextContent, DMessageId, DMessageUserFlag, DMetaReferenceItem, messageToggleUserFlag } from '~/common/stores/chat/chat.message'; import { getConversation, useChatStore } from '~/common/stores/chat/store-chats'; import { optimaOpenPreferences } from '~/common/layout/optima/useOptima'; import { useBrowserTranslationWarning } from '~/common/components/useIsBrowserTranslating'; @@ -24,6 +24,7 @@ import { CleanerMessage, MessagesSelectionHeader } from './message/CleanerMessag import { Ephemerals } from './Ephemerals'; import { PersonaSelector } from './persona-selector/PersonaSelector'; import { useChatAutoSuggestHTMLUI, useChatShowSystemMessages } from '../store-app-chat'; +import { useChatComposerOverlayStore } from '~/common/chat-overlay/store-chat-overlay'; /** @@ -68,7 +69,7 @@ export function ChatMessageList(props: { // derived state const { conversationHandler, conversationId, capabilityHasT2I, onConversationBranch, onConversationExecuteHistory, onTextDiagram, onTextImagine, onTextSpeak } = props; - + const composeCanAddReplyTo = useChatComposerOverlayStore(conversationHandler?.getOverlayStore() ?? null, state => state.inReferenceTo?.length < 5); // text actions @@ -150,8 +151,8 @@ export function ChatMessageList(props: { }), false, false); }, [props.conversationHandler]); - const handleReplyTo = React.useCallback((_messageId: DMessageId, text: string) => { - props.conversationHandler?.getOverlayStore().getState().setReplyToText(text); + const handleAddInReferenceTo = React.useCallback((item: DMetaReferenceItem) => { + props.conversationHandler?.getOverlayStore().getState().addInReferenceTo(item); }, [props.conversationHandler]); const handleTextDiagram = React.useCallback(async (messageId: DMessageId, text: string) => { @@ -294,6 +295,7 @@ export function ChatMessageList(props: { isImagining={isImagining} isSpeaking={isSpeaking} showUnsafeHtml={danger_experimentalHtmlWebUi} + onAddInReferenceTo={!composeCanAddReplyTo ? undefined : handleAddInReferenceTo} onMessageAssistantFrom={handleMessageAssistantFrom} onMessageBeam={handleMessageBeam} onMessageBranch={handleMessageBranch} @@ -304,7 +306,6 @@ export function ChatMessageList(props: { onMessageFragmentReplace={handleMessageReplaceFragment} onMessageToggleUserFlag={handleMessageToggleUserFlag} onMessageTruncate={handleMessageTruncate} - onReplyTo={handleReplyTo} onTextDiagram={handleTextDiagram} onTextImagine={capabilityHasT2I ? handleTextImagine : undefined} onTextSpeak={isSpeakable ? handleTextSpeak : undefined} diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index e89a3b1cb..e7c3f3fd4 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -26,7 +26,7 @@ import { AudioPlayer } from '~/common/util/audio/AudioPlayer'; import { ButtonAttachFilesMemo } from '~/common/components/ButtonAttachFiles'; import { ChatBeamIcon } from '~/common/components/icons/ChatBeamIcon'; import { ConversationsManager } from '~/common/chat-overlay/ConversationsManager'; -import { DMessageMetadata, messageFragmentsReduceText } from '~/common/stores/chat/chat.message'; +import { DMessageMetadata, DMetaReferenceItem, messageFragmentsReduceText } from '~/common/stores/chat/chat.message'; import { ShortcutKey, ShortcutObject, useGlobalShortcuts } from '~/common/components/shortcuts/useGlobalShortcuts'; import { animationEnterBelow } from '~/common/util/animUtils'; import { browserSpeechRecognitionCapability, SpeechResult, useSpeechRecognition } from '~/common/components/useSpeechRecognition'; @@ -42,7 +42,7 @@ import { optimaOpenPreferences } from '~/common/layout/optima/useOptima'; import { platformAwareKeystrokes } from '~/common/components/KeyStroke'; import { supportsScreenCapture } from '~/common/util/screenCaptureUtils'; import { useAppStateStore } from '~/common/state/store-appstate'; -import { useChatOverlayStore } from '~/common/chat-overlay/store-chat-overlay'; +import { useChatComposerOverlayStore } from '~/common/chat-overlay/store-chat-overlay'; import { useDebouncer } from '~/common/components/useDebouncer'; import { useUICounter, useUIPreferencesStore } from '~/common/state/store-ui'; import { useUXLabsStore } from '~/common/state/store-ux-labs'; @@ -141,10 +141,9 @@ export function Composer(props: { ? ConversationsManager.getHandler(props.targetConversationId)?.getOverlayStore() || null : null; - // composer-overlay: for the reply-to state, comes from the conversation overlay - const { replyToGenerateText } = useChatOverlayStore(conversationOverlayStore, useShallow(store => ({ - replyToGenerateText: (chatExecuteMode === 'generate-content' || chatExecuteMode === 'generate-text-v1') ? store.replyToText?.trim() ?? undefined : undefined, - }))); + // composer-overlay: for the in-reference-to state, comes from the conversation overlay + const allowInReferenceTo = chatExecuteMode === 'generate-content' || chatExecuteMode === 'generate-text-v1'; + const inReferenceTo = useChatComposerOverlayStore(conversationOverlayStore, store => allowInReferenceTo ? store.inReferenceTo : null); // don't load URLs if the user is typing a command or there's no capability const enableLoadURLsInComposer = useBrowseCapability().inComposer && !composeText.startsWith('/'); @@ -206,14 +205,18 @@ export function Composer(props: { // Overlay actions - const handleReplyToClear = React.useCallback(() => { - conversationOverlayStore?.getState().setReplyToText(null); + const handleRemoveInReferenceTo = React.useCallback((item: DMetaReferenceItem) => { + conversationOverlayStore?.getState().removeInReferenceTo(item); + }, [conversationOverlayStore]); + + const handleInReferenceToClear = React.useCallback(() => { + conversationOverlayStore?.getState().clearInReferenceTo(); }, [conversationOverlayStore]); React.useEffect(() => { - if (replyToGenerateText) + if (inReferenceTo?.length) setTimeout(() => composerTextAreaRef.current?.focus(), 1 /* prevent focus theft */); - }, [composerTextAreaRef, replyToGenerateText]); + }, [composerTextAreaRef, inReferenceTo]); // Primary button @@ -221,8 +224,8 @@ export function Composer(props: { const handleClear = React.useCallback(() => { setComposeText(''); attachmentsRemoveAll(); - handleReplyToClear(); - }, [attachmentsRemoveAll, handleReplyToClear, setComposeText]); + handleInReferenceToClear(); + }, [attachmentsRemoveAll, handleInReferenceToClear, setComposeText]); const handleSendAction = React.useCallback(async (_chatExecuteMode: ChatExecuteMode, composerText: string): Promise => { @@ -250,13 +253,15 @@ export function Composer(props: { return false; } + // prepare the metadata + const metadata = inReferenceTo?.length ? { inReferenceTo: inReferenceTo } : undefined; + // send the message - NOTE: if successful, the ownership of the fragments is transferred to the receiver, so we just clear them - const metadata = replyToGenerateText ? { inReplyToText: replyToGenerateText } : undefined; const enqueued = onAction(targetConversationId, _chatExecuteMode, fragments, metadata); if (enqueued) handleClear(); return enqueued; - }, [attachmentsTakeAllFragments, handleClear, onAction, replyToGenerateText, targetConversationId]); + }, [attachmentsTakeAllFragments, handleClear, inReferenceTo, onAction, targetConversationId]); const handleAppendTextAndSend = React.useCallback(async (appendText: string) => { @@ -518,8 +523,8 @@ export function Composer(props: { const isReAct = chatExecuteMode === 'react-content'; const isDraw = chatExecuteMode === 'generate-image'; - const showChatReplyTo = !!replyToGenerateText; - const showChatExtras = isText && !showChatReplyTo; + const showChatInReferenceTo = !!inReferenceTo?.length; + const showChatExtras = isText && !showChatInReferenceTo; const sendButtonVariant: VariantProp = (isAppend || (isMobile && isTextBeam)) ? 'outlined' : 'solid'; @@ -539,7 +544,7 @@ export function Composer(props: { isDraw ? 'Describe an idea or a drawing...' : isReAct ? 'Multi-step reasoning question...' : isTextBeam ? 'Beam: combine the smarts of models...' - : showChatReplyTo ? 'Chat about this' + : showChatInReferenceTo ? 'Chat about this' : props.isDeveloperMode ? 'Chat with me' + (isDesktop ? ' · drop source' : '') + ' · attach code...' : props.capabilityHasT2I ? 'Chat · /beam · /draw · drop files...' : 'Chat · /react · drop files...'; @@ -659,7 +664,7 @@ export function Composer(props: { variant='outlined' color={isDraw ? 'warning' : isReAct ? 'success' : undefined} autoFocus - minRows={isMobile ? 4 : showChatReplyTo ? 4 : 5} + minRows={isMobile ? 4 : showChatInReferenceTo ? 4 : 5} maxRows={isMobile ? 8 : 10} placeholder={textPlaceholder} value={composeText} @@ -672,9 +677,9 @@ export function Composer(props: { } slotProps={{ @@ -689,15 +694,15 @@ export function Composer(props: { }} sx={{ backgroundColor: 'background.level1', - '&:focus-within': { backgroundColor: 'background.popup', '.reply-to-bubble': { backgroundColor: 'background.popup' } }, + '&:focus-within': { backgroundColor: 'background.popup', '.in-reference-to-bubble': { backgroundColor: 'background.popup' } }, lineHeight: lineHeightTextareaMd, }} /> - {!showChatReplyTo && tokenLimit > 0 && (tokensComposer > 0 || (tokensHistory + tokensReponseMax) > 0) && ( + {!showChatInReferenceTo && tokenLimit > 0 && (tokensComposer > 0 || (tokensHistory + tokensReponseMax) > 0) && ( )} - {!showChatReplyTo && tokenLimit > 0 && ( + {!showChatInReferenceTo && tokenLimit > 0 && ( )} diff --git a/src/apps/chat/components/composer/ComposerTextAreaActions.tsx b/src/apps/chat/components/composer/ComposerTextAreaActions.tsx index b095124f8..a07aa410b 100644 --- a/src/apps/chat/components/composer/ComposerTextAreaActions.tsx +++ b/src/apps/chat/components/composer/ComposerTextAreaActions.tsx @@ -2,19 +2,21 @@ import * as React from 'react'; import { Box, Sheet } from '@mui/joy'; -import { ReplyToBubble } from '../message/ReplyToBubble'; +import type { DMetaReferenceItem } from '~/common/stores/chat/chat.message'; + +import { InReferenceToBubble } from '../message/InReferenceToBubble'; export function ComposerTextAreaActions(props: { agiAttachmentButton?: React.ReactNode, agiAttachmentPrompts?: string[], - replyToText?: string, + inReferenceTo?: DMetaReferenceItem[] | null onAppendAndSend: (appendText: string) => Promise, - onReplyToClear: () => void, + onRemoveReferenceTo: (item: DMetaReferenceItem) => void, }) { // skip the component if there's nothing to show - if (!props.agiAttachmentPrompts?.length && !props.agiAttachmentButton && props.replyToText === undefined) + if (!props.agiAttachmentPrompts?.length && !props.agiAttachmentButton && !props.inReferenceTo?.length) return null; return ( @@ -38,14 +40,15 @@ export function ComposerTextAreaActions(props: { }, }}> - {/* Reply-To bubble */} - {props.replyToText !== undefined && ( - ( + - )} + ))} {/* Auto-Prompts from attachments */} {!!props.agiAttachmentPrompts?.length && ( diff --git a/src/apps/chat/components/message/ChatMessage.tsx b/src/apps/chat/components/message/ChatMessage.tsx index a208f8158..5beb9b631 100644 --- a/src/apps/chat/components/message/ChatMessage.tsx +++ b/src/apps/chat/components/message/ChatMessage.tsx @@ -25,7 +25,7 @@ import VerticalAlignBottomIcon from '@mui/icons-material/VerticalAlignBottom'; import { ChatBeamIcon } from '~/common/components/icons/ChatBeamIcon'; import { CloseableMenu } from '~/common/components/CloseableMenu'; -import { DMessage, DMessageId, DMessageUserFlag, messageFragmentsReduceText, messageHasUserFlag } from '~/common/stores/chat/chat.message'; +import { DMessage, DMessageId, DMessageUserFlag, DMetaReferenceItem, messageFragmentsReduceText, messageHasUserFlag } from '~/common/stores/chat/chat.message'; import { KeyStroke } from '~/common/components/KeyStroke'; import { adjustContentScaling, themeScalingMap, themeZIndexPageBar } from '~/common/app.theme'; import { animationColorRainbow } from '~/common/util/animUtils'; @@ -38,7 +38,7 @@ import { ContentFragmentsWithInlineEdit } from './fragments-content/ContentFragm import { ContinueFragment } from './ContinueFragment'; import { DocumentAttachmentFragments } from './fragments-attachment-doc/DocumentAttachmentFragments'; import { ImageAttachmentFragments } from './fragments-attachment-image/ImageAttachmentFragments'; -import { ReplyToBubble } from './ReplyToBubble'; +import { InReferenceToBubble } from './InReferenceToBubble'; import { avatarIconSx, makeMessageAvatarIcon, messageAsideColumnSx, messageBackground, messageZenAsideColumnSx } from './messageUtils'; import { useChatShowTextDiff } from '../../store-app-chat'; @@ -77,6 +77,7 @@ export function ChatMessage(props: { showUnsafeHtml?: boolean, adjustContentScaling?: number, topDecorator?: React.ReactNode, + onAddInReferenceTo?: (item: DMetaReferenceItem) => void, onMessageAssistantFrom?: (messageId: string, offset: number) => Promise, onMessageBeam?: (messageId: string) => Promise, onMessageBranch?: (messageId: string) => void, @@ -87,7 +88,6 @@ export function ChatMessage(props: { onMessageFragmentReplace?: (messageId: DMessageId, fragmentId: DMessageFragmentId, newFragment: DMessageFragment) => void, onMessageToggleUserFlag?: (messageId: string, flag: DMessageUserFlag) => void, onMessageTruncate?: (messageId: string) => void, - onReplyTo?: (messageId: string, selectedText: string) => void, onTextDiagram?: (messageId: string, text: string) => Promise, onTextImagine?: (text: string) => Promise, onTextSpeak?: (text: string) => Promise, @@ -203,7 +203,7 @@ export function ChatMessage(props: { // Message Operations Menu - const { onMessageToggleUserFlag } = props; + const { onAddInReferenceTo, onMessageToggleUserFlag } = props; const handleOpsMenuToggle = React.useCallback((event: React.MouseEvent) => { event.preventDefault(); // added for the Right mouse click (to prevent the menu) @@ -273,10 +273,10 @@ export function ChatMessage(props: { } }; - const handleOpsReplyTo = (e: React.MouseEvent) => { + const handleOpsAddInReferenceTo = (e: React.MouseEvent) => { e.preventDefault(); - if (props.onReplyTo && textSel.trim().length >= BUBBLE_MIN_TEXT_LENGTH) { - props.onReplyTo(messageId, textSel.trim()); + if (onAddInReferenceTo && textSel.trim().length >= BUBBLE_MIN_TEXT_LENGTH) { + onAddInReferenceTo({ mrt: 'dmsg', mText: textSel.trim(), mRole: messageRole /*, messageId*/ }); handleCloseOpsMenu(); closeContextMenu(); closeBubble(); @@ -573,14 +573,15 @@ export function ChatMessage(props: { )} - {/* Reply-To Bubble */} - {!!messageMetadata?.inReplyToText && ( - ( + - )} + ))} {/* Image Attachment Fragments - just for a prettier display on top of the message */} {imageAttachments.length >= 1 && !isEditingText && ( @@ -829,8 +830,8 @@ export function ChatMessage(props: { }, }} > - {!!props.onReplyTo && - + {!!onAddInReferenceTo && + } @@ -839,7 +840,7 @@ export function ChatMessage(props: { {/* */} {/* */} {/*}*/} - {!!props.onReplyTo && fromAssistant && } + {!!onAddInReferenceTo && fromAssistant && } diff --git a/src/apps/chat/components/message/ReplyToBubble.tsx b/src/apps/chat/components/message/InReferenceToBubble.tsx similarity index 66% rename from src/apps/chat/components/message/ReplyToBubble.tsx rename to src/apps/chat/components/message/InReferenceToBubble.tsx index c37cf3cc1..f7a27391a 100644 --- a/src/apps/chat/components/message/ReplyToBubble.tsx +++ b/src/apps/chat/components/message/InReferenceToBubble.tsx @@ -5,6 +5,8 @@ import { Box, IconButton, Tooltip, Typography } from '@mui/joy'; import CloseRoundedIcon from '@mui/icons-material/CloseRounded'; import ReplyRoundedIcon from '@mui/icons-material/ReplyRounded'; +import type { DMetaReferenceItem } from '~/common/stores/chat/chat.message'; + // configuration const INLINE_COLOR = 'primary'; @@ -52,38 +54,52 @@ export const inlineMessageBubbleSx: SxProps = { }; -export function ReplyToBubble(props: { - replyToText?: string, - inlineUserMessage?: boolean - onClear?: () => void, +export function InReferenceToBubble(props: { + item: DMetaReferenceItem, + onRemove?: (item: DMetaReferenceItem) => void, className?: string, + bubbleVariant?: 'message', }) { + + // derived state + + const variantMessage = props.bubbleVariant === 'message'; + + // handlers + + const { onRemove } = props; + + const handleRemoveClicked = React.useCallback(() => { + onRemove?.(props.item); + }, [onRemove, props.item]); + return ( - + - {props.replyToText} + {props.item.mText} - {!!props.onClear && ( - + {!!props.onRemove && ( + )} diff --git a/src/common/chat-overlay/ConversationHandler.ts b/src/common/chat-overlay/ConversationHandler.ts index 535965555..f9d7d57f8 100644 --- a/src/common/chat-overlay/ConversationHandler.ts +++ b/src/common/chat-overlay/ConversationHandler.ts @@ -40,7 +40,7 @@ export class ConversationHandler { inlineUpdatePurposeInHistory(history: DMessage[], assistantLlmId: DLLMId | undefined): DMessage[] { const purposeId = getConversationSystemPurposeId(this.conversationId); - // TODO: HACK: find the persona identiy separately from the "first system message", as e.g. right now would take the reply-to and promote as system + // TODO: HACK: find the persona identiy separately from the "first system message" const systemMessageIndex = history.findIndex(m => m.role === 'system'); let systemMessage: DMessage = systemMessageIndex >= 0 diff --git a/src/common/chat-overlay/store-chat-overlay.ts b/src/common/chat-overlay/store-chat-overlay.ts index 9eec3da7e..3f8a509e4 100644 --- a/src/common/chat-overlay/store-chat-overlay.ts +++ b/src/common/chat-overlay/store-chat-overlay.ts @@ -1,8 +1,8 @@ -import { StoreApi, useStore } from 'zustand'; +import { useStore } from 'zustand'; import { createStore as createVanillaStore } from 'zustand/vanilla'; import { AttachmentDraftsStoreApi, AttachmentsDraftsStore, createAttachmentDraftsStoreSlice } from '~/common/attachment-drafts/store-attachment-drafts-slice'; -import { ComposerOverlayStore, createComposerOverlayStoreSlice } from './store-composeroverlay-slice'; +import { ComposerOverlayStore, ComposerOverlayStoreApi, createComposerOverlayStoreSlice } from './store-composeroverlay-slice'; // Note: at this time there are numerous overlay stores, including beam (vanilla), ephemerals (EventTarget), and this one. @@ -19,7 +19,10 @@ export const createPerChatVanillaStore = () => createVanillaStore(vanillaStore: Readonly> | null, selector: (store: PerChatOverlayStore) => T): T => +// export const useChatOverlayStore = (vanillaStore: Readonly> | null, selector: (store: PerChatOverlayStore) => T): T => +// useStore(vanillaStore || fallbackStoreApi, selector); + +export const useChatComposerOverlayStore = (vanillaStore: Readonly | null, selector: (store: ComposerOverlayStore) => T): T => useStore(vanillaStore || fallbackStoreApi, selector); export const useChatAttachmentsStore = (vanillaStore: Readonly | null, selector: (store: AttachmentsDraftsStore) => T): T => diff --git a/src/common/chat-overlay/store-composeroverlay-slice.ts b/src/common/chat-overlay/store-composeroverlay-slice.ts index 1b1c1a8d6..4184a0602 100644 --- a/src/common/chat-overlay/store-composeroverlay-slice.ts +++ b/src/common/chat-overlay/store-composeroverlay-slice.ts @@ -1,34 +1,49 @@ import type { StateCreator } from 'zustand/vanilla'; +import type { DMetaReferenceItem } from '~/common/stores/chat/chat.message'; +import type { StoreApi } from 'zustand'; + /// Chat Overlay Store: per-chat overlay state /// interface ComposerOverlayState { - // if set, this is the 'reply to' mode text - replyToText: string | null; + // list of all the references that the composer is holding to, before sending them out in the next message + inReferenceTo: DMetaReferenceItem[]; } export interface ComposerOverlayStore extends ComposerOverlayState { - setReplyToText: (text: string | null) => void; + addInReferenceTo: (item: DMetaReferenceItem) => void; + removeInReferenceTo: (item: DMetaReferenceItem) => void; + clearInReferenceTo: () => void; } +export type ComposerOverlayStoreApi = StoreApi; + /** * NOTE: the Composer state is managed primarily by the component, however there's some state that's: - * - associated with the chat (e.g. reply-to text) + * - associated with the chat (e.g. in-reference-to text) * - persisted across chats * - * This slice manages the reply-to text state, but there's also a sister slice that manages the attachment drafts. + * This slice manages the in-reference-to text state, but there's also a sister slice that manages the attachment drafts. */ export const createComposerOverlayStoreSlice: StateCreator = (_set, _get) => ({ // init state - replyToText: null, + inReferenceTo: [], // actions - setReplyToText: (text: string | null) => _set({ replyToText: text }), + addInReferenceTo: (item) => _set(state => ({ + inReferenceTo: [...state.inReferenceTo, item], + })), -}); \ No newline at end of file + removeInReferenceTo: (item) => _set(state => ({ + inReferenceTo: state.inReferenceTo.filter((i) => i !== item), + })), + + clearInReferenceTo: () => _set({ inReferenceTo: [] }), + +}); diff --git a/src/common/stores/chat/chat.message.ts b/src/common/stores/chat/chat.message.ts index 91efe4c7f..c0c966085 100644 --- a/src/common/stores/chat/chat.message.ts +++ b/src/common/stores/chat/chat.message.ts @@ -51,8 +51,15 @@ export type DMessageGenerator = { // Metadata export interface DMessageMetadata { - inReplyToText?: string; // text this was in reply to - ranOutOfTokens?: true; // if the message was cut off due to token limit + inReferenceTo?: DMetaReferenceItem[]; // text this was in reply to + ranOutOfTokens?: true; // if the message was cut off due to token limit +} + +export interface DMetaReferenceItem { + mrt: 'dmsg'; // for future type discrimination + mText: string; + mRole: DMessageRole; + // messageId?: string; } diff --git a/src/common/stores/chat/chats.converters.ts b/src/common/stores/chat/chats.converters.ts index 1f5ed908a..2e7566f05 100644 --- a/src/common/stores/chat/chats.converters.ts +++ b/src/common/stores/chat/chats.converters.ts @@ -54,6 +54,9 @@ export namespace V4ToHeadConverters { } export function dev_inMemHeadUpgradeDMessage(m: DMessage): void { + // TODO: remove insides for 2.0.0 + + // uplevel fragments for (const fragment of m.fragments) { // Result of a rename of DMessageDocPart.type -> .vdt @@ -64,7 +67,15 @@ export namespace V4ToHeadConverters { delete docPart.type; } } + } + // uplevel metadata: if (metadata?.inReplyToText) cm.metadata = { inReferenceTo: [{ mrt: 'dmsg', mText: metadata.inReplyToText, mRole: 'assistant' }] }; + if (m.metadata && 'inReplyToText' in m.metadata && m.metadata.inReplyToText) { + const replyToText = m.metadata.inReplyToText as string; + delete m.metadata.inReplyToText; + m.metadata.inReferenceTo = [{ + mrt: 'dmsg', mText: replyToText, mRole: 'assistant', + }]; } } @@ -231,7 +242,10 @@ export namespace V3StoreDataToHead { if (id) cm.id = id; if (purposeId) cm.purposeId = purposeId; if (originLLM) cm.originLLM = originLLM; - if (metadata) cm.metadata = metadata; + if (metadata?.inReplyToText) { + if (!cm.metadata) cm.metadata = {}; + cm.metadata.inReferenceTo = [{ mrt: 'dmsg', mText: metadata.inReplyToText, mRole: 'assistant' }]; + } if (userFlags) cm.userFlags = userFlags; cm.tokenCount = tokenCount || 0; if (created) cm.created = created; @@ -268,9 +282,9 @@ export namespace V3StoreDataToHead { originLLM?: string; // only assistant - model that generated this message, goes beyond known models metadata?: { // metadata, mainly at creation and for UI - inReplyToText?: string; // text this was in reply to + inReplyToText?: string; // V3: text this was in reply to }; - userFlags?: ('starred')[]; // (UI) user-set per-message flags + userFlags?: ('starred')[]; // (UI) user-set per-message flags tokenCount: number; // cache for token count, using the current Conversation model (0 = not yet calculated) diff --git a/src/modules/aix/client/aix.client.fromDMessages.api.ts b/src/modules/aix/client/aix.client.fromDMessages.api.ts index 52f2431d7..dedc954d8 100644 --- a/src/modules/aix/client/aix.client.fromDMessages.api.ts +++ b/src/modules/aix/client/aix.client.fromDMessages.api.ts @@ -1,11 +1,11 @@ import { getImageAsset } from '~/modules/dblobs/dblobs.images'; -import type { DMessage } from '~/common/stores/chat/chat.message'; +import type { DMessage, DMetaReferenceItem } from '~/common/stores/chat/chat.message'; import { DMessageImageRefPart, isContentFragment, isContentOrAttachmentFragment, isTextPart } from '~/common/stores/chat/chat.fragments'; import { LLMImageResizeMode, resizeBase64ImageIfNeeded } from '~/common/util/imageUtils'; // NOTE: pay particular attention to the "import type", as this is importing from the server-side Zod definitions -import type { AixAPIChatGenerate_Request, AixMessages_ModelMessage, AixMessages_UserMessage, AixParts_InlineImagePart, AixParts_MetaReplyToPart } from '../server/api/aix.wiretypes'; +import type { AixAPIChatGenerate_Request, AixMessages_ModelMessage, AixMessages_UserMessage, AixParts_InlineImagePart, AixParts_MetaInReferenceToPart } from '../server/api/aix.wiretypes'; // TODO: remove console messages to zero, or replace with throws or something @@ -78,12 +78,12 @@ export async function aixChatGenerateRequestFromDMessages(messageSequence: Reado return uMsg; }, Promise.resolve({ role: 'user', parts: [] } as AixMessages_UserMessage)); - // handle reply-to metadata, adding a part right after the user text (or at the beginning) - if (m.metadata?.inReplyToText) { + // handle in-reference-to metadata, adding a part right after the user text (or at the beginning) + if (m.metadata?.inReferenceTo?.length) { // find the index of the tast text part const lastTextPartIndex = aixChatMessageUser.parts.findLastIndex(p => p.pt === 'text'); // insert the meta part after the last text part (and before the first attachment) - aixChatMessageUser.parts.splice(lastTextPartIndex + 1, 0, _clientCreateAixMetaReplyToPart(m.metadata.inReplyToText)); + aixChatMessageUser.parts.splice(lastTextPartIndex + 1, 0, _clientCreateAixMetaInReferenceToPart(m.metadata.inReferenceTo)); } acc.chatSequence.push(aixChatMessageUser); @@ -179,6 +179,6 @@ function _clientCreateAixInlineImagePart(base64: string, mimeType: string): AixP return { pt: 'inline_image', mimeType: (mimeType || 'image/png') as AixParts_InlineImagePart['mimeType'], base64 }; } -function _clientCreateAixMetaReplyToPart(replyTo: string): AixParts_MetaReplyToPart { - return { pt: 'meta_reply_to', replyTo }; +function _clientCreateAixMetaInReferenceToPart(items: DMetaReferenceItem[]): AixParts_MetaInReferenceToPart { + return { pt: 'meta_in_reference_to', referTo: items }; } diff --git a/src/modules/aix/server/api/aix.wiretypes.ts b/src/modules/aix/server/api/aix.wiretypes.ts index 5a1a6e339..2868224c0 100644 --- a/src/modules/aix/server/api/aix.wiretypes.ts +++ b/src/modules/aix/server/api/aix.wiretypes.ts @@ -19,7 +19,7 @@ import { openAIAccessSchema } from '~/modules/llms/server/openai/openai.router'; // Export types export type AixParts_DocPart = z.infer; export type AixParts_InlineImagePart = z.infer; -export type AixParts_MetaReplyToPart = z.infer; +export type AixParts_MetaInReferenceToPart = z.infer; export type AixMessages_SystemMessage = z.infer; export type AixMessages_UserMessage = z.infer; @@ -196,9 +196,13 @@ export namespace AixWire_Parts { // Metas - export const MetaReplyToPart_schema = z.object({ - pt: z.literal('meta_reply_to'), - replyTo: z.string(), + export const MetaInReferenceToPart_schema = z.object({ + pt: z.literal('meta_in_reference_to'), + referTo: z.array(z.object({ + mrt: z.literal('dmsg'), + mText: z.string(), + mRole: z.string(), + })), }); } @@ -219,7 +223,7 @@ export namespace AixWire_Content { AixWire_Parts.TextPart_schema, AixWire_Parts.InlineImagePart_schema, AixWire_Parts.DocPart_schema, - AixWire_Parts.MetaReplyToPart_schema, + AixWire_Parts.MetaInReferenceToPart_schema, ])), }); diff --git a/src/modules/aix/server/dispatch/chatGenerate/adapters/anthropic.messageCreate.ts b/src/modules/aix/server/dispatch/chatGenerate/adapters/anthropic.messageCreate.ts index 7a7669557..e695776f4 100644 --- a/src/modules/aix/server/dispatch/chatGenerate/adapters/anthropic.messageCreate.ts +++ b/src/modules/aix/server/dispatch/chatGenerate/adapters/anthropic.messageCreate.ts @@ -1,4 +1,6 @@ -import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes'; +import { escapeXml } from '~/server/wire'; + +import { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes'; import { AnthropicWire_API_Message_Create, AnthropicWire_Blocks } from '../../wiretypes/anthropic.wiretypes'; @@ -98,8 +100,10 @@ function* _generateAnthropicMessagesContentBlocks({ parts, role }: AixMessages_C yield { role: 'user', content: AnthropicWire_Blocks.TextBlock('```' + (part.ref || '') + '\n' + part.data.text + '\n```\n') }; break; - case 'meta_reply_to': - yield { role: 'user', content: AnthropicWire_Blocks.TextBlock(`The user is referring to this in particular: ${part.replyTo}`) }; + case 'meta_in_reference_to': + const irtXMLString = inReferenceTo_To_XMLString(part); + if (irtXMLString) + yield { role: 'user', content: AnthropicWire_Blocks.TextBlock(irtXMLString) }; break; default: @@ -206,3 +210,12 @@ function _toAnthropicToolChoice(itp: AixTools_ToolsPolicy): NonNullable escapeXml(r.mText)); + if (!refs.length) + return null; // `User provides no specific references`; + return refs.length === 1 + ? `User refers to this in particular:${refs[0]}` + : `User refers to ${refs.length} items:${refs.join('')}`; +} diff --git a/src/modules/aix/server/dispatch/chatGenerate/adapters/gemini.generateContent.ts b/src/modules/aix/server/dispatch/chatGenerate/adapters/gemini.generateContent.ts index 5b0f7101f..867f95544 100644 --- a/src/modules/aix/server/dispatch/chatGenerate/adapters/gemini.generateContent.ts +++ b/src/modules/aix/server/dispatch/chatGenerate/adapters/gemini.generateContent.ts @@ -1,6 +1,8 @@ import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixParts_DocPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes'; import { GeminiWire_API_Generate_Content, GeminiWire_ContentParts, GeminiWire_Messages, GeminiWire_Safety } from '../../wiretypes/gemini.wiretypes'; +import { inReferenceTo_To_XMLString } from './anthropic.messageCreate'; + // configuration const hotFixImagePartsFirst = true; @@ -90,8 +92,10 @@ function _toGeminiContents(chatSequence: AixMessages_ChatMessage[]): GeminiWire_ parts.push(_toApproximateGeminiDocPart(part)); break; - case 'meta_reply_to': - parts.push(_toApproximateGeminiReplyTo(part.replyTo)); + case 'meta_in_reference_to': + const irtXMLString = inReferenceTo_To_XMLString(part); + if (irtXMLString) + parts.push(GeminiWire_ContentParts.TextPart(irtXMLString)); break; case 'tool_invocation': @@ -242,7 +246,3 @@ function _toGeminiSafetySettings(threshold: GeminiWire_Safety.HarmBlockThreshold function _toApproximateGeminiDocPart(aixPartsDocPart: AixParts_DocPart): GeminiWire_ContentParts.ContentPart { return GeminiWire_ContentParts.TextPart(`\`\`\`${aixPartsDocPart.ref || ''}\n${aixPartsDocPart.data.text}\n\`\`\`\n`); } - -function _toApproximateGeminiReplyTo(replyTo: string): GeminiWire_ContentParts.ContentPart { - return GeminiWire_ContentParts.TextPart(`The user is referring to this in particular: ${replyTo}`); -} diff --git a/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts b/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts index e8f3a2bc4..e3cef2cd1 100644 --- a/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts +++ b/src/modules/aix/server/dispatch/chatGenerate/adapters/openai.chatCompletions.ts @@ -1,6 +1,6 @@ import type { OpenAIDialects } from '~/modules/llms/server/openai/openai.router'; -import type { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes'; +import { AixAPI_Model, AixAPIChatGenerate_Request, AixMessages_ChatMessage, AixMessages_SystemMessage, AixParts_MetaInReferenceToPart, AixTools_ToolDefinition, AixTools_ToolsPolicy } from '../../../api/aix.wiretypes'; import { OpenAIWire_API_Chat_Completions, OpenAIWire_ContentParts, OpenAIWire_Messages } from '../../wiretypes/openai.wiretypes'; @@ -209,9 +209,8 @@ function _toOpenAIMessages(systemMessage: AixMessages_SystemMessage | undefined, chatMessages.push({ role: 'user', content: [imageContentPart] }); break; - case 'meta_reply_to': - const context = `The user is referring to this in particular:\n{{ReplyToText}}`.replace('{{ReplyToText}}', part.replyTo); - chatMessages.push({ role: 'system', content: context }); + case 'meta_in_reference_to': + chatMessages.push({ role: 'system', content: _toOpenAIInReferenceToText(part) }); break; default: @@ -353,3 +352,29 @@ function _toOpenAIToolChoice(openAIDialect: OpenAIDialects, itp: AixTools_ToolsP return { type: 'function' as const, function: { name: itp.function_call.name } }; } } + +function _toOpenAIInReferenceToText(irt: AixParts_MetaInReferenceToPart): string { + // Get the item texts without roles + const items = irt.referTo.map(r => r.mText); + if (items.length === 0) + return 'CONTEXT: The user provides no specific references.'; + + const isShortItem = (text: string): boolean => + text.split('\n').length <= 3 && text.length <= 200; + + const formatItem = (text: string, index?: number): string => { + if (isShortItem(text)) { + const formatted = text.replace(/\n/g, ' ').replace(/\s+/g, ' '); + return index !== undefined ? `${index + 1}. "${formatted}"` : `"${formatted}"`; + } + return `${index !== undefined ? `ITEM ${index + 1}:\n` : ''}---\n${text}\n---`; + }; + + // Formely: `The user is referring to this in particular:\n{{ReplyToText}}`.replace('{{ReplyToText}}', part.replyTo); + if (items.length === 1) + return `CONTEXT: The user is referring to this in particular:\n${formatItem(items[0])}`; + + const allShort = items.every(isShortItem); + return `CONTEXT: The user is referring to these ${items.length} in particular:\n\n${ + items.map((text, index) => formatItem(text, index)).join(allShort ? '\n' : '\n\n')}`; +} diff --git a/src/server/wire.ts b/src/server/wire.ts index 0fde64dee..662caece8 100644 --- a/src/server/wire.ts +++ b/src/server/wire.ts @@ -120,3 +120,24 @@ export class ServerDebugWireEvents { } export const createServerDebugWireEvents = () => SERVER_DEBUG_WIRE ? new ServerDebugWireEvents() : null; + + +/** Utility to escape XML, for example to avoid XSS attacks. */ +export function escapeXml(unsafe: string): string { + return unsafe.replace(/[&<>"']/g, (match) => { + switch (match) { + case '&': + return '&'; + case '<': + return '<'; + case '>': + return '>'; + case '"': + return '"'; + case '\'': + return '''; + default: + return match; + } + }); +} \ No newline at end of file