From bf5e80a462171dcb1fa44e4982269bbb3c3560f2 Mon Sep 17 00:00:00 2001 From: Enrico Ros Date: Sun, 9 Jun 2024 19:00:01 -0700 Subject: [PATCH] MP: adapt ego attachment, messageSingleTextOrThrow-- --- .../chat/components/composer/Composer.tsx | 25 +++++++++------- .../llmattachments/LLMAttachmentItem.tsx | 2 +- .../attachment-drafts/attachment.pipeline.ts | 30 ++++++++++++------- .../attachment-drafts/attachment.types.ts | 12 ++++---- .../attachment-drafts/useAttachmentDrafts.tsx | 10 +++---- src/common/stores/chat/chat.message.ts | 11 +++++-- 6 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index fe1ee43bf..6483d0358 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -29,7 +29,7 @@ import { SpeechResult, useSpeechRecognition } from '~/common/components/useSpeec import { animationEnterBelow } from '~/common/util/animUtils'; import { conversationTitle, DConversationId } from '~/common/stores/chat/chat.conversation'; import { copyToClipboard, supportsClipboardRead } from '~/common/util/clipboardUtils'; -import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, DMessageMetadata, messageSingleTextOrThrow } from '~/common/stores/chat/chat.message'; +import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, DMessageMetadata, duplicateDMessageFragments, isContentFragment, messageFragmentsReduceText } from '~/common/stores/chat/chat.message'; import { estimateTextTokens, glueForMessageTokens } from '~/common/stores/chat/chat.tokens'; import { getConversation, useChatStore } from '~/common/stores/chat/store-chats'; import { isMacUser } from '~/common/util/pwaUtils'; @@ -155,7 +155,7 @@ export function Composer(props: { // attachments-overlay: comes from the attachments slice of the conversation overlay const { attachmentDrafts, - attachAppendClipboardItems, attachAppendDataTransfer, attachAppendEgoMessage, attachAppendFile, + attachAppendClipboardItems, attachAppendDataTransfer, attachAppendEgoContent, attachAppendFile, attachmentsClear, attachmentsTakeAllFragments, attachmentsTakeTextFragments, } = useAttachmentDrafts(conversationOverlayStore, enableLoadURLsInComposer); @@ -321,19 +321,22 @@ export function Composer(props: { } }, [props.composerTextAreaRef, setComposeText]); - const onActileMessageAttach = React.useCallback((item: StarredMessageItem) => { + const onActileMessageAttach = React.useCallback(async (item: StarredMessageItem) => { // get the message const conversation = getConversation(item.conversationId); const messageToAttach = conversation?.messages.find(m => m.id === item.messageId); - const messageText = messageToAttach ? messageSingleTextOrThrow(messageToAttach) : null; - if (conversation && messageToAttach && messageText) { - // Testing with this serialization for LLM. Note it will still be within a multi-part message, - // this could be in a titled markdown block. Don't know yet how this fares with different LLMs. - const chatTitle = conversationTitle(conversation); - const textPlain = `---\nitem id: ${messageToAttach.id}\ncontext title: ${chatTitle}\n---\n${messageText.trim()}\n`; - void attachAppendEgoMessage('context-item', textPlain, `${chatTitle} > ${messageText.slice(0, 10)}...`); + if (conversation && messageToAttach) { + const contentToAttach = duplicateDMessageFragments(messageToAttach.fragments) + .filter(isContentFragment); + if (contentToAttach.length) { + const chatTitle = conversationTitle(conversation); + const messageText = messageFragmentsReduceText(contentToAttach); + const refLabel = `${chatTitle} > ${messageText.slice(0, 10)}...`; + const refId = `${item.messageId} - ${chatTitle}`; + await attachAppendEgoContent(refLabel, refId, contentToAttach); + } } - }, [attachAppendEgoMessage]); + }, [attachAppendEgoContent]); const actileProviders = React.useMemo(() => { return [providerCommands(onActileCommandPaste), providerStarredMessage(onActileMessageAttach)]; diff --git a/src/apps/chat/components/composer/llmattachments/LLMAttachmentItem.tsx b/src/apps/chat/components/composer/llmattachments/LLMAttachmentItem.tsx index 56fc2f377..9c53c0352 100644 --- a/src/apps/chat/components/composer/llmattachments/LLMAttachmentItem.tsx +++ b/src/apps/chat/components/composer/llmattachments/LLMAttachmentItem.tsx @@ -80,7 +80,7 @@ const converterTypeToIconMap: { [key in AttachmentDraftConverterType]: React.Com 'image-resized-low': PhotoSizeSelectSmallOutlinedIcon, 'image-to-default': ImageOutlinedIcon, 'image-ocr': AbcIcon, - 'ego-message-md': TelegramIcon, + 'ego-contents-inlined': TelegramIcon, 'unhandled': TextureIcon, }; diff --git a/src/common/attachment-drafts/attachment.pipeline.ts b/src/common/attachment-drafts/attachment.pipeline.ts index 273f698cc..d520cded0 100644 --- a/src/common/attachment-drafts/attachment.pipeline.ts +++ b/src/common/attachment-drafts/attachment.pipeline.ts @@ -4,7 +4,7 @@ import { createBase64UuidV4 } from '~/common/util/textUtils'; import { htmlTableToMarkdown } from '~/common/util/htmlTableToMarkdown'; import { pdfToImageDataURLs, pdfToText } from '~/common/util/pdfUtils'; -import { createTextAttachmentFragment, DMessageAttachmentFragment } from '~/common/stores/chat/chat.message'; +import { createAttachmentFragment, createTextAttachmentFragment, DMessageAttachmentFragment, DMessageContentFragment } from '~/common/stores/chat/chat.message'; import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftInput, AttachmentDraftSource } from './attachment.types'; import type { AttachmentsDraftsStore } from './store-attachment-drafts-slice'; @@ -198,11 +198,11 @@ export async function attachmentLoadInputAsync(source: Readonly { + const attachAppendEgoContent = React.useCallback((label: string, refId: string, contents: DMessageContentFragment[]) => { if (ATTACHMENTS_DEBUG_INTAKE) - console.log('attachAppendEgo', { blockTitle, textPlain, attachmentLabel }); + console.log('attachAppendEgoContent', label, refId, contents); return _createAttachmentDraft({ - media: 'ego', method: 'ego-message', label: attachmentLabel, blockTitle: blockTitle, textPlain: textPlain, + media: 'ego', method: 'ego-contents', label, refId, contents, }); }, [_createAttachmentDraft]); @@ -205,7 +205,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp // create drafts attachAppendClipboardItems, attachAppendDataTransfer, - attachAppendEgoMessage, + attachAppendEgoContent, attachAppendFile, // manage attachments diff --git a/src/common/stores/chat/chat.message.ts b/src/common/stores/chat/chat.message.ts index 9305e4327..f4ee9befc 100644 --- a/src/common/stores/chat/chat.message.ts +++ b/src/common/stores/chat/chat.message.ts @@ -216,7 +216,7 @@ export function duplicateDMessage(message: Readonly): DMessage { id: createBase64UuidV4(), role: message.role, - fragments: message.fragments.map(_duplicateFragment), + fragments: duplicateDMessageFragments(message.fragments), ...(message.pendingIncomplete ? { pendingIncomplete: true } : {}), ...(message.pendingPlaceholderText ? { pendingPlaceholderText: message.pendingPlaceholderText } : {}), @@ -236,6 +236,10 @@ export function duplicateDMessage(message: Readonly): DMessage { }; } +export function duplicateDMessageFragments(fragments: Readonly): DMessageFragment[] { + return fragments.map(_duplicateFragment); +} + function _duplicateFragment(fragment: DMessageFragment): DMessageFragment { switch (fragment.ft) { case 'content': @@ -284,7 +288,10 @@ function _duplicateReference(ref: DMessageDataRef): DMessageDataRef { // helpers during the transition from V3 -// specialize output type with 'is' +export function isContentFragment(fragment: DMessageFragment): fragment is DMessageContentFragment { + return fragment.ft === 'content'; +} + export function isContentOrAttachmentFragment(fragment: DMessageFragment): fragment is DMessageContentFragment | DMessageAttachmentFragment { return fragment.ft === 'content' || fragment.ft === 'attachment'; }