diff --git a/package-lock.json b/package-lock.json index c5bb4d938..0209d2892 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "big-agi", - "version": "1.16.0", + "version": "1.91.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "big-agi", - "version": "1.16.0", + "version": "1.91.0", "hasInstallScript": true, "dependencies": { "@emotion/cache": "^11.13.1", diff --git a/src/apps/chat/AppChat.tsx b/src/apps/chat/AppChat.tsx index 6836dac5f..d973597c8 100644 --- a/src/apps/chat/AppChat.tsx +++ b/src/apps/chat/AppChat.tsx @@ -18,7 +18,7 @@ import type { DConversation, DConversationId } from '~/common/stores/chat/chat.c import type { OptimaBarControlMethods } from '~/common/layout/optima/bar/OptimaBarDropdown'; import { ConfirmationModal } from '~/common/components/modals/ConfirmationModal'; import { ConversationsManager } from '~/common/chat-overlay/ConversationsManager'; -import { createErrorContentFragment, createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragmentsNoPH } from '~/common/stores/chat/chat.fragments'; +import { createErrorContentFragment, createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragmentsNoVoid } from '~/common/stores/chat/chat.fragments'; import { LLM_IF_ANT_PromptCaching, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types'; import { OptimaDrawerIn, OptimaToolbarIn } from '~/common/layout/optima/portals/OptimaPortalsIn'; import { PanelResizeInset } from '~/common/components/panes/GoodPanelResizeHandler'; @@ -249,7 +249,7 @@ export function AppChat() { // create the user:message // NOTE: this can lead to multiple chat messages with data refs that are referring to the same dblobs, // however, we already got transferred ownership of the dblobs at this point. - const userMessage = createDMessageFromFragments('user', duplicateDMessageFragmentsNoPH(fragments)); // [chat] create user:message to send per-chat + const userMessage = createDMessageFromFragments('user', duplicateDMessageFragmentsNoVoid(fragments)); // [chat] create user:message to send per-chat if (metadata) userMessage.metadata = duplicateDMessageMetadata(metadata); ConversationsManager.getHandler(conversation.id).messageAppend(userMessage); // [chat] append user message in each conversation diff --git a/src/apps/chat/commands/commands.dmessage.ts b/src/apps/chat/commands/commands.dmessage.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index 15cd332a6..1dbfcd5a7 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -33,7 +33,7 @@ import { animationEnterBelow } from '~/common/util/animUtils'; import { browserSpeechRecognitionCapability, PLACEHOLDER_INTERIM_TRANSCRIPT, SpeechResult, useSpeechRecognition } from '~/common/components/useSpeechRecognition'; import { conversationTitle, DConversationId } from '~/common/stores/chat/chat.conversation'; import { copyToClipboard, supportsClipboardRead } from '~/common/util/clipboardUtils'; -import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragmentsNoPH } from '~/common/stores/chat/chat.fragments'; +import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragmentsNoVoid } from '~/common/stores/chat/chat.fragments'; import { estimateTextTokens, glueForMessageTokens, marshallWrapDocFragments } from '~/common/stores/chat/chat.tokens'; import { getConversation, isValidConversation, useChatStore } from '~/common/stores/chat/store-chats'; import { launchAppCall } from '~/common/app.routes'; @@ -476,7 +476,7 @@ export function Composer(props: { const conversation = getConversation(conversationId); const messageToEmbed = conversation?.messages.find(m => m.id === messageId); if (conversation && messageToEmbed) { - const fragmentsCopy = duplicateDMessageFragmentsNoPH(messageToEmbed.fragments); // [attach] deep copy a message's fragments to attach to ego + const fragmentsCopy = duplicateDMessageFragmentsNoVoid(messageToEmbed.fragments); // [attach] deep copy a message's fragments to attach to ego if (fragmentsCopy.length) { const chatTitle = conversationTitle(conversation); const messageText = messageFragmentsReduceText(fragmentsCopy); diff --git a/src/apps/chat/components/message/ChatMessage.tsx b/src/apps/chat/components/message/ChatMessage.tsx index e6bb1ef57..84862720b 100644 --- a/src/apps/chat/components/message/ChatMessage.tsx +++ b/src/apps/chat/components/message/ChatMessage.tsx @@ -207,13 +207,13 @@ export function ChatMessage(props: { const isVndAndCacheUser = !!props.showAntPromptCaching && messageHasUserFlag(props.message, MESSAGE_FLAG_VND_ANT_CACHE_USER); const { - imageAttachments, // Stamp-sized Images - contentFragments, // Text (Markdown + Code + ... blocks), Errors, (large) Images, Placeholders - nonImageAttachments, // Document Attachments, likely the User dropped them in + imageAttachments, // Stamp-sized Images + contentOrVoidFragments, // Text (Markdown + Code + ... blocks), Errors, (large) Images, Placeholders + nonImageAttachments, // Document Attachments, likely the User dropped them in } = useFragmentBuckets(messageFragments); const fragmentFlattenedText = React.useMemo(() => messageFragmentsReduceText(messageFragments), [messageFragments]); - const handleHighlightSelText = useSelHighlighterMemo(messageId, selText, contentFragments, fromAssistant, props.onMessageFragmentReplace); + const handleHighlightSelText = useSelHighlighterMemo(messageId, selText, contentOrVoidFragments, fromAssistant, props.onMessageFragmentReplace); const textSubject = selText ? selText : fragmentFlattenedText; const isSpecialT2I = textSubject.startsWith('https://images.prodia.xyz/') || textSubject.startsWith('/draw ') || textSubject.startsWith('/imagine ') || textSubject.startsWith('/img '); @@ -695,7 +695,7 @@ export function ChatMessage(props: { {/* Content Fragments */} { - // only proceed with DMessageContentFragment + // Render VOID fragments + if (isVoidFragment(fragment)) { + const { fId, part } = fragment; + switch (part.pt) { + case 'ph': { + return ( + + ); + } + + case '_pt_sentinel': + default: + ; + break; + } + } + + // Render CONTENT fragments if (!isContentFragment(fragment)) return null; @@ -153,18 +183,6 @@ export function ContentFragments(props: { /> ); - case 'ph': - return ( - - ); - // This is the most frequent part by far, and can be broken down into sub-blocks case 'text': return ( diff --git a/src/apps/chat/components/message/useFragmentBuckets.ts b/src/apps/chat/components/message/useFragmentBuckets.ts index a3d27f857..76e2b82b0 100644 --- a/src/apps/chat/components/message/useFragmentBuckets.ts +++ b/src/apps/chat/components/message/useFragmentBuckets.ts @@ -1,11 +1,11 @@ import * as React from 'react'; -import { DMessageAttachmentFragment, DMessageContentFragment, DMessageFragment, isAttachmentFragment, isContentFragment, isImageRefPart } from '~/common/stores/chat/chat.fragments'; +import { DMessageAttachmentFragment, DMessageContentFragment, DMessageFragment, DMessageVoidFragment, isAttachmentFragment, isContentFragment, isImageRefPart, isPlaceholderPart, isVoidFragment } from '~/common/stores/chat/chat.fragments'; import { shallowEquals } from '~/common/util/hooks/useShallowObject'; interface FragmentBuckets { - contentFragments: DMessageContentFragment[]; + contentOrVoidFragments: (DMessageContentFragment | DMessageVoidFragment)[]; imageAttachments: DMessageAttachmentFragment[]; nonImageAttachments: DMessageAttachmentFragment[]; } @@ -16,32 +16,35 @@ interface FragmentBuckets { export function useFragmentBuckets(messageFragments: DMessageFragment[]): FragmentBuckets { // Refs to store the last stable value for each bucket - const contentFragmentsRef = React.useRef([]); + const contentOrVoidFragmentsRef = React.useRef<(DMessageContentFragment | DMessageVoidFragment)[]>([]); const imageAttachmentsRef = React.useRef([]); const nonImageAttachmentsRef = React.useRef([]); // Use useMemo to recalculate buckets only when messageFragments changes return React.useMemo(() => { - const contentFragments: DMessageContentFragment[] = []; + const contentOrVoidFragments: (DMessageContentFragment | DMessageVoidFragment)[] = []; const imageAttachments: DMessageAttachmentFragment[] = []; const nonImageAttachments: DMessageAttachmentFragment[] = []; messageFragments.forEach(fragment => { if (isContentFragment(fragment)) - contentFragments.push(fragment); + contentOrVoidFragments.push(fragment); else if (isAttachmentFragment(fragment)) { if (isImageRefPart(fragment.part)) imageAttachments.push(fragment); else nonImageAttachments.push(fragment); + } else if (isVoidFragment(fragment)) { + if (isPlaceholderPart(fragment.part)) + contentOrVoidFragments.push(fragment); } else console.warn('[DEV] Unexpected fragment type:', fragment.ft); }); // For each bucket, return the new value if it's different, otherwise return the stable ref - if (!shallowEquals(contentFragments, contentFragmentsRef.current)) - contentFragmentsRef.current = contentFragments; + if (!shallowEquals(contentOrVoidFragments, contentOrVoidFragmentsRef.current)) + contentOrVoidFragmentsRef.current = contentOrVoidFragments; if (!shallowEquals(imageAttachments, imageAttachmentsRef.current)) imageAttachmentsRef.current = imageAttachments; @@ -50,7 +53,7 @@ export function useFragmentBuckets(messageFragments: DMessageFragment[]): Fragme nonImageAttachmentsRef.current = nonImageAttachments; return { - contentFragments: contentFragmentsRef.current, + contentOrVoidFragments: contentOrVoidFragmentsRef.current, imageAttachments: imageAttachmentsRef.current, nonImageAttachments: nonImageAttachmentsRef.current, }; diff --git a/src/apps/chat/components/message/useSelHighlighterMemo.ts b/src/apps/chat/components/message/useSelHighlighterMemo.ts index d930c2e7e..d1d7690a0 100644 --- a/src/apps/chat/components/message/useSelHighlighterMemo.ts +++ b/src/apps/chat/components/message/useSelHighlighterMemo.ts @@ -1,7 +1,7 @@ import * as React from 'react'; import type { DMessageId } from '~/common/stores/chat/chat.message'; -import { createTextContentFragment, DMessageContentFragment, DMessageFragment, DMessageFragmentId, isTextPart } from '~/common/stores/chat/chat.fragments'; +import { createTextContentFragment, DMessageFragment, DMessageFragmentId, isContentFragment, isTextPart } from '~/common/stores/chat/chat.fragments'; import { wrapWithMarkdownSyntax } from '~/modules/blocks/markdown/markdown.wrapper'; import { BUBBLE_MIN_TEXT_LENGTH } from './ChatMessage'; @@ -29,7 +29,7 @@ type HighlightTool = 'highlight' | 'strike' | 'strong'; export function useSelHighlighterMemo( messageId: DMessageId, selText: string | null, - contentFragments: DMessageContentFragment[], + fragments: DMessageFragment[], fromAssistant: boolean, onMessageFragmentReplace?: (messageId: DMessageId, fragmentId: DMessageFragmentId, newFragment: DMessageFragment) => void, ): ((tool: HighlightTool) => void) | null { @@ -40,8 +40,8 @@ export function useSelHighlighterMemo( return null; // Create the highlighter function, if there's 1 and only 1 occurrence of the selection - const highlightFunction = contentFragments.reduce((acc: false /* not found */ | ((tool: HighlightTool) => void) | true /* more than one */, fragment) => { - if (!acc && isTextPart(fragment.part)) { + const highlightFunction = fragments.reduce((acc: false /* not found */ | ((tool: HighlightTool) => void) | true /* more than one */, fragment) => { + if (!acc && isContentFragment(fragment) && isTextPart(fragment.part)) { const fragmentText = fragment.part.text; let index = fragmentText.indexOf(selText); @@ -73,5 +73,5 @@ export function useSelHighlighterMemo( }, false); return typeof highlightFunction === 'function' ? highlightFunction : null; - }, [selText, fromAssistant, onMessageFragmentReplace, contentFragments, messageId]); + }, [fragments, fromAssistant, messageId, onMessageFragmentReplace, selText]); } diff --git a/src/common/stores/chat/chat.conversation.ts b/src/common/stores/chat/chat.conversation.ts index b63e2172f..7a355622d 100644 --- a/src/common/stores/chat/chat.conversation.ts +++ b/src/common/stores/chat/chat.conversation.ts @@ -2,7 +2,7 @@ import { defaultSystemPurposeId, SystemPurposeId } from '../../../data'; import { agiUuid } from '~/common/util/idUtils'; -import { DMessage, DMessageId, duplicateDMessageNoPH } from './chat.message'; +import { DMessage, DMessageId, duplicateDMessageNoVoid } from './chat.message'; /// Conversation @@ -66,7 +66,7 @@ export function createDConversation(systemPurposeId?: SystemPurposeId): DConvers }; } -export function duplicateDConversationNoPH(conversation: DConversation, lastMessageId?: DMessageId): DConversation { +export function duplicateDConversationNoVoid(conversation: DConversation, lastMessageId?: DMessageId): DConversation { // cut short messages, if requested let messagesToKeep = conversation.messages.length; // By default, include all messages if messageId is null @@ -84,7 +84,7 @@ export function duplicateDConversationNoPH(conversation: DConversation, lastMess messages: conversation.messages .slice(0, messagesToKeep) - .map(duplicateDMessageNoPH), // [*] duplicate conversation - see downstream + .map(duplicateDMessageNoVoid), // [*] duplicate conversation - see downstream // userTitle: conversation.userTitle, // undefined autoTitle: newTitle, diff --git a/src/common/stores/chat/chat.fragments.ts b/src/common/stores/chat/chat.fragments.ts index 89d8b4cff..3576aef62 100644 --- a/src/common/stores/chat/chat.fragments.ts +++ b/src/common/stores/chat/chat.fragments.ts @@ -22,26 +22,32 @@ import { agiId } from '~/common/util/idUtils'; export type DMessageFragment = | DMessageContentFragment | DMessageAttachmentFragment + | DMessageVoidFragment // | DMessageBeamFragment - | _DMessageSentinelFragment + | _SentinelFragment ; -// Content: real signal, needs to be sent to the llm +/** + * Content Fragments: understood by ai and humans, processed by llms and stored + */ export type DMessageContentFragment = _DMessageFragmentWrapper<'content', | DMessageTextPart // plain text or mixed content -> BlockRenderer | DMessageImageRefPart // large image | DMessageToolInvocationPart // shown to dev only, singature of the llm function call | DMessageToolResponsePart // shown to dev only, response of the llm | DMessageErrorPart // red message, e.g. non-content application issues - | DMetaPlaceholderPart // (non submitted) placeholder to be replaced by another part - | _DMetaSentinelPart + | _SentinelPart >; -// Attachments: labeled docs or images, output of Composer > Attachments +/** + * Attachment Fragments: higher level representation of content, usually from attachments, + * - image references, documents, etc. + * - may still have upstream links for instance + */ export type DMessageAttachmentFragment = _DMessageFragmentWrapper<'attachment', | DMessageDocPart // document Attachment | DMessageImageRefPart // image Attachment - | _DMetaSentinelPart + | _SentinelPart > & { title: string; // label of the attachment (filename, named id, content overview, title..) caption: string; // additional information, such as provenance, content preview, etc. @@ -49,6 +55,15 @@ export type DMessageAttachmentFragment = _DMessageFragmentWrapper<'attachment', liveFileId?: LiveFileId; // [LiveFile] Optional. Relate to a LiveFile; if present, it may still be invalid, hence we cleanup on load }; +/** + * Void Fragments: no meaning, pure cosmetic, not stored, not processed + */ +export type DMessageVoidFragment = _DMessageFragmentWrapper<'void', + | DVoidPlaceholderPart // (non submitted) placeholder to be replaced by another part + | _SentinelPart +>; + + // Future Examples: up to 1 per message, containing the Rays and Merges that would be used to restore the Beam state - could be volatile (omitted at save) // could not be the data store itself, but only used for save/reload // export type DMessageBeamFragment = DMessageBaseFragment<'beam'> & { @@ -58,7 +73,7 @@ export type DMessageAttachmentFragment = _DMessageFragmentWrapper<'attachment', // } // Sentinel: force the typesystem to work, bark, and detect/reveal corner cases - unused aside from revealing fragment type issues -type _DMessageSentinelFragment = { ft: '_ft_sentinel', fId: DMessageFragmentId }; +type _SentinelFragment = { ft: '_ft_sentinel', fId: DMessageFragmentId }; export type DMessageFragmentId = string; // not unique, 8 bytes type _DMessageFragmentWrapper = { @@ -135,9 +150,9 @@ export type DMessageToolResponsePart = { }; type DMessageToolEnvironment = 'upstream' | 'server' | 'client'; -export type DMetaPlaceholderPart = { pt: 'ph', pText: string }; +type DVoidPlaceholderPart = { pt: 'ph', pText: string }; -type _DMetaSentinelPart = { pt: '_pt_sentinel' }; +type _SentinelPart = { pt: '_pt_sentinel' }; // @@ -169,6 +184,10 @@ export function isContentOrAttachmentFragment(fragment: DMessageFragment) { return fragment.ft === 'content' || fragment.ft === 'attachment'; } +export function isVoidFragment(fragment: DMessageFragment) { + return fragment.ft === 'void'; +} + export function isDocPart(part: DMessageContentFragment['part'] | DMessageAttachmentFragment['part']) { return part.pt === 'doc'; @@ -190,6 +209,10 @@ export function isToolResponseFunctionCallPart(part: DMessageContentFragment['pa return part.pt === 'tool_response' && part.response.type === 'function_call'; } +export function isPlaceholderPart(part: DMessageVoidFragment['part']) { + return part.pt === 'ph'; +} + /// Content Fragments - Creation & Duplication @@ -221,11 +244,6 @@ export function create_CodeExecutionResponse_ContentFragment(id: string, error: return _createContentFragment(_create_CodeExecutionResponse_Part(id, error, result, executor, environment)); } -export function specialShallowReplaceTextContentFragment(copyFragment: DMessageContentFragment, text: string): DMessageContentFragment { - // TODO: remove? - return { ...copyFragment, part: _create_Text_Part(text) }; -} - function _createContentFragment(part: DMessageContentFragment['part']): DMessageContentFragment { return { ft: 'content', fId: agiId('chat-dfragment' /* -content */), part }; } @@ -257,20 +275,26 @@ function _createAttachmentFragment(title: string, caption: string, part: DMessag } -/// Meta Fragments - Creation & Duplication +/// Void Fragments - Creation & Duplication -export function createPlaceholderMetaFragment(placeholderText: string): DMessageContentFragment { - return _createContentFragment(_create_Placeholder_Part(placeholderText)); +export function createPlaceholderVoidFragment(placeholderText: string): DMessageVoidFragment { + return _createVoidFragment(_create_Placeholder_Part(placeholderText)); +} + +function _createVoidFragment(part: DMessageVoidFragment['part']): DMessageVoidFragment { + return { ft: 'void', fId: agiId('chat-dfragment' /* -void */), part }; } -function _createSentinelFragment(): _DMessageSentinelFragment { +/// Sentinel Fragments - only here to force the typesystem to work + +function _createSentinelFragment(): _SentinelFragment { return { ft: '_ft_sentinel', fId: agiId('chat-dfragment' /* -_sentinel */) }; } -export function duplicateDMessageFragmentsNoPH(fragments: Readonly): DMessageFragment[] { - return fragments.map(_duplicateFragment).filter(f => f.ft !== 'content' || f.part.pt !== 'ph'); +export function duplicateDMessageFragmentsNoVoid(fragments: Readonly): DMessageFragment[] { + return fragments.map(_duplicateFragment).filter(f => f.ft !== 'void'); } function _duplicateFragment(fragment: DMessageFragment): DMessageFragment { @@ -281,6 +305,9 @@ function _duplicateFragment(fragment: DMessageFragment): DMessageFragment { case 'attachment': return _createAttachmentFragment(fragment.title, fragment.caption, _duplicate_Part(fragment.part), fragment.liveFileId); + case 'void': + return _createVoidFragment(_duplicate_Part(fragment.part)); + case '_ft_sentinel': return _createSentinelFragment(); @@ -324,15 +351,15 @@ function _create_CodeExecutionResponse_Part(id: string, error: boolean | string, return { pt: 'tool_response', id, error, response: { type: 'code_execution', result, executor }, environment }; } -function _create_Placeholder_Part(placeholderText: string): DMetaPlaceholderPart { +function _create_Placeholder_Part(placeholderText: string): DVoidPlaceholderPart { return { pt: 'ph', pText: placeholderText }; } -function _create_Sentinel_Part(): _DMetaSentinelPart { +function _create_Sentinel_Part(): _SentinelPart { return { pt: '_pt_sentinel' }; } -function _duplicate_Part(part: TPart): TPart { +function _duplicate_Part(part: TPart): TPart { switch (part.pt) { case 'doc': return _create_Doc_Part(part.vdt, _duplicate_InlineData(part.data), part.ref, part.l1Title, part.meta ? { ...part.meta } : undefined) as TPart; @@ -414,7 +441,6 @@ function _duplicate_DataReference(ref: DMessageDataRef): DMessageDataRef { /// Editor Helpers - Fragment Editing - export function editTextPartsInline(fragments: DMessageFragment[], editText: (text: string, idx: number) => string): void { fragments.forEach((fragment, idx) => { if (isContentFragment(fragment) && isTextPart(fragment.part)) diff --git a/src/common/stores/chat/chat.message.ts b/src/common/stores/chat/chat.message.ts index 5b97dda05..cbce6ac9b 100644 --- a/src/common/stores/chat/chat.message.ts +++ b/src/common/stores/chat/chat.message.ts @@ -1,6 +1,6 @@ import { agiUuid } from '~/common/util/idUtils'; -import { createPlaceholderMetaFragment, createTextContentFragment, DMessageContentFragment, DMessageFragment, duplicateDMessageFragmentsNoPH, isAttachmentFragment, isContentFragment, isContentOrAttachmentFragment, isTextPart, specialShallowReplaceTextContentFragment } from './chat.fragments'; +import { createPlaceholderVoidFragment, createTextContentFragment, DMessageFragment, duplicateDMessageFragmentsNoVoid, isAttachmentFragment, isContentFragment, isContentOrAttachmentFragment, isVoidFragment } from './chat.fragments'; import type { ModelVendorId } from '~/modules/llms/vendors/vendors.registry'; @@ -112,7 +112,7 @@ export function createDMessageTextContent(role: DMessageRole, text: string): DMe } export function createDMessagePlaceholderIncomplete(role: DMessageRole, placeholderText: string): DMessage { - const placeholderFragment = createPlaceholderMetaFragment(placeholderText); + const placeholderFragment = createPlaceholderVoidFragment(placeholderText); const message = createDMessageFromFragments(role, [placeholderFragment]); message.pendingIncomplete = true; return message; @@ -145,12 +145,12 @@ export function createDMessageFromFragments(role: DMessageRole, fragments: DMess // helpers - duplication -export function duplicateDMessageNoPH(message: Readonly): DMessage { +export function duplicateDMessageNoVoid(message: Readonly): DMessage { return { id: agiUuid('chat-dmessage'), role: message.role, - fragments: duplicateDMessageFragmentsNoPH(message.fragments), // [*] full message duplication (see downstream) + fragments: duplicateDMessageFragmentsNoVoid(message.fragments), // [*] full message duplication (see downstream) ...(message.pendingIncomplete ? { pendingIncomplete: true } : {}), @@ -245,11 +245,9 @@ export function messageFragmentsReduceText(fragments: DMessageFragment[], fragme return fragment.part.error; case 'image_ref': return ''; - case 'ph': - return ''; - // ignore tools case 'tool_invocation': case 'tool_response': + // Ignore tools for the text reduction return ''; } } else if (isAttachmentFragment(fragment)) { @@ -259,6 +257,9 @@ export function messageFragmentsReduceText(fragments: DMessageFragment[], fragme case 'image_ref': return ''; } + } else if (isVoidFragment(fragment)) { + // all void fragments are ignored by definition when doing a text reduction + return ''; } console.warn(`DEV: messageFragmentsReduceText: unexpected '${fragment.ft}' fragment with '${(fragment as any)?.part?.pt}' part`); return ''; @@ -267,20 +268,6 @@ export function messageFragmentsReduceText(fragments: DMessageFragment[], fragme .join(fragmentSeparator); } -export function messageFragmentsReplaceLastContentText(fragments: Readonly, newText: string, appendText?: boolean): DMessageFragment[] { - - // if there's no text fragment, create it - const lastTextFragment = fragments.findLast(f => isContentFragment(f) && isTextPart(f.part)) as DMessageContentFragment | undefined; - if (!lastTextFragment) - return [...fragments, createTextContentFragment(newText)]; - - // append/replace the last text fragment - return fragments.map(fragment => - (fragment === lastTextFragment) - ? specialShallowReplaceTextContentFragment(lastTextFragment, (appendText && isTextPart(lastTextFragment.part)) ? lastTextFragment.part.text + newText : newText) - : fragment, - ); -} // TODO: remove once the port is fully done - at 2.0.0 ? export function messageSingleTextOrThrow(message: DMessage): string { diff --git a/src/common/stores/chat/chat.tokens.ts b/src/common/stores/chat/chat.tokens.ts index c1f068980..9064a40be 100644 --- a/src/common/stores/chat/chat.tokens.ts +++ b/src/common/stores/chat/chat.tokens.ts @@ -3,7 +3,7 @@ import { imageTokensForLLM } from '~/common/tokens/tokens.image'; import { textTokensForLLM } from '~/common/tokens/tokens.text'; import type { DMessageRole } from './chat.message'; -import { DMessageAttachmentFragment, DMessageFragment, isAttachmentFragment, isContentFragment, isContentOrAttachmentFragment, isDocPart } from './chat.fragments'; +import { DMessageAttachmentFragment, DMessageFragment, isAttachmentFragment, isContentFragment, isContentOrAttachmentFragment, isDocPart, isVoidFragment } from './chat.fragments'; export function estimateTokensForFragments(llm: DLLM, role: DMessageRole, fragments: DMessageFragment[], addTopGlue: boolean, debugFrom: string) { @@ -54,8 +54,6 @@ function _fragmentTokens(llm: DLLM, role: DMessageRole, fragment: DMessageFragme case 'image_ref': const forcedSize = role === 'assistant' ? 512 : undefined; return estimateImageTokens(forcedSize || cPart.width, forcedSize || cPart.height, debugFrom, llm); - case 'ph': - return 0; case 'text': return estimateTextTokens(cPart.text, llm, debugFrom); case 'tool_invocation': @@ -63,6 +61,10 @@ function _fragmentTokens(llm: DLLM, role: DMessageRole, fragment: DMessageFragme console.warn('Unhandled token preview for content type:', cPart.pt); return 0; } + } else if (isVoidFragment(fragment)) { + // all void fragments are ignored by definition and never sent to the llm + // NOTE: make sure you collapse/don't account for the containing message as well, if left empty + return 0; } else { console.warn('Unhandled token preview for fragment type:', (fragment as any).ft); return 0; diff --git a/src/common/stores/chat/chats.converters.ts b/src/common/stores/chat/chats.converters.ts index dbd76b77f..9100933c3 100644 --- a/src/common/stores/chat/chats.converters.ts +++ b/src/common/stores/chat/chats.converters.ts @@ -8,7 +8,7 @@ import type { DModelsService } from '~/common/stores/llms/modelsservice.types'; import { createDConversation, DConversation, type DConversationId } from './chat.conversation'; import { createDMessageTextContent, DMessage, MESSAGE_FLAG_NOTIFY_COMPLETE, messageSetUserFlag } from './chat.message'; -import { createErrorContentFragment, isAttachmentFragment, isContentFragment, isContentOrAttachmentFragment, isDocPart, isTextPart } from './chat.fragments'; +import { createErrorContentFragment, isAttachmentFragment, isContentFragment, isContentOrAttachmentFragment, isDocPart, isPlaceholderPart, isTextPart, isVoidFragment } from './chat.fragments'; // configuration @@ -56,8 +56,9 @@ export namespace V4ToHeadConverters { delete fragment.liveFileId; // show the aborted ops: convert a Placeholder fragment [part.pt='ph'] to an Error fragment - if (isContentFragment(fragment) && fragment.part.pt === 'ph') - m.fragments[i] = createErrorContentFragment(`${fragment.part.pText} (did not complete)`); + if ((isVoidFragment(fragment) && isPlaceholderPart(fragment.part)) + || (isContentFragment(fragment) && (fragment.part as any)?.pt === 'ph') /* NOTE: REMOVE FOR 2.0: helper during the 'void' fragment transition */) + m.fragments[i] = createErrorContentFragment(`${(fragment.part as any).pText} (did not complete)`); // [Emergency] validate part types, can mess up in development if (EMERGENCY_CLEANUP_PARTS) { diff --git a/src/common/stores/chat/store-chats.ts b/src/common/stores/chat/store-chats.ts index 9ccdf9414..b49c070e8 100644 --- a/src/common/stores/chat/store-chats.ts +++ b/src/common/stores/chat/store-chats.ts @@ -16,7 +16,7 @@ import { workspaceForConversationIdentity } from '~/common/stores/workspace/work import { DMessage, DMessageId, DMessageMetadata, MESSAGE_FLAG_AIX_SKIP, messageHasUserFlag } from './chat.message'; import type { DMessageFragment, DMessageFragmentId } from './chat.fragments'; import { V3StoreDataToHead, V4ToHeadConverters } from './chats.converters'; -import { conversationTitle, createDConversation, DConversation, DConversationId, duplicateDConversationNoPH } from './chat.conversation'; +import { conversationTitle, createDConversation, DConversation, DConversationId, duplicateDConversationNoVoid } from './chat.conversation'; import { estimateTokensForFragments } from './chat.tokens'; @@ -120,7 +120,7 @@ export const useChatStore = create()(/*devtools(*/ if (!conversation) return null; - const branched = duplicateDConversationNoPH(conversation, messageId ?? undefined); + const branched = duplicateDConversationNoVoid(conversation, messageId ?? undefined); _set({ conversations: [branched, ...conversations], diff --git a/src/modules/aifn/auto-chat-follow-ups/autoChatFollowUps.ts b/src/modules/aifn/auto-chat-follow-ups/autoChatFollowUps.ts index 72c970c9c..fa3a8166a 100644 --- a/src/modules/aifn/auto-chat-follow-ups/autoChatFollowUps.ts +++ b/src/modules/aifn/auto-chat-follow-ups/autoChatFollowUps.ts @@ -7,7 +7,7 @@ import { aixChatGenerateContent_DMessage, aixCreateChatGenerateContext } from '~ import { ConversationsManager } from '~/common/chat-overlay/ConversationsManager'; import { createDMessageTextContent, messageFragmentsReduceText } from '~/common/stores/chat/chat.message'; -import { createErrorContentFragment, createPlaceholderMetaFragment, createTextContentFragment } from '~/common/stores/chat/chat.fragments'; +import { createErrorContentFragment, createPlaceholderVoidFragment, createTextContentFragment } from '~/common/stores/chat/chat.fragments'; import { getLLMIdOrThrow } from '~/common/stores/llms/store-llms'; import { marshallWrapText } from '~/common/stores/chat/chat.tokens'; import { processPromptTemplate } from '~/common/util/promptUtils'; @@ -175,7 +175,7 @@ export async function autoChatFollowUps(conversationId: string, assistantMessage if (suggestDiagrams && !['@startuml', '@startmindmap', '```plantuml', '```mermaid'].some(s => assistantMessageText.includes(s))) { // Placeholder for the diagram - const placeholderFragment = createPlaceholderMetaFragment('Auto-Diagram ...'); + const placeholderFragment = createPlaceholderVoidFragment('Auto-Diagram ...'); cHandler.messageFragmentAppend(assistantMessageId, placeholderFragment, false, false); // Instructions @@ -226,7 +226,7 @@ export async function autoChatFollowUps(conversationId: string, assistantMessage if (suggestHTMLUI && ![' assistantMessageText.includes(s))) { // Placeholder for the UI - const placeholderFragment = createPlaceholderMetaFragment('Auto-UI ...'); + const placeholderFragment = createPlaceholderVoidFragment('Auto-UI ...'); cHandler.messageFragmentAppend(assistantMessageId, placeholderFragment, false, false); // Instructions diff --git a/src/modules/aix/client/aix.client.chatGenerateRequest.ts b/src/modules/aix/client/aix.client.chatGenerateRequest.ts index 208cd6392..d66edf7fa 100644 --- a/src/modules/aix/client/aix.client.chatGenerateRequest.ts +++ b/src/modules/aix/client/aix.client.chatGenerateRequest.ts @@ -105,7 +105,7 @@ export async function aixCGR_FromDMessagesOrThrow( const aixChatMessageUser = await dMessageUserFragments.reduce(async (uMsgPromise, uFragment: DMessageFragment) => { const uMsg = await uMsgPromise; - if (!isContentOrAttachmentFragment(uFragment) || uFragment.part.pt === '_pt_sentinel' || uFragment.part.pt === 'ph') + if (!isContentOrAttachmentFragment(uFragment) || uFragment.part.pt === '_pt_sentinel') return uMsg; switch (uFragment.part.pt) { @@ -158,7 +158,7 @@ export async function aixCGR_FromDMessagesOrThrow( for (const aFragment of m.fragments) { - if (!isContentOrAttachmentFragment(aFragment) || aFragment.part.pt === '_pt_sentinel' || aFragment.part.pt === 'ph') + if (!isContentOrAttachmentFragment(aFragment) || aFragment.part.pt === '_pt_sentinel') continue; switch (aFragment.part.pt) { diff --git a/src/modules/aix/client/aix.client.ts b/src/modules/aix/client/aix.client.ts index bc503d54e..17bc443c5 100644 --- a/src/modules/aix/client/aix.client.ts +++ b/src/modules/aix/client/aix.client.ts @@ -309,7 +309,6 @@ function _llToText(src: AixChatGenerateContent_LL, dest: AixChatGenerateText_Sim break; case 'tool_invocation': throw new Error(`AIX: Unexpected tool invocation ${fragment.part.invocation?.type === 'function_call' ? fragment.part.invocation.name : fragment.part.id} in the Text response.`); - case 'ph': // impossible case 'image_ref': // impossible case 'tool_response': // impossible - stopped at the invocation alrady case '_pt_sentinel': // impossible diff --git a/src/modules/beam/gather/instructions/beam.gather.execution.tsx b/src/modules/beam/gather/instructions/beam.gather.execution.tsx index 8ee3e6b2d..411cbe4af 100644 --- a/src/modules/beam/gather/instructions/beam.gather.execution.tsx +++ b/src/modules/beam/gather/instructions/beam.gather.execution.tsx @@ -3,7 +3,7 @@ import { Typography } from '@mui/joy'; import type { DLLMId } from '~/common/stores/llms/llms.types'; import { createDMessageEmpty, DMessage } from '~/common/stores/chat/chat.message'; -import { createPlaceholderMetaFragment } from '~/common/stores/chat/chat.fragments'; +import { createPlaceholderVoidFragment } from '~/common/stores/chat/chat.fragments'; import type { BFusion, FusionUpdateOrFn } from '../beam.gather'; import { executeGatherInstruction, GatherInstruction } from './GatherInstruction'; @@ -110,7 +110,7 @@ export function gatherStartFusion( ); // reset the intermediate message - inputState.intermediateDMessage.fragments = [createPlaceholderMetaFragment(GATHER_PLACEHOLDER)]; + inputState.intermediateDMessage.fragments = [createPlaceholderVoidFragment(GATHER_PLACEHOLDER)]; inputState.intermediateDMessage.pendingIncomplete = true; inputState.intermediateDMessage.updated = null; diff --git a/src/modules/beam/scatter/beam.scatter.ts b/src/modules/beam/scatter/beam.scatter.ts index ffe40d71d..5a974433f 100644 --- a/src/modules/beam/scatter/beam.scatter.ts +++ b/src/modules/beam/scatter/beam.scatter.ts @@ -4,8 +4,8 @@ import { AixChatGenerateContent_DMessage, aixChatGenerateContent_DMessage_FromHi import type { DLLMId } from '~/common/stores/llms/llms.types'; import { agiUuid } from '~/common/util/idUtils'; -import { createDMessageEmpty, DMessage, duplicateDMessageNoPH, messageWasInterruptedAtStart } from '~/common/stores/chat/chat.message'; -import { createPlaceholderMetaFragment } from '~/common/stores/chat/chat.fragments'; +import { createDMessageEmpty, DMessage, duplicateDMessageNoVoid, messageWasInterruptedAtStart } from '~/common/stores/chat/chat.message'; +import { createPlaceholderVoidFragment } from '~/common/stores/chat/chat.fragments'; import { findLLMOrThrow } from '~/common/stores/llms/store-llms'; import { getUXLabsHighPerformance } from '~/common/state/store-ux-labs'; @@ -93,7 +93,7 @@ function rayScatterStart(ray: BRay, llmId: DLLMId | null, inputHistory: DMessage const newMessage: DMessage = { ...ray.message, - fragments: [createPlaceholderMetaFragment(SCATTER_PLACEHOLDER)], + fragments: [createPlaceholderVoidFragment(SCATTER_PLACEHOLDER)], pendingIncomplete: true, created: Date.now(), updated: null, @@ -261,7 +261,7 @@ export const createScatterSlice: StateCreator