diff --git a/src/apps/chat/AppChat.tsx b/src/apps/chat/AppChat.tsx index eb5ebb036..c4601a1ea 100644 --- a/src/apps/chat/AppChat.tsx +++ b/src/apps/chat/AppChat.tsx @@ -20,7 +20,7 @@ import type { OptimaBarControlMethods } from '~/common/layout/optima/bar/OptimaB import { ConfirmationModal } from '~/common/components/modals/ConfirmationModal'; import { ConversationsManager } from '~/common/chat-overlay/ConversationsManager'; import { DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragments } from '~/common/stores/chat/chat.fragments'; -import { LLM_IF_ANT_PromptCaching } from '~/common/stores/llms/llms.types'; +import { LLM_IF_ANT_PromptCaching, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types'; import { OptimaDrawerIn, OptimaToolbarIn } from '~/common/layout/optima/portals/OptimaPortalsIn'; import { PanelResizeInset } from '~/common/components/panes/GoodPanelResizeHandler'; import { ScrollToBottom } from '~/common/scroll-to-bottom/ScrollToBottom'; @@ -559,6 +559,7 @@ export function AppChat() { capabilityHasT2I={capabilityHasT2I} chatLLMAntPromptCaching={chatLLM?.interfaces?.includes(LLM_IF_ANT_PromptCaching) ?? false} chatLLMContextTokens={chatLLM?.contextTokens ?? null} + chatLLMSupportsImages={chatLLM?.interfaces?.includes(LLM_IF_OAI_Vision) ?? false} fitScreen={isMobile || isMultiPane} isMobile={isMobile} isMessageSelectionMode={isMessageSelectionMode} diff --git a/src/apps/chat/components/ChatMessageList.tsx b/src/apps/chat/components/ChatMessageList.tsx index 8dc7d0498..cc0a4ee65 100644 --- a/src/apps/chat/components/ChatMessageList.tsx +++ b/src/apps/chat/components/ChatMessageList.tsx @@ -41,6 +41,7 @@ export function ChatMessageList(props: { capabilityHasT2I: boolean, chatLLMAntPromptCaching: boolean, chatLLMContextTokens: number | null, + chatLLMSupportsImages: boolean, fitScreen: boolean, isMobile: boolean, isMessageSelectionMode: boolean, @@ -99,7 +100,9 @@ export function ChatMessageList(props: { await openFileForAttaching(true, async (filesWithHandle) => { // Retrieve fully-fledged Attachment Fragments (converted/extracted, with sources, mimes, etc.) from the selected files - const attachmentFragments = await convertFilesToDAttachmentFragments('file-open', filesWithHandle); + const attachmentFragments = await convertFilesToDAttachmentFragments('file-open', filesWithHandle, { + hintAddImages: props.chatLLMSupportsImages, + }); // Create a User message with the prompt and the attachment fragments if (attachmentFragments.length) { @@ -112,7 +115,7 @@ export function ChatMessageList(props: { }); break; } - }, [conversationHandler, conversationId, onConversationExecuteHistory]); + }, [conversationHandler, conversationId, onConversationExecuteHistory, props.chatLLMSupportsImages]); const handleMessageContinue = React.useCallback(async (_messageId: DMessageId /* Ignored for now */) => { if (conversationId && conversationHandler) { diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index 4aa0e7680..dd13ce093 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -20,7 +20,7 @@ import type { DOpenAILLMOptions } from '~/modules/llms/vendors/openai/openai.ven import { useAgiAttachmentPrompts } from '~/modules/aifn/agiattachmentprompts/useAgiAttachmentPrompts'; import { useBrowseCapability } from '~/modules/browse/store-module-browsing'; -import type { DLLM } from '~/common/stores/llms/llms.types'; +import { DLLM, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types'; import { AudioGenerator } from '~/common/util/audio/AudioGenerator'; import { AudioPlayer } from '~/common/util/audio/AudioPlayer'; import { ButtonAttachFilesMemo, openFileForAttaching } from '~/common/components/ButtonAttachFiles'; @@ -149,6 +149,10 @@ export function Composer(props: { const allowInReferenceTo = chatExecuteMode === 'generate-content'; const inReferenceTo = useChatComposerOverlayStore(conversationOverlayStore, store => allowInReferenceTo ? store.inReferenceTo : null); + // LLM-derived + const noLLM = !props.chatLLM; + const chatLLMSupportsImages = !!props.chatLLM?.interfaces?.includes(LLM_IF_OAI_Vision); + // don't load URLs if the user is typing a command or there's no capability const hasComposerBrowseCapability = useBrowseCapability().inComposer; const enableLoadURLsInComposer = hasComposerBrowseCapability && !composeText.startsWith('/'); @@ -158,10 +162,10 @@ export function Composer(props: { /* items */ attachmentDrafts, /* append */ attachAppendClipboardItems, attachAppendDataTransfer, attachAppendEgoFragments, attachAppendFile, /* take */ attachmentsRemoveAll, attachmentsTakeAllFragments, attachmentsTakeFragmentsByType, - } = useAttachmentDrafts(conversationOverlayStore, enableLoadURLsInComposer); + } = useAttachmentDrafts(conversationOverlayStore, enableLoadURLsInComposer, chatLLMSupportsImages); // attachments derived state - const llmAttachmentDraftsCollection = useLLMAttachmentDrafts(attachmentDrafts, props.chatLLM); + const llmAttachmentDraftsCollection = useLLMAttachmentDrafts(attachmentDrafts, props.chatLLM, chatLLMSupportsImages); // drag/drop const { dragContainerSx, dropComponent, handleContainerDragEnter, handleContainerDragStart } = useComposerDragDrop(!props.isMobile, attachAppendDataTransfer); @@ -176,8 +180,7 @@ export function Composer(props: { const isMobile = props.isMobile; const isDesktop = !props.isMobile; const noConversation = !targetConversationId; - const noLLM = !props.chatLLM; - const showLLMAttachments = chatExecuteModeCanAttach(chatExecuteMode); + const showChatAttachments = chatExecuteModeCanAttach(chatExecuteMode); // tokens derived state @@ -286,7 +289,7 @@ export function Composer(props: { if (enqueued) handleClear(); return enqueued; - }, [attachmentsTakeAllFragments, handleClear, inReferenceTo, onAction, targetConversationId]); + }, [attachmentsTakeAllFragments, confirmProceedIfAttachmentsNotSupported, handleClear, inReferenceTo, onAction, targetConversationId]); const handleAppendTextAndSend = React.useCallback(async (appendText: string) => { @@ -539,7 +542,7 @@ export function Composer(props: { useGlobalShortcuts('ChatComposer', React.useMemo(() => { const composerShortcuts: ShortcutObject[] = []; - if (showLLMAttachments) { + if (showChatAttachments) { composerShortcuts.push({ key: 'f', ctrl: true, shift: true, action: () => openFileForAttaching(true, handleAttachFiles), description: 'Attach File' }); if (supportsClipboardRead) composerShortcuts.push({ key: 'v', ctrl: true, shift: true, action: attachAppendClipboardItems, description: 'Attach Clipboard' }); @@ -561,7 +564,7 @@ export function Composer(props: { }, description: 'Microphone', }); return composerShortcuts; - }, [attachAppendClipboardItems, handleAttachFiles, recognitionState.hasSpeech, recognitionState.isActive, showLLMAttachments, toggleRecognition])); + }, [attachAppendClipboardItems, handleAttachFiles, recognitionState.hasSpeech, recognitionState.isActive, showChatAttachments, toggleRecognition])); // ... @@ -659,10 +662,10 @@ export function Composer(props: { {recognitionState.isAvailable && } {/* Responsive Camera OCR button */} - {showLLMAttachments && } + {showChatAttachments && } {/* [mobile] [+] button */} - {showLLMAttachments && ( + {showChatAttachments && ( @@ -690,7 +693,7 @@ export function Composer(props: { )} {/* [Desktop, Col1] Insert Multi-modal content buttons */} - {isDesktop && showLLMAttachments && ( + {isDesktop && showChatAttachments && ( {/**/} @@ -847,7 +850,7 @@ export function Composer(props: { {/* Render any Attachments & menu items */} - {!!conversationOverlayStore && showLLMAttachments && ( + {!!conversationOverlayStore && showChatAttachments && ( pt === 'doc'); // Add LLM-specific properties to each attachment draft @@ -87,5 +86,5 @@ export function useLLMAttachmentDrafts(attachmentDrafts: AttachmentDraft[], chat llmTokenCountApprox, }; - }, [attachmentDrafts, chatLLM]); // Dependencies for the outer useMemo + }, [attachmentDrafts, chatLLM, chatLLMSupportsImages]); // Dependencies for the outer useMemo } diff --git a/src/common/attachment-drafts/attachment.pipeline.ts b/src/common/attachment-drafts/attachment.pipeline.ts index ebb3142e3..e00ed32e0 100644 --- a/src/common/attachment-drafts/attachment.pipeline.ts +++ b/src/common/attachment-drafts/attachment.pipeline.ts @@ -12,7 +12,7 @@ import { pdfToImageDataURLs, pdfToText } from '~/common/util/pdfUtils'; import { createDMessageDataInlineText, createDocAttachmentFragment, DMessageAttachmentFragment, DMessageDataInline, DMessageDocPart, DVMimeType, isContentOrAttachmentFragment, isDocPart, specialContentPartToDocAttachmentFragment } from '~/common/stores/chat/chat.fragments'; -import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftInput, AttachmentDraftSource, AttachmentDraftSourceOriginFile, DraftEgoFragmentsInputData, DraftWebInputData, DraftYouTubeInputData } from './attachment.types'; +import type { AttachmentCreationOptions, AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftInput, AttachmentDraftSource, AttachmentDraftSourceOriginFile, DraftEgoFragmentsInputData, DraftWebInputData, DraftYouTubeInputData } from './attachment.types'; import type { AttachmentsDraftsStore } from './store-attachment-drafts-slice'; import { attachmentGetLiveFileId, attachmentSourceSupportsLiveFile } from './attachment.livefile'; import { guessInputContentTypeFromMime, heuristicMimeTypeFixup, mimeTypeIsDocX, mimeTypeIsPDF, mimeTypeIsPlainText, mimeTypeIsSupportedImage, reverseLookupMimeType } from './attachment.mimetypes'; @@ -24,7 +24,7 @@ export const DEFAULT_ADRAFT_IMAGE_MIMETYPE = !Is.Browser.Safari ? 'image/webp' : export const DEFAULT_ADRAFT_IMAGE_QUALITY = 0.96; const PDF_IMAGE_PAGE_SCALE = 1.5; const PDF_IMAGE_QUALITY = 0.5; -const PDF_PREFER_TEXT_AND_IMAGES = false; +const ENABLE_TEXT_AND_IMAGES = false; // 2.0 // internal mimes, only used to route data within us (source -> input -> converters) @@ -209,13 +209,16 @@ export async function attachmentLoadInputAsync(source: Readonly} source - The source of the AttachmentDraft object. * @param {Readonly} input - The input of the AttachmentDraft object. + * @param options conversion preferences, if any * @param {(changes: Partial) => void} edit - A function to edit the AttachmentDraft object. */ -export function attachmentDefineConverters(source: AttachmentDraftSource, input: Readonly, edit: (changes: Partial>) => void) { +export function attachmentDefineConverters(source: AttachmentDraftSource, input: Readonly, options: AttachmentCreationOptions, edit: (changes: Partial>) => void) { // return all the possible converters for the input const converters: AttachmentDraftConverter[] = []; + const autoAddImages = ENABLE_TEXT_AND_IMAGES && !!options?.hintAddImages; + switch (true) { // plain text types @@ -240,20 +243,20 @@ export function attachmentDefineConverters(source: AttachmentDraftSource, input: // Images (Known/Unknown) case input.mimeType.startsWith('image/'): - const imageSupported = mimeTypeIsSupportedImage(input.mimeType); - converters.push({ id: 'image-resized-high', name: 'Image (high detail)', disabled: !imageSupported }); - converters.push({ id: 'image-resized-low', name: 'Image (low detail)', disabled: !imageSupported }); - converters.push({ id: 'image-original', name: 'Image (original quality)', disabled: !imageSupported }); - if (!imageSupported) + const inputImageMimeSupported = mimeTypeIsSupportedImage(input.mimeType); + converters.push({ id: 'image-resized-high', name: 'Image (high detail)', disabled: !inputImageMimeSupported }); + converters.push({ id: 'image-resized-low', name: 'Image (low detail)', disabled: !inputImageMimeSupported }); + converters.push({ id: 'image-original', name: 'Image (original quality)', disabled: !inputImageMimeSupported }); + if (!inputImageMimeSupported) converters.push({ id: 'image-to-default', name: `As Image (${DEFAULT_ADRAFT_IMAGE_MIMETYPE})` }); converters.push({ id: 'image-ocr', name: 'As Text (OCR)' }); break; // PDF case mimeTypeIsPDF(input.mimeType): - converters.push({ id: 'pdf-text', name: 'PDF To Text', isActive: !PDF_PREFER_TEXT_AND_IMAGES || undefined }); + converters.push({ id: 'pdf-text', name: 'PDF To Text', isActive: !autoAddImages || undefined }); converters.push({ id: 'pdf-images', name: 'PDF To Images' }); - converters.push({ id: 'pdf-text-and-images', name: 'PDF Text & Images (best)', isActive: PDF_PREFER_TEXT_AND_IMAGES || undefined }); + converters.push({ id: 'pdf-text-and-images', name: 'PDF Text & Images (best)', isActive: autoAddImages }); break; // DOCX @@ -274,16 +277,16 @@ export function attachmentDefineConverters(source: AttachmentDraftSource, input: if (input.urlImage) { if (converters.length) converters.push({ id: 'url-page-null', name: 'Do not attach' }); - converters.push({ id: 'url-page-image', name: 'Add Screenshot', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true }); + converters.push({ id: 'url-page-image', name: 'Add Screenshot', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true, isActive: autoAddImages || undefined }); } break; // YouTube: custom converters case input.mimeType === INT_MIME_VND_AGI_YOUTUBE: - converters.push({ id: 'youtube-transcript', name: 'Video Transcript' }); + converters.push({ id: 'youtube-transcript', name: 'Video Transcript', isActive: true }); converters.push({ id: 'youtube-transcript-simple', name: 'Video Transcript (simple)' }); if (input.urlImage) - converters.push({ id: 'url-page-image', name: 'Add Thumbnail', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true }); + converters.push({ id: 'url-page-image', name: 'Add Thumbnail', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true, isActive: autoAddImages }); break; // EGO @@ -778,7 +781,7 @@ function _inputDataToString(data: AttachmentDraftInput['data']): string { * * Only returns the fragments that were successfully converted. */ -export async function convertFilesToDAttachmentFragments(origin: AttachmentDraftSourceOriginFile, files: FileWithHandle[]): Promise { +export async function convertFilesToDAttachmentFragments(origin: AttachmentDraftSourceOriginFile, files: FileWithHandle[], options: AttachmentCreationOptions): Promise { const validOutputFragmentsList: DMessageAttachmentFragment[][] = []; for (const fileWithHandle of files) { @@ -801,7 +804,7 @@ export async function convertFilesToDAttachmentFragments(origin: AttachmentDraft } // 2. Define converters - attachmentDefineConverters(_draft.source, _draft.input, updateDraft); + attachmentDefineConverters(_draft.source, _draft.input, options, updateDraft); if (!_draft.converters.length) { console.warn(`No converters defined for file: ${fileWithHandle.name}`); continue; diff --git a/src/common/attachment-drafts/attachment.types.ts b/src/common/attachment-drafts/attachment.types.ts index e67a755ba..63a62c039 100644 --- a/src/common/attachment-drafts/attachment.types.ts +++ b/src/common/attachment-drafts/attachment.types.ts @@ -62,6 +62,10 @@ export type AttachmentDraftSourceOriginFile = 'camera' | 'screencapture' | 'file export type AttachmentDraftSourceOriginDTO = 'drop' | 'paste'; +export type AttachmentCreationOptions = { + hintAddImages?: boolean; +} + // 1. draft input (loaded from the source) diff --git a/src/common/attachment-drafts/store-attachment-drafts-slice.tsx b/src/common/attachment-drafts/store-attachment-drafts-slice.tsx index ba3b1be2c..f799be5b8 100644 --- a/src/common/attachment-drafts/store-attachment-drafts-slice.tsx +++ b/src/common/attachment-drafts/store-attachment-drafts-slice.tsx @@ -5,7 +5,7 @@ import type { DBlobDBContextId, DBlobDBScopeId } from '~/modules/dblobs/dblobs.t import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.fragments'; -import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftSource } from './attachment.types'; +import type { AttachmentCreationOptions, AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftSource } from './attachment.types'; import { attachmentCreate, attachmentDefineConverters, attachmentLoadInputAsync, attachmentPerformConversion } from './attachment.pipeline'; import { removeAttachmentOwnedDBAsset, transferAttachmentOwnedDBAsset } from './attachment.dblobs'; @@ -20,7 +20,7 @@ interface AttachmentDraftsState { export interface AttachmentsDraftsStore extends AttachmentDraftsState { - createAttachmentDraft: (source: AttachmentDraftSource) => Promise; + createAttachmentDraft: (source: AttachmentDraftSource, options: AttachmentCreationOptions) => Promise; removeAllAttachmentDrafts: () => void; removeAttachmentDraft: (attachmentDraftId: AttachmentDraftId) => void; moveAttachmentDraft: (attachmentDraftId: AttachmentDraftId, delta: 1 | -1) => void; @@ -52,7 +52,7 @@ export const createAttachmentDraftsStoreSlice: StateCreator { + createAttachmentDraft: async (source: AttachmentDraftSource, options: AttachmentCreationOptions) => { const { _getAttachment, _editAttachment, toggleAttachmentDraftConverterAndConvert } = _get(); const _attachmentDraft = attachmentCreate(source); @@ -70,7 +70,7 @@ export const createAttachmentDraftsStoreSlice: StateCreatorO Converters - attachmentDefineConverters(source, loaded.input, editFn); + attachmentDefineConverters(source, loaded.input, options, editFn); const defined = _getAttachment(attachmentDraftId); if (!defined?.converters.length) return; diff --git a/src/common/attachment-drafts/useAttachmentDrafts.tsx b/src/common/attachment-drafts/useAttachmentDrafts.tsx index 3f7629e6e..0ae89090d 100644 --- a/src/common/attachment-drafts/useAttachmentDrafts.tsx +++ b/src/common/attachment-drafts/useAttachmentDrafts.tsx @@ -20,7 +20,7 @@ import type { AttachmentDraftsStoreApi } from './store-attachment-drafts-slice'; const ATTACHMENTS_DEBUG_INTAKE = false; -export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreApi | null, enableLoadURLs: boolean) => { +export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreApi | null, enableLoadURLs: boolean, hintAddImages: boolean) => { // state const { _createAttachmentDraft, attachmentDrafts, attachmentsRemoveAll, attachmentsTakeAllFragments, attachmentsTakeFragmentsByType } = useChatAttachmentsStore(attachmentsStoreApi, useShallow(state => ({ @@ -43,8 +43,8 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp return _createAttachmentDraft({ media: 'file', origin, fileWithHandle, refPath: overrideFileName || fileWithHandle.name, - }); - }, [_createAttachmentDraft]); + }, { hintAddImages }); + }, [_createAttachmentDraft, hintAddImages]); /** * Append data transfer to the attachments. @@ -145,7 +145,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp if (textPlainUrl && textPlainUrl.trim()) { void _createAttachmentDraft({ media: 'url', url: textPlainUrl, refUrl: textPlain, - }); + }, { hintAddImages}); return 'as_url'; } @@ -155,7 +155,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp if (attachText && (textHtml || textPlain)) { void _createAttachmentDraft({ media: 'text', method, textPlain, textHtml, - }); + }, { hintAddImages }); return 'as_text'; } @@ -165,7 +165,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp // did not attach anything from this data transfer return false; - }, [attachAppendFile, _createAttachmentDraft, enableLoadURLs]); + }, [_createAttachmentDraft, attachAppendFile, enableLoadURLs, hintAddImages]); /** * Append clipboard items to the attachments. @@ -223,7 +223,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp if (textPlainUrl && textPlainUrl.trim()) { void _createAttachmentDraft({ media: 'url', url: textPlainUrl.trim(), refUrl: textPlain, - }); + }, { hintAddImages }); continue; } } @@ -232,13 +232,13 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp if (textHtml || textPlain) { void _createAttachmentDraft({ media: 'text', method: 'clipboard-read', textPlain, textHtml, - }); + }, { hintAddImages }); continue; } console.warn('Clipboard item has no text/html or text/plain item.', clipboardItem.types, clipboardItem); } - }, [attachAppendFile, _createAttachmentDraft, enableLoadURLs]); + }, [_createAttachmentDraft, attachAppendFile, enableLoadURLs, hintAddImages]); /** * Append ego content to the attachments. @@ -257,8 +257,8 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp conversationId, messageId, }, - }); - }, [_createAttachmentDraft]); + }, { hintAddImages }); + }, [_createAttachmentDraft, hintAddImages]); return {