Double-mode attachments

This commit is contained in:
Enrico Ros
2024-09-22 10:33:14 -07:00
parent 2258dee8c7
commit de3aa4a5f7
8 changed files with 63 additions and 50 deletions
+2 -1
View File
@@ -20,7 +20,7 @@ import type { OptimaBarControlMethods } from '~/common/layout/optima/bar/OptimaB
import { ConfirmationModal } from '~/common/components/modals/ConfirmationModal';
import { ConversationsManager } from '~/common/chat-overlay/ConversationsManager';
import { DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragments } from '~/common/stores/chat/chat.fragments';
import { LLM_IF_ANT_PromptCaching } from '~/common/stores/llms/llms.types';
import { LLM_IF_ANT_PromptCaching, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types';
import { OptimaDrawerIn, OptimaToolbarIn } from '~/common/layout/optima/portals/OptimaPortalsIn';
import { PanelResizeInset } from '~/common/components/panes/GoodPanelResizeHandler';
import { ScrollToBottom } from '~/common/scroll-to-bottom/ScrollToBottom';
@@ -559,6 +559,7 @@ export function AppChat() {
capabilityHasT2I={capabilityHasT2I}
chatLLMAntPromptCaching={chatLLM?.interfaces?.includes(LLM_IF_ANT_PromptCaching) ?? false}
chatLLMContextTokens={chatLLM?.contextTokens ?? null}
chatLLMSupportsImages={chatLLM?.interfaces?.includes(LLM_IF_OAI_Vision) ?? false}
fitScreen={isMobile || isMultiPane}
isMobile={isMobile}
isMessageSelectionMode={isMessageSelectionMode}
+5 -2
View File
@@ -41,6 +41,7 @@ export function ChatMessageList(props: {
capabilityHasT2I: boolean,
chatLLMAntPromptCaching: boolean,
chatLLMContextTokens: number | null,
chatLLMSupportsImages: boolean,
fitScreen: boolean,
isMobile: boolean,
isMessageSelectionMode: boolean,
@@ -99,7 +100,9 @@ export function ChatMessageList(props: {
await openFileForAttaching(true, async (filesWithHandle) => {
// Retrieve fully-fledged Attachment Fragments (converted/extracted, with sources, mimes, etc.) from the selected files
const attachmentFragments = await convertFilesToDAttachmentFragments('file-open', filesWithHandle);
const attachmentFragments = await convertFilesToDAttachmentFragments('file-open', filesWithHandle, {
hintAddImages: props.chatLLMSupportsImages,
});
// Create a User message with the prompt and the attachment fragments
if (attachmentFragments.length) {
@@ -112,7 +115,7 @@ export function ChatMessageList(props: {
});
break;
}
}, [conversationHandler, conversationId, onConversationExecuteHistory]);
}, [conversationHandler, conversationId, onConversationExecuteHistory, props.chatLLMSupportsImages]);
const handleMessageContinue = React.useCallback(async (_messageId: DMessageId /* Ignored for now */) => {
if (conversationId && conversationHandler) {
+15 -12
View File
@@ -20,7 +20,7 @@ import type { DOpenAILLMOptions } from '~/modules/llms/vendors/openai/openai.ven
import { useAgiAttachmentPrompts } from '~/modules/aifn/agiattachmentprompts/useAgiAttachmentPrompts';
import { useBrowseCapability } from '~/modules/browse/store-module-browsing';
import type { DLLM } from '~/common/stores/llms/llms.types';
import { DLLM, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types';
import { AudioGenerator } from '~/common/util/audio/AudioGenerator';
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
import { ButtonAttachFilesMemo, openFileForAttaching } from '~/common/components/ButtonAttachFiles';
@@ -149,6 +149,10 @@ export function Composer(props: {
const allowInReferenceTo = chatExecuteMode === 'generate-content';
const inReferenceTo = useChatComposerOverlayStore(conversationOverlayStore, store => allowInReferenceTo ? store.inReferenceTo : null);
// LLM-derived
const noLLM = !props.chatLLM;
const chatLLMSupportsImages = !!props.chatLLM?.interfaces?.includes(LLM_IF_OAI_Vision);
// don't load URLs if the user is typing a command or there's no capability
const hasComposerBrowseCapability = useBrowseCapability().inComposer;
const enableLoadURLsInComposer = hasComposerBrowseCapability && !composeText.startsWith('/');
@@ -158,10 +162,10 @@ export function Composer(props: {
/* items */ attachmentDrafts,
/* append */ attachAppendClipboardItems, attachAppendDataTransfer, attachAppendEgoFragments, attachAppendFile,
/* take */ attachmentsRemoveAll, attachmentsTakeAllFragments, attachmentsTakeFragmentsByType,
} = useAttachmentDrafts(conversationOverlayStore, enableLoadURLsInComposer);
} = useAttachmentDrafts(conversationOverlayStore, enableLoadURLsInComposer, chatLLMSupportsImages);
// attachments derived state
const llmAttachmentDraftsCollection = useLLMAttachmentDrafts(attachmentDrafts, props.chatLLM);
const llmAttachmentDraftsCollection = useLLMAttachmentDrafts(attachmentDrafts, props.chatLLM, chatLLMSupportsImages);
// drag/drop
const { dragContainerSx, dropComponent, handleContainerDragEnter, handleContainerDragStart } = useComposerDragDrop(!props.isMobile, attachAppendDataTransfer);
@@ -176,8 +180,7 @@ export function Composer(props: {
const isMobile = props.isMobile;
const isDesktop = !props.isMobile;
const noConversation = !targetConversationId;
const noLLM = !props.chatLLM;
const showLLMAttachments = chatExecuteModeCanAttach(chatExecuteMode);
const showChatAttachments = chatExecuteModeCanAttach(chatExecuteMode);
// tokens derived state
@@ -286,7 +289,7 @@ export function Composer(props: {
if (enqueued)
handleClear();
return enqueued;
}, [attachmentsTakeAllFragments, handleClear, inReferenceTo, onAction, targetConversationId]);
}, [attachmentsTakeAllFragments, confirmProceedIfAttachmentsNotSupported, handleClear, inReferenceTo, onAction, targetConversationId]);
const handleAppendTextAndSend = React.useCallback(async (appendText: string) => {
@@ -539,7 +542,7 @@ export function Composer(props: {
useGlobalShortcuts('ChatComposer', React.useMemo(() => {
const composerShortcuts: ShortcutObject[] = [];
if (showLLMAttachments) {
if (showChatAttachments) {
composerShortcuts.push({ key: 'f', ctrl: true, shift: true, action: () => openFileForAttaching(true, handleAttachFiles), description: 'Attach File' });
if (supportsClipboardRead)
composerShortcuts.push({ key: 'v', ctrl: true, shift: true, action: attachAppendClipboardItems, description: 'Attach Clipboard' });
@@ -561,7 +564,7 @@ export function Composer(props: {
}, description: 'Microphone',
});
return composerShortcuts;
}, [attachAppendClipboardItems, handleAttachFiles, recognitionState.hasSpeech, recognitionState.isActive, showLLMAttachments, toggleRecognition]));
}, [attachAppendClipboardItems, handleAttachFiles, recognitionState.hasSpeech, recognitionState.isActive, showChatAttachments, toggleRecognition]));
// ...
@@ -659,10 +662,10 @@ export function Composer(props: {
{recognitionState.isAvailable && <ButtonMicMemo variant={micVariant} color={micColor} errorMessage={recognitionState.errorMessage} onClick={handleToggleMic} />}
{/* Responsive Camera OCR button */}
{showLLMAttachments && <ButtonAttachCameraMemo isMobile onOpenCamera={openCamera} />}
{showChatAttachments && <ButtonAttachCameraMemo isMobile onOpenCamera={openCamera} />}
{/* [mobile] [+] button */}
{showLLMAttachments && (
{showChatAttachments && (
<Dropdown>
<MenuButton slots={{ root: IconButton }}>
<AddCircleOutlineIcon />
@@ -690,7 +693,7 @@ export function Composer(props: {
)}
{/* [Desktop, Col1] Insert Multi-modal content buttons */}
{isDesktop && showLLMAttachments && (
{isDesktop && showChatAttachments && (
<Box sx={{ flexGrow: 0, display: 'grid', gap: 1 }}>
{/*<FormHelperText sx={{ mx: 'auto' }}>*/}
@@ -847,7 +850,7 @@ export function Composer(props: {
</Box>
{/* Render any Attachments & menu items */}
{!!conversationOverlayStore && showLLMAttachments && (
{!!conversationOverlayStore && showChatAttachments && (
<LLMAttachmentsList
agiAttachmentPrompts={agiAttachmentPrompts}
attachmentDraftsStoreApi={conversationOverlayStore}
@@ -1,8 +1,8 @@
import * as React from 'react';
import type { AttachmentDraft } from '~/common/attachment-drafts/attachment.types';
import type { DLLM } from '~/common/stores/llms/llms.types';
import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.fragments';
import { DLLM, LLM_IF_OAI_Vision } from '~/common/stores/llms/llms.types';
import { estimateTokensForFragments } from '~/common/stores/chat/chat.tokens';
@@ -22,7 +22,7 @@ export interface LLMAttachmentDraft {
}
export function useLLMAttachmentDrafts(attachmentDrafts: AttachmentDraft[], chatLLM: DLLM | null): LLMAttachmentDraftsCollection {
export function useLLMAttachmentDrafts(attachmentDrafts: AttachmentDraft[], chatLLM: DLLM | null, chatLLMSupportsImages: boolean): LLMAttachmentDraftsCollection {
/* [Optimization] Use a Ref to store the previous state of llmAttachmentDrafts and chatLLM
*
@@ -44,8 +44,7 @@ export function useLLMAttachmentDrafts(attachmentDrafts: AttachmentDraft[], chat
const equalChatLLM = chatLLM === prevStateRef.current.chatLLM;
// LLM-dependent multi-modal enablement
const supportsImages = !!chatLLM?.interfaces?.includes(LLM_IF_OAI_Vision);
const supportedTypes: DMessageAttachmentFragment['part']['pt'][] = supportsImages ? ['image_ref', 'doc'] : ['doc'];
const supportedTypes: DMessageAttachmentFragment['part']['pt'][] = chatLLMSupportsImages ? ['image_ref', 'doc'] : ['doc'];
const supportedTextTypes: DMessageAttachmentFragment['part']['pt'][] = supportedTypes.filter(pt => pt === 'doc');
// Add LLM-specific properties to each attachment draft
@@ -87,5 +86,5 @@ export function useLLMAttachmentDrafts(attachmentDrafts: AttachmentDraft[], chat
llmTokenCountApprox,
};
}, [attachmentDrafts, chatLLM]); // Dependencies for the outer useMemo
}, [attachmentDrafts, chatLLM, chatLLMSupportsImages]); // Dependencies for the outer useMemo
}
@@ -12,7 +12,7 @@ import { pdfToImageDataURLs, pdfToText } from '~/common/util/pdfUtils';
import { createDMessageDataInlineText, createDocAttachmentFragment, DMessageAttachmentFragment, DMessageDataInline, DMessageDocPart, DVMimeType, isContentOrAttachmentFragment, isDocPart, specialContentPartToDocAttachmentFragment } from '~/common/stores/chat/chat.fragments';
import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftInput, AttachmentDraftSource, AttachmentDraftSourceOriginFile, DraftEgoFragmentsInputData, DraftWebInputData, DraftYouTubeInputData } from './attachment.types';
import type { AttachmentCreationOptions, AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftInput, AttachmentDraftSource, AttachmentDraftSourceOriginFile, DraftEgoFragmentsInputData, DraftWebInputData, DraftYouTubeInputData } from './attachment.types';
import type { AttachmentsDraftsStore } from './store-attachment-drafts-slice';
import { attachmentGetLiveFileId, attachmentSourceSupportsLiveFile } from './attachment.livefile';
import { guessInputContentTypeFromMime, heuristicMimeTypeFixup, mimeTypeIsDocX, mimeTypeIsPDF, mimeTypeIsPlainText, mimeTypeIsSupportedImage, reverseLookupMimeType } from './attachment.mimetypes';
@@ -24,7 +24,7 @@ export const DEFAULT_ADRAFT_IMAGE_MIMETYPE = !Is.Browser.Safari ? 'image/webp' :
export const DEFAULT_ADRAFT_IMAGE_QUALITY = 0.96;
const PDF_IMAGE_PAGE_SCALE = 1.5;
const PDF_IMAGE_QUALITY = 0.5;
const PDF_PREFER_TEXT_AND_IMAGES = false;
const ENABLE_TEXT_AND_IMAGES = false; // 2.0
// internal mimes, only used to route data within us (source -> input -> converters)
@@ -209,13 +209,16 @@ export async function attachmentLoadInputAsync(source: Readonly<AttachmentDraftS
*
* @param {Readonly<AttachmentDraftSource>} source - The source of the AttachmentDraft object.
* @param {Readonly<AttachmentDraftInput>} input - The input of the AttachmentDraft object.
* @param options conversion preferences, if any
* @param {(changes: Partial<AttachmentDraft>) => void} edit - A function to edit the AttachmentDraft object.
*/
export function attachmentDefineConverters(source: AttachmentDraftSource, input: Readonly<AttachmentDraftInput>, edit: (changes: Partial<Omit<AttachmentDraft, 'outputFragments'>>) => void) {
export function attachmentDefineConverters(source: AttachmentDraftSource, input: Readonly<AttachmentDraftInput>, options: AttachmentCreationOptions, edit: (changes: Partial<Omit<AttachmentDraft, 'outputFragments'>>) => void) {
// return all the possible converters for the input
const converters: AttachmentDraftConverter[] = [];
const autoAddImages = ENABLE_TEXT_AND_IMAGES && !!options?.hintAddImages;
switch (true) {
// plain text types
@@ -240,20 +243,20 @@ export function attachmentDefineConverters(source: AttachmentDraftSource, input:
// Images (Known/Unknown)
case input.mimeType.startsWith('image/'):
const imageSupported = mimeTypeIsSupportedImage(input.mimeType);
converters.push({ id: 'image-resized-high', name: 'Image (high detail)', disabled: !imageSupported });
converters.push({ id: 'image-resized-low', name: 'Image (low detail)', disabled: !imageSupported });
converters.push({ id: 'image-original', name: 'Image (original quality)', disabled: !imageSupported });
if (!imageSupported)
const inputImageMimeSupported = mimeTypeIsSupportedImage(input.mimeType);
converters.push({ id: 'image-resized-high', name: 'Image (high detail)', disabled: !inputImageMimeSupported });
converters.push({ id: 'image-resized-low', name: 'Image (low detail)', disabled: !inputImageMimeSupported });
converters.push({ id: 'image-original', name: 'Image (original quality)', disabled: !inputImageMimeSupported });
if (!inputImageMimeSupported)
converters.push({ id: 'image-to-default', name: `As Image (${DEFAULT_ADRAFT_IMAGE_MIMETYPE})` });
converters.push({ id: 'image-ocr', name: 'As Text (OCR)' });
break;
// PDF
case mimeTypeIsPDF(input.mimeType):
converters.push({ id: 'pdf-text', name: 'PDF To Text', isActive: !PDF_PREFER_TEXT_AND_IMAGES || undefined });
converters.push({ id: 'pdf-text', name: 'PDF To Text', isActive: !autoAddImages || undefined });
converters.push({ id: 'pdf-images', name: 'PDF To Images' });
converters.push({ id: 'pdf-text-and-images', name: 'PDF Text & Images (best)', isActive: PDF_PREFER_TEXT_AND_IMAGES || undefined });
converters.push({ id: 'pdf-text-and-images', name: 'PDF Text & Images (best)', isActive: autoAddImages });
break;
// DOCX
@@ -274,16 +277,16 @@ export function attachmentDefineConverters(source: AttachmentDraftSource, input:
if (input.urlImage) {
if (converters.length)
converters.push({ id: 'url-page-null', name: 'Do not attach' });
converters.push({ id: 'url-page-image', name: 'Add Screenshot', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true });
converters.push({ id: 'url-page-image', name: 'Add Screenshot', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true, isActive: autoAddImages || undefined });
}
break;
// YouTube: custom converters
case input.mimeType === INT_MIME_VND_AGI_YOUTUBE:
converters.push({ id: 'youtube-transcript', name: 'Video Transcript' });
converters.push({ id: 'youtube-transcript', name: 'Video Transcript', isActive: true });
converters.push({ id: 'youtube-transcript-simple', name: 'Video Transcript (simple)' });
if (input.urlImage)
converters.push({ id: 'url-page-image', name: 'Add Thumbnail', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true });
converters.push({ id: 'url-page-image', name: 'Add Thumbnail', disabled: !input.urlImage.width || !input.urlImage.height, isCheckbox: true, isActive: autoAddImages });
break;
// EGO
@@ -778,7 +781,7 @@ function _inputDataToString(data: AttachmentDraftInput['data']): string {
*
* Only returns the fragments that were successfully converted.
*/
export async function convertFilesToDAttachmentFragments(origin: AttachmentDraftSourceOriginFile, files: FileWithHandle[]): Promise<DMessageAttachmentFragment[]> {
export async function convertFilesToDAttachmentFragments(origin: AttachmentDraftSourceOriginFile, files: FileWithHandle[], options: AttachmentCreationOptions): Promise<DMessageAttachmentFragment[]> {
const validOutputFragmentsList: DMessageAttachmentFragment[][] = [];
for (const fileWithHandle of files) {
@@ -801,7 +804,7 @@ export async function convertFilesToDAttachmentFragments(origin: AttachmentDraft
}
// 2. Define converters
attachmentDefineConverters(_draft.source, _draft.input, updateDraft);
attachmentDefineConverters(_draft.source, _draft.input, options, updateDraft);
if (!_draft.converters.length) {
console.warn(`No converters defined for file: ${fileWithHandle.name}`);
continue;
@@ -62,6 +62,10 @@ export type AttachmentDraftSourceOriginFile = 'camera' | 'screencapture' | 'file
export type AttachmentDraftSourceOriginDTO = 'drop' | 'paste';
export type AttachmentCreationOptions = {
hintAddImages?: boolean;
}
// 1. draft input (loaded from the source)
@@ -5,7 +5,7 @@ import type { DBlobDBContextId, DBlobDBScopeId } from '~/modules/dblobs/dblobs.t
import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.fragments';
import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftSource } from './attachment.types';
import type { AttachmentCreationOptions, AttachmentDraft, AttachmentDraftConverter, AttachmentDraftId, AttachmentDraftSource } from './attachment.types';
import { attachmentCreate, attachmentDefineConverters, attachmentLoadInputAsync, attachmentPerformConversion } from './attachment.pipeline';
import { removeAttachmentOwnedDBAsset, transferAttachmentOwnedDBAsset } from './attachment.dblobs';
@@ -20,7 +20,7 @@ interface AttachmentDraftsState {
export interface AttachmentsDraftsStore extends AttachmentDraftsState {
createAttachmentDraft: (source: AttachmentDraftSource) => Promise<void>;
createAttachmentDraft: (source: AttachmentDraftSource, options: AttachmentCreationOptions) => Promise<void>;
removeAllAttachmentDrafts: () => void;
removeAttachmentDraft: (attachmentDraftId: AttachmentDraftId) => void;
moveAttachmentDraft: (attachmentDraftId: AttachmentDraftId, delta: 1 | -1) => void;
@@ -52,7 +52,7 @@ export const createAttachmentDraftsStoreSlice: StateCreator<AttachmentsDraftsSto
attachmentDrafts: [],
// actions
createAttachmentDraft: async (source: AttachmentDraftSource) => {
createAttachmentDraft: async (source: AttachmentDraftSource, options: AttachmentCreationOptions) => {
const { _getAttachment, _editAttachment, toggleAttachmentDraftConverterAndConvert } = _get();
const _attachmentDraft = attachmentCreate(source);
@@ -70,7 +70,7 @@ export const createAttachmentDraftsStoreSlice: StateCreator<AttachmentsDraftsSto
return;
// 2. Define the I->O Converters
attachmentDefineConverters(source, loaded.input, editFn);
attachmentDefineConverters(source, loaded.input, options, editFn);
const defined = _getAttachment(attachmentDraftId);
if (!defined?.converters.length)
return;
@@ -20,7 +20,7 @@ import type { AttachmentDraftsStoreApi } from './store-attachment-drafts-slice';
const ATTACHMENTS_DEBUG_INTAKE = false;
export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreApi | null, enableLoadURLs: boolean) => {
export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreApi | null, enableLoadURLs: boolean, hintAddImages: boolean) => {
// state
const { _createAttachmentDraft, attachmentDrafts, attachmentsRemoveAll, attachmentsTakeAllFragments, attachmentsTakeFragmentsByType } = useChatAttachmentsStore(attachmentsStoreApi, useShallow(state => ({
@@ -43,8 +43,8 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
return _createAttachmentDraft({
media: 'file', origin, fileWithHandle, refPath: overrideFileName || fileWithHandle.name,
});
}, [_createAttachmentDraft]);
}, { hintAddImages });
}, [_createAttachmentDraft, hintAddImages]);
/**
* Append data transfer to the attachments.
@@ -145,7 +145,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
if (textPlainUrl && textPlainUrl.trim()) {
void _createAttachmentDraft({
media: 'url', url: textPlainUrl, refUrl: textPlain,
});
}, { hintAddImages});
return 'as_url';
}
@@ -155,7 +155,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
if (attachText && (textHtml || textPlain)) {
void _createAttachmentDraft({
media: 'text', method, textPlain, textHtml,
});
}, { hintAddImages });
return 'as_text';
}
@@ -165,7 +165,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
// did not attach anything from this data transfer
return false;
}, [attachAppendFile, _createAttachmentDraft, enableLoadURLs]);
}, [_createAttachmentDraft, attachAppendFile, enableLoadURLs, hintAddImages]);
/**
* Append clipboard items to the attachments.
@@ -223,7 +223,7 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
if (textPlainUrl && textPlainUrl.trim()) {
void _createAttachmentDraft({
media: 'url', url: textPlainUrl.trim(), refUrl: textPlain,
});
}, { hintAddImages });
continue;
}
}
@@ -232,13 +232,13 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
if (textHtml || textPlain) {
void _createAttachmentDraft({
media: 'text', method: 'clipboard-read', textPlain, textHtml,
});
}, { hintAddImages });
continue;
}
console.warn('Clipboard item has no text/html or text/plain item.', clipboardItem.types, clipboardItem);
}
}, [attachAppendFile, _createAttachmentDraft, enableLoadURLs]);
}, [_createAttachmentDraft, attachAppendFile, enableLoadURLs, hintAddImages]);
/**
* Append ego content to the attachments.
@@ -257,8 +257,8 @@ export const useAttachmentDrafts = (attachmentsStoreApi: AttachmentDraftsStoreAp
conversationId,
messageId,
},
});
}, [_createAttachmentDraft]);
}, { hintAddImages });
}, [_createAttachmentDraft, hintAddImages]);
return {