diff --git a/src/apps/chat/components/attachments/AttachmentItem.tsx b/src/apps/chat/components/attachments/AttachmentItem.tsx index 454f14dbd..b7e689c96 100644 --- a/src/apps/chat/components/attachments/AttachmentItem.tsx +++ b/src/apps/chat/components/attachments/AttachmentItem.tsx @@ -1,8 +1,11 @@ import * as React from 'react'; import { Box, Button, CircularProgress, ColorPaletteProp, ListDivider, ListItem, ListItemDecorator, MenuItem, Radio, Sheet, Typography } from '@mui/joy'; +import AbcIcon from '@mui/icons-material/Abc'; import ClearIcon from '@mui/icons-material/Clear'; +import CloseIcon from '@mui/icons-material/Close'; import CodeIcon from '@mui/icons-material/Code'; +import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined'; import KeyboardArrowLeftIcon from '@mui/icons-material/KeyboardArrowLeft'; import KeyboardArrowRightIcon from '@mui/icons-material/KeyboardArrowRight'; import PivotTableChartIcon from '@mui/icons-material/PivotTableChart'; @@ -14,7 +17,7 @@ import { CloseableMenu } from '~/common/components/CloseableMenu'; import { GoodTooltip } from '~/common/components/GoodTooltip'; import { ellipsizeFront, ellipsizeMiddle } from '~/common/util/textUtils'; -import { Attachment, AttachmentConversion, useAttachmentsStore } from './store-attachments'; +import { Attachment, useAttachmentsStore } from './store-attachments'; // default attachment width @@ -62,20 +65,25 @@ const InputErrorIndicator = () => ; -function attachmentIcon(conversion: AttachmentConversion | null) { - const iconSx = { - width: 24, - height: 24, - }; - switch (conversion?.id) { +function attachmentIcon(attachment: Attachment) { + const conversion = attachment.conversionIdx !== null ? attachment.conversions[attachment.conversionIdx] ?? null : null; + if (!conversion) + return null; + const iconSx = { width: 24, height: 24 }; + switch (conversion.id) { case 'text': return ; case 'rich-text': return ; case 'rich-text-table': return ; - // case 'image': - // return {conversion.name}; + case 'image': + // return {conversion.name}; + return ; + case 'image-ocr': + return ; + case 'unhandled': + return ; default: return null; } @@ -149,13 +157,10 @@ export function AttachmentItem(props: { const isInputError = !!attachment.inputError; const hasInput = !!aInput; - const isUnsupported = aConversions.length === 0; + const isUnconverted = aConversions.length === 0; - const conversion = (aConversionIdx !== null ? aConversions[aConversionIdx] : null) || null; - - const hasOutputs = aOutputs ? aOutputs.length >= 1 : false; - - const areOutputsEjectable = hasOutputs && aOutputs?.every(output => output.isEjectable); + const isNoOutput = aOutputs?.length === 0; + // const areOutputsEjectable = hasOutputs && aOutputs?.every(output => output.isEjectable); let variant: 'soft' | 'outlined' | 'contained'; @@ -168,8 +173,8 @@ export function AttachmentItem(props: { tooltip += aLabel; if (hasInput) tooltip += `\n(${aInput.mimeType}: ${aInput.dataSize.toLocaleString()} bytes)`; - if (hasOutputs) - tooltip += `\n\n${JSON.stringify(aOutputs)}`; + // if (aOutputs && aOutputs.length >= 1) + // tooltip += `\n\n${JSON.stringify(aOutputs)}`; if (isInputLoading) { variant = 'soft'; @@ -178,10 +183,14 @@ export function AttachmentItem(props: { tooltip = `Issue loading the attachment: ${attachment.inputError}\n\n${tooltip}`; variant = 'soft'; color = 'danger'; - } else if (isUnsupported) { + } else if (isUnconverted) { tooltip = `Attachments of type '${aInput?.mimeType}' are not supported yet. You can open a feature request on GitHub.\n\n${tooltip}`; variant = 'soft'; color = 'warning'; + } else if (isNoOutput) { + tooltip = 'Not compatible with the selected LLM or not supported. Please select another format.\n\n' + tooltip; + variant = 'soft'; + color = 'warning'; } else { // all good tooltip = null; @@ -192,7 +201,12 @@ export function AttachmentItem(props: { return - + {isInputLoading ? : ( @@ -214,7 +228,7 @@ export function AttachmentItem(props: { {isInputError ? : <> - {attachmentIcon(conversion)} + {attachmentIcon(attachment)} {attachmentText(attachment)} @@ -253,12 +267,12 @@ export function AttachmentItem(props: { {!isUnmoveable && } {/* Render Conversions as menu items */} - {!isUnsupported && - - Attach as: - - } - {!isUnsupported && aConversions.map((conversion, idx) => + {/*{!isUnconverted && */} + {/* */} + {/* Attach as:*/} + {/* */} + {/*}*/} + {!isUnconverted && aConversions.map((conversion, idx) => - + {/**/} + {conversion.name} , )} - {!isUnsupported && } + {!isUnconverted && } {/* Destructive Operations */} diff --git a/src/apps/chat/components/attachments/logic.tsx b/src/apps/chat/components/attachments/logic.tsx index a9b6c9d4a..d9182a93f 100644 --- a/src/apps/chat/components/attachments/logic.tsx +++ b/src/apps/chat/components/attachments/logic.tsx @@ -120,11 +120,12 @@ export function attachmentDefineConversions(sourceType: AttachmentSource['media' // return all the possible conversions for the input const conversions: AttachmentConversion[] = []; - switch (input.mimeType) { + switch (true) { // plain text types - case 'text/csv': - case 'text/plain': + case input.mimeType === 'application/json': + case input.mimeType === 'text/csv': + case input.mimeType === 'text/plain': // handle a secondary layer of HTML 'text' origins (drop, paste, clipboard-read) const textOriginHtml = sourceType === 'text' && input.altMimeType === 'text/html' && !!input.altData; @@ -153,7 +154,13 @@ export function attachmentDefineConversions(sourceType: AttachmentSource['media' } break; - // catch-all for unsupported types + // images + case input.mimeType.startsWith('image/'): + conversions.push({ id: 'image', name: `Image (GPT Vision)` }); + conversions.push({ id: 'image-ocr', name: 'As OCR' }); + break; + + // catch-all default: conversions.push({ id: 'unhandled', name: `Unsupported ${input.mimeType}` }); conversions.push({ id: 'text', name: 'As Text' }); @@ -183,6 +190,7 @@ export function attachmentConvert(attachment: Readonly, conversionId // apply conversion to the input const outputs: AttachmentOutput[] = []; switch (conversion.id) { + // text as-is case 'text': outputs.push({ @@ -217,6 +225,14 @@ export function attachmentConvert(attachment: Readonly, conversionId }); break; + case 'image': + // TODO: extract base64 + break; + + case 'image-ocr': + // TODO: port + break; + case 'unhandled': // force the user to explicitly select 'as text' if they want to proceed break; diff --git a/src/apps/chat/components/attachments/store-attachments.tsx b/src/apps/chat/components/attachments/store-attachments.tsx index a2a8bf6f7..169d852ff 100644 --- a/src/apps/chat/components/attachments/store-attachments.tsx +++ b/src/apps/chat/components/attachments/store-attachments.tsx @@ -36,7 +36,7 @@ export type AttachmentInput = { }; export type AttachmentConversion = { - id: 'text' | 'rich-text' | 'rich-text-table' | 'unhandled'; + id: 'text' | 'rich-text' | 'rich-text-table' | 'image' | 'image-ocr' | 'unhandled'; name: string; // outputType: ConversionOutputType; // The type of the output after conversion // isAutonomous: boolean; // Whether the conversion does not require user input diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index 3be3a42bd..8d9aa1be9 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -365,7 +365,7 @@ export function Composer(props: { {/* Vertical (insert) buttons */} {isMobile ? ( - + {/* [mobile] Mic button */} {isSpeechEnabled && } @@ -381,7 +381,7 @@ export function Composer(props: { ) : ( - + {/**/} {/* Attach*/}