Attachment: initial image support

src/apps/chat/components/attachments/AttachmentItem.tsx

@@ -1,8 +1,11 @@
 import * as React from 'react';
 
 import { Box, Button, CircularProgress, ColorPaletteProp, ListDivider, ListItem, ListItemDecorator, MenuItem, Radio, Sheet, Typography } from '@mui/joy';
+import AbcIcon from '@mui/icons-material/Abc';
 import ClearIcon from '@mui/icons-material/Clear';
+import CloseIcon from '@mui/icons-material/Close';
 import CodeIcon from '@mui/icons-material/Code';
+import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined';
 import KeyboardArrowLeftIcon from '@mui/icons-material/KeyboardArrowLeft';
 import KeyboardArrowRightIcon from '@mui/icons-material/KeyboardArrowRight';
 import PivotTableChartIcon from '@mui/icons-material/PivotTableChart';
@@ -14,7 +17,7 @@ import { CloseableMenu } from '~/common/components/CloseableMenu';
 import { GoodTooltip } from '~/common/components/GoodTooltip';
 import { ellipsizeFront, ellipsizeMiddle } from '~/common/util/textUtils';
 
-import { Attachment, AttachmentConversion, useAttachmentsStore } from './store-attachments';
+import { Attachment, useAttachmentsStore } from './store-attachments';
 
 
 // default attachment width
@@ -62,20 +65,25 @@ const InputErrorIndicator = () =>
   <WarningRoundedIcon sx={{ color: 'danger.solidBg' }} />;
 
 
-function attachmentIcon(conversion: AttachmentConversion | null) {
-  const iconSx = {
-    width: 24,
-    height: 24,
-  };
-  switch (conversion?.id) {
+function attachmentIcon(attachment: Attachment) {
+  const conversion = attachment.conversionIdx !== null ? attachment.conversions[attachment.conversionIdx] ?? null : null;
+  if (!conversion)
+    return null;
+  const iconSx = { width: 24, height: 24 };
+  switch (conversion.id) {
     case 'text':
       return <TextFieldsIcon sx={iconSx} />;
     case 'rich-text':
       return <CodeIcon sx={iconSx} />;
     case 'rich-text-table':
       return <PivotTableChartIcon sx={iconSx} />;
-    // case 'image':
-    //   return <img src={conversion.url} alt={conversion.name} style={{ maxHeight: '100%', maxWidth: '100%' }} />;
+    case 'image':
+      //   return <img src={conversion.url} alt={conversion.name} style={{ maxHeight: '100%', maxWidth: '100%' }} />;
+      return <ImageOutlinedIcon sx={iconSx} />;
+    case 'image-ocr':
+      return <AbcIcon sx={iconSx} />;
+    case 'unhandled':
+      return <CloseIcon sx={iconSx} color='warning' />;
     default:
       return null;
   }
@@ -149,13 +157,10 @@ export function AttachmentItem(props: {
   const isInputError = !!attachment.inputError;
   const hasInput = !!aInput;
 
-  const isUnsupported = aConversions.length === 0;
+  const isUnconverted = aConversions.length === 0;
 
-  const conversion = (aConversionIdx !== null ? aConversions[aConversionIdx] : null) || null;
-
-  const hasOutputs = aOutputs ? aOutputs.length >= 1 : false;
-
-  const areOutputsEjectable = hasOutputs && aOutputs?.every(output => output.isEjectable);
+  const isNoOutput = aOutputs?.length === 0;
+  // const areOutputsEjectable = hasOutputs && aOutputs?.every(output => output.isEjectable);
 
 
   let variant: 'soft' | 'outlined' | 'contained';
@@ -168,8 +173,8 @@ export function AttachmentItem(props: {
   tooltip += aLabel;
   if (hasInput)
     tooltip += `\n(${aInput.mimeType}: ${aInput.dataSize.toLocaleString()} bytes)`;
-  if (hasOutputs)
-    tooltip += `\n\n${JSON.stringify(aOutputs)}`;
+  // if (aOutputs && aOutputs.length >= 1)
+  //   tooltip += `\n\n${JSON.stringify(aOutputs)}`;
 
   if (isInputLoading) {
     variant = 'soft';
@@ -178,10 +183,14 @@ export function AttachmentItem(props: {
     tooltip = `Issue loading the attachment: ${attachment.inputError}\n\n${tooltip}`;
     variant = 'soft';
     color = 'danger';
-  } else if (isUnsupported) {
+  } else if (isUnconverted) {
     tooltip = `Attachments of type '${aInput?.mimeType}' are not supported yet. You can open a feature request on GitHub.\n\n${tooltip}`;
     variant = 'soft';
     color = 'warning';
+  } else if (isNoOutput) {
+    tooltip = 'Not compatible with the selected LLM or not supported. Please select another format.\n\n' + tooltip;
+    variant = 'soft';
+    color = 'warning';
   } else {
     // all good
     tooltip = null;
@@ -192,7 +201,12 @@ export function AttachmentItem(props: {
 
   return <Box>
 
-    <GoodTooltip title={tooltip} isError={isInputError} isWarning={isUnsupported} sx={{ p: 1, whiteSpace: 'break-spaces' }}>
+    <GoodTooltip
+      title={tooltip}
+      isError={isInputError}
+      isWarning={isUnconverted || isNoOutput}
+      sx={{ p: 1, whiteSpace: 'break-spaces' }}
+    >
       {isInputLoading
         ? <LoadingIndicator label={aLabel} />
         : (
@@ -214,7 +228,7 @@ export function AttachmentItem(props: {
             {isInputError
               ? <InputErrorIndicator />
               : <>
-                {attachmentIcon(conversion)}
+                {attachmentIcon(attachment)}
                 <Typography level='title-sm' sx={{ whiteSpace: 'nowrap' }}>
                   {attachmentText(attachment)}
                 </Typography>
@@ -253,12 +267,12 @@ export function AttachmentItem(props: {
         {!isUnmoveable && <ListDivider sx={{ mt: 0 }} />}
 
         {/* Render Conversions as menu items */}
-        {!isUnsupported && <ListItem>
-          <Typography level='body-md'>
-            Attach as:
-          </Typography>
-        </ListItem>}
-        {!isUnsupported && aConversions.map((conversion, idx) =>
+        {/*{!isUnconverted && <ListItem>*/}
+        {/*  <Typography level='body-md'>*/}
+        {/*    Attach as:*/}
+        {/*  </Typography>*/}
+        {/*</ListItem>}*/}
+        {!isUnconverted && aConversions.map((conversion, idx) =>
           <MenuItem
             // disabled={aConversions.length === 1}
             key={'c-' + conversion.id}
@@ -267,13 +281,14 @@ export function AttachmentItem(props: {
             <ListItemDecorator>
               <Radio checked={idx === aConversionIdx} />
             </ListItemDecorator>
-            <Typography level={idx === aConversionIdx ? 'title-md' : 'body-md'}>
+            {/*<Typography level={idx === aConversionIdx ? 'title-md' : 'body-md'}>*/}
+            <Typography>
               {conversion.name}
             </Typography>
           </MenuItem>,
         )}
 
-        {!isUnsupported && <ListDivider />}
+        {!isUnconverted && <ListDivider />}
 
         {/* Destructive Operations */}
         <MenuItem onClick={handleInline}>

src/apps/chat/components/attachments/logic.tsx

@@ -120,11 +120,12 @@ export function attachmentDefineConversions(sourceType: AttachmentSource['media'
   // return all the possible conversions for the input
   const conversions: AttachmentConversion[] = [];
 
-  switch (input.mimeType) {
+  switch (true) {
 
     // plain text types
-    case 'text/csv':
-    case 'text/plain':
+    case input.mimeType === 'application/json':
+    case input.mimeType === 'text/csv':
+    case input.mimeType === 'text/plain':
 
       // handle a secondary layer of HTML 'text' origins (drop, paste, clipboard-read)
       const textOriginHtml = sourceType === 'text' && input.altMimeType === 'text/html' && !!input.altData;
@@ -153,7 +154,13 @@ export function attachmentDefineConversions(sourceType: AttachmentSource['media'
       }
       break;
 
-    // catch-all for unsupported types
+    // images
+    case input.mimeType.startsWith('image/'):
+      conversions.push({ id: 'image', name: `Image (GPT Vision)` });
+      conversions.push({ id: 'image-ocr', name: 'As OCR' });
+      break;
+
+    // catch-all
     default:
       conversions.push({ id: 'unhandled', name: `Unsupported ${input.mimeType}` });
       conversions.push({ id: 'text', name: 'As Text' });
@@ -183,6 +190,7 @@ export function attachmentConvert(attachment: Readonly<Attachment>, conversionId
   // apply conversion to the input
   const outputs: AttachmentOutput[] = [];
   switch (conversion.id) {
+
     // text as-is
     case 'text':
       outputs.push({
@@ -217,6 +225,14 @@ export function attachmentConvert(attachment: Readonly<Attachment>, conversionId
       });
       break;
 
+    case 'image':
+      // TODO: extract base64
+      break;
+
+    case 'image-ocr':
+      // TODO: port
+      break;
+
     case 'unhandled':
       // force the user to explicitly select 'as text' if they want to proceed
       break;

src/apps/chat/components/attachments/store-attachments.tsx

@@ -36,7 +36,7 @@ export type AttachmentInput = {
 };
 
 export type AttachmentConversion = {
-  id: 'text' | 'rich-text' | 'rich-text-table' | 'unhandled';
+  id: 'text' | 'rich-text' | 'rich-text-table' | 'image' | 'image-ocr' | 'unhandled';
   name: string;
   // outputType: ConversionOutputType; // The type of the output after conversion
   // isAutonomous: boolean; // Whether the conversion does not require user input

src/apps/chat/components/composer/Composer.tsx

@@ -365,7 +365,7 @@ export function Composer(props: {
 
           {/* Vertical (insert) buttons */}
           {isMobile ? (
-            <Box sx={{ display: 'flex', flexDirection: 'column', gap: { xs: 0, md: 2 } }}>
+            <Box sx={{ display: 'flex', flexDirection: 'column', gap: { md: 1 } }}>
 
               {/* [mobile] Mic button */}
               {isSpeechEnabled && <MicButton variant={micVariant} color={micColor} onClick={handleToggleMic} />}
@@ -381,7 +381,7 @@ export function Composer(props: {
 
             </Box>
           ) : (
-            <Box sx={{ display: 'flex', flexDirection: 'column', gap: { xs: 0, md: 2 } }}>
+            <Box sx={{ display: 'flex', flexDirection: 'column', gap: { md: 1 } }}>
 
               {/*<FormHelperText sx={{ mx: 'auto' }}>*/}
               {/*  Attach*/}