Attachments: open images from the menu in new tabs

src/apps/chat/components/composer/llmattachments/LLMAttachmentMenu.tsx

@@ -1,12 +1,16 @@
 import * as React from 'react';
 
-import { Box, ListDivider, ListItemDecorator, MenuItem, Radio, Typography } from '@mui/joy';
+import { Box, IconButton, Link, ListDivider, ListItem, ListItemDecorator, MenuItem, Radio, Tooltip, Typography } from '@mui/joy';
 import ClearIcon from '@mui/icons-material/Clear';
 import ContentCopyIcon from '@mui/icons-material/ContentCopy';
 import KeyboardArrowLeftIcon from '@mui/icons-material/KeyboardArrowLeft';
 import KeyboardArrowRightIcon from '@mui/icons-material/KeyboardArrowRight';
+import LaunchIcon from '@mui/icons-material/Launch';
 import VerticalAlignBottomIcon from '@mui/icons-material/VerticalAlignBottom';
 
+import { getImageBlobURLById } from '~/modules/dblobs/dblobs.db';
+
+import type { DContentRef } from '~/common/stores/chat/chat.message';
 import { CloseableMenu } from '~/common/components/CloseableMenu';
 import { copyToClipboard } from '~/common/util/clipboardUtils';
 
@@ -19,6 +23,22 @@ import type { LLMAttachment } from './useLLMAttachments';
 export const DEBUG_LLMATTACHMENTS = true;
 
 
+/**
+ * Note: this utility function could be extracted more broadly to chat.message.ts, but
+ * I don't want to introduce a (circular) dependency from chat.message.ts to dblobs.db.ts.
+ */
+async function handleShowContentInNewTab(source: DContentRef) {
+  console.log('handleShowContentInNewTab', source);
+  let imageUrl: string | null = null;
+  if (source.reftype === 'url')
+    imageUrl = source.url;
+  else if (source.reftype === 'dblob')
+    imageUrl = await getImageBlobURLById(source.dblobId);
+  if (imageUrl && typeof window !== 'undefined')
+    window.open(imageUrl, '_blank', 'noopener,noreferrer');
+}
+
+
 export function LLMAttachmentMenu(props: {
   attachmentDraftsStoreApi: AttachmentDraftsStoreApi,
   llmAttachment: LLMAttachment,
@@ -145,8 +165,16 @@ export function LLMAttachmentMenu(props: {
       {!isUnconvertible && <ListDivider />}
 
       {DEBUG_LLMATTACHMENTS && !!aInput && (
-        <MenuItem onClick={handleCopyOutputToClipboard} disabled={!isOutputTextInlineable}>
-          <ListItemDecorator><ContentCopyIcon /></ListItemDecorator>
+        <ListItem>
+          <ListItemDecorator>
+            {isOutputTextInlineable && (
+              <Tooltip title='Copy Text to clipboard'>
+                <IconButton size='sm' onClick={handleCopyOutputToClipboard} disabled={!isOutputTextInlineable} sx={{ ml: -0.5 }}>
+                  <ContentCopyIcon />
+                </IconButton>
+              </Tooltip>
+            )}
+          </ListItemDecorator>
           <Box>
             {!!aInput && <Typography level='body-xs'>
               🡐 {aInput.mimeType}, {aInput.dataSize.toLocaleString()} bytes
@@ -160,11 +188,15 @@ export function LLMAttachmentMenu(props: {
               ) : (
                 aOutputParts.map((output, index) => {
                   if (output.atype === 'aimage') {
-                    const resolution = output.width && output.height ? `${output.width}x${output.height}` : 'unknown resolution';
+                    const resolution = output.width && output.height ? `${output.width} x ${output.height}` : 'unknown resolution';
                     const mime = output.source.reftype === 'dblob' ? output.source.mimeType : 'unknown image';
                     return (
                       <Typography key={index} level='body-xs'>
-                        🡒 {mime}: {resolution}, {output.source.reftype === 'dblob' ? output.source.bytesSize?.toLocaleString() : '(remote)'} bytes
+                        🡒 {mime.replace('image/', 'img: ')}, {resolution}, {output.source.reftype === 'dblob' ? output.source.bytesSize?.toLocaleString() : '(remote)'} bytes,
+                        {' '}
+                        <Link onClick={() => handleShowContentInNewTab(output.source)}>
+                          show <LaunchIcon sx={{ mx: 0.5, fontSize: 16 }} />
+                        </Link>
                       </Typography>
                     );
                   } else if (output.atype === 'atext') {
@@ -189,7 +221,7 @@ export function LLMAttachmentMenu(props: {
               )}
             </Box>
           </Box>
-        </MenuItem>
+        </ListItem>
       )}
       {DEBUG_LLMATTACHMENTS && !!aInput && <ListDivider />}
 

src/common/attachment-drafts/attachment.pipeline.ts

@@ -10,6 +10,13 @@ import type { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftInput, A
 import { attachmentImageToPartViaDBlob } from './attachment.dblobs';
 
 
+// configuration
+export const DEFAULT_ADRAFT_IMAGE_MIMETYPE = 'image/webp';
+export const DEFAULT_ADRAFT_IMAGE_QUALITY = 0.98;
+const PDF_IMAGE_PAGE_SCALE = 1.5;
+const PDF_IMAGE_QUALITY = 0.5;
+
+
 // extensions to treat as plain text
 const PLAIN_TEXT_EXTENSIONS: string[] = ['.ts', '.tsx'];
 
@@ -70,9 +77,6 @@ const IMAGE_MIMETYPES: string[] = [
   'image/gif',
 ];
 
-export const DEFAULT_ADRAFT_IMAGE_MIMETYPE = 'image/webp';
-export const DEFAULT_ADRAFT_IMAGE_QUALITY = 0.98;
-
 
 /**
  * Creates a new AttachmentDraft object.
@@ -430,7 +434,7 @@ export async function attachmentPerformConversion(attachment: Readonly<Attachmen
       // duplicate the ArrayBuffer to avoid mutation
       const pdfData2 = new Uint8Array(input.data.slice(0));
       try {
-        const imageDataURLs = await pdfToImageDataURLs(pdfData2, DEFAULT_ADRAFT_IMAGE_MIMETYPE, DEFAULT_ADRAFT_IMAGE_QUALITY);
+        const imageDataURLs = await pdfToImageDataURLs(pdfData2, DEFAULT_ADRAFT_IMAGE_MIMETYPE, PDF_IMAGE_QUALITY, PDF_IMAGE_PAGE_SCALE);
         for (const pdfPageImage of imageDataURLs) {
           const pdfPageImagePart = await attachmentImageToPartViaDBlob(pdfPageImage.mimeType, pdfPageImage.base64Data, source, ref, `Page ${outputParts.length + 1}`, false, false);
           if (pdfPageImagePart)

src/common/stores/chat/chat.message.ts

@@ -40,7 +40,7 @@ export type DMessageRole = 'user' | 'assistant' | 'system';
 
 // Content Reference - we use a Ref and the DBlob framework to store media locally, or remote URLs
 
-type DContentRef =
+export type DContentRef =
   | { reftype: 'url'; url: string } // remotely accessible URL
   | { reftype: 'dblob'; dblobId: DBlobId, mimeType: string; bytesSize: number; } // reference to a DBlob
   ;

src/common/util/pdfUtils.ts

@@ -64,7 +64,7 @@ interface PdfPageImage {
  * @param imageQuality The quality of the image (default 0.95 for moderate quality)
  * @param scale The scale factor for the image resolution (default 1.5 for moderate quality)
  */
-export async function pdfToImageDataURLs(pdfBuffer: ArrayBuffer, imageMimeType: string, imageQuality: number /* = 0.95 */, scale = 1.5): Promise<PdfPageImage[]> {
+export async function pdfToImageDataURLs(pdfBuffer: ArrayBuffer, imageMimeType: string, imageQuality: number /* = 0.95 */, scale: number /*= 1.5*/): Promise<PdfPageImage[]> {
   const { getDocument } = await dynamicImportPdfJs();
   const pdf = await getDocument({ data: pdfBuffer }).promise;
   const images: PdfPageImage[] = [];

src/modules/dblobs/dblobs.db.ts

@@ -48,6 +48,31 @@ export async function deleteDBlobItem(id: string) {
 }
 
 
+// Specific item types
+async function getImageItemById(id: string) {
+  return await getItemById<DBlobImageItem>(id);
+}
+
+export async function getImageDataURLById(id: string) {
+  const item = await getImageItemById(id);
+  return item ? `data:${item.data.mimeType};base64,${item.data.base64}` : null;
+}
+
+export async function getImageBlobURLById(id: string) {
+  const item = await getImageItemById(id);
+  if (item) {
+    const byteCharacters = atob(item.data.base64);
+    const byteNumbers = new Array(byteCharacters.length);
+    for (let i = 0; i < byteCharacters.length; i++) {
+      byteNumbers[i] = byteCharacters.charCodeAt(i);
+    }
+    const byteArray = new Uint8Array(byteNumbers);
+    const blob = new Blob([byteArray], { type: item.data.mimeType });
+    return URL.createObjectURL(blob);
+  }
+  return null;
+}
+
 // Example usage:
 async function getAllImages(): Promise<DBlobImageItem[]> {
   return await getDBlobItemsByType<DBlobImageItem>(DBlobMetaDataType.IMAGE);