LiveFile: AttachmentFragments

src/apps/chat/components/composer/llmattachments/LLMAttachmentButton.tsx

@@ -19,7 +19,9 @@ import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
 import { RenderImageURL } from '~/modules/blocks/image/RenderImageURL';
 
 import { GoodTooltip } from '~/common/components/GoodTooltip';
+import { LiveFileIcon } from '~/common/livefile/LiveFileIcon';
 import { ellipsizeFront, ellipsizeMiddle } from '~/common/util/textUtils';
+import { liveFileInAttachmentFragment } from '~/common/livefile/liveFile';
 
 import type { AttachmentDraft, AttachmentDraftConverterType, AttachmentDraftId } from '~/common/attachment-drafts/attachment.types';
 import type { LLMAttachmentDraft } from './useLLMAttachmentDrafts';
@@ -95,7 +97,7 @@ const converterTypeToIconMap: { [key in AttachmentDraftConverterType]: React.Com
   'unhandled': TextureIcon,
 };
 
-function attachmentIcon(attachmentDraft: AttachmentDraft): React.ReactNode {
+function attachmentIcons(attachmentDraft: AttachmentDraft): React.ReactNode {
   const activeConterters = attachmentDraft.converters.filter(c => c.isActive);
   if (activeConterters.length === 0)
     return null;
@@ -115,7 +117,7 @@ function attachmentIcon(attachmentDraft: AttachmentDraft): React.ReactNode {
   return <Typography sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
 
     {/* If we have a Web preview, show it first */}
-    {/*!imageDataRefs.length &&*/ !!attachmentDraft.input?.urlImage?.webpDataUrl && (
+    {!!attachmentDraft.input?.urlImage?.webpDataUrl && /*!imageDataRefs.length &&*/ (
       <Tooltip title={<>This was the page.<br />You can also Add the Screenshot as attachment</>}>
         <RenderImageURL
           imageURL={attachmentDraft.input.urlImage.webpDataUrl}
@@ -172,6 +174,7 @@ export function LLMAttachmentButton(props: {
   const isUnconvertible = !draft.converters.length;
   const isOutputLoading = draft.outputsConverting;
   const isOutputMissing = !draft.outputFragments.length;
+  const hasLiveFile = draft.outputFragments.some(liveFileInAttachmentFragment);
 
   const showWarning = isUnconvertible || (isOutputMissing || !llmSupportsAllFragments);
 
@@ -232,7 +235,8 @@ export function LLMAttachmentButton(props: {
         : (
           <Button
             size='sm'
-            variant={variant} color={color}
+            color={color}
+            variant={variant}
             onClick={handleToggleMenu}
             onContextMenu={handleToggleMenu}
             sx={{
@@ -248,12 +252,19 @@ export function LLMAttachmentButton(props: {
             {isInputError
               ? <InputErrorIndicator />
               : <>
-                {attachmentIcon(draft)}
-                {isOutputLoading
-                  ? <>Converting <CircularProgress color='success' size='sm' /></>
-                  : <Typography level='title-sm' sx={{ whiteSpace: 'nowrap' }}>
-                    {attachmentLabelText(draft)}
-                  </Typography>}
+                {/* Icons: Web Page Screenshot, Converter[s] */}
+                {attachmentIcons(draft)}
+
+                {/* Label */}
+                <Typography level='title-sm' sx={{ whiteSpace: 'nowrap' }}>
+                  {isOutputLoading ? 'Converting... ' : attachmentLabelText(draft)}
+                </Typography>
+
+                {/* Loading icon */}
+                {isOutputLoading && <CircularProgress color='success' size='sm' />}
+
+                {/* Live file icon */}
+                {hasLiveFile && <LiveFileIcon color='primary' sx={{ width: 20, height: 20 }} />}
               </>}
           </Button>
         )}

src/apps/chat/components/composer/llmattachments/LLMAttachmentMenu.tsx

@@ -14,7 +14,9 @@ import { showImageDataRefInNewTab } from '~/modules/blocks/image/RenderImageRefD
 
 import { CloseableMenu } from '~/common/components/CloseableMenu';
 import { DMessageAttachmentFragment, isDocPart, isImageRefPart } from '~/common/stores/chat/chat.fragments';
+import { LiveFileIcon } from '~/common/livefile/LiveFileIcon';
 import { copyToClipboard } from '~/common/util/clipboardUtils';
+import { liveFileInAttachmentFragment } from '~/common/livefile/liveFile';
 import { showImageDataURLInNewTab } from '~/common/util/imageUtils';
 
 import type { AttachmentDraftId } from '~/common/attachment-drafts/attachment.types';
@@ -55,6 +57,7 @@ export function LLMAttachmentMenu(props: {
   const isConverting = draft.outputsConverting;
   const isUnconvertible = !draft.converters.length;
   const isOutputMissing = !draft.outputFragments.length;
+  const hasLiveFile = draft.outputFragments.some(liveFileInAttachmentFragment);
 
   const isUnmoveable = props.isPositionFirst && props.isPositionLast;
 
@@ -102,7 +105,7 @@ export function LLMAttachmentMenu(props: {
     >
 
       {/* Move Arrows */}
-      {!isUnmoveable && <Box sx={{ display: 'flex', alignItems: 'center' }}>
+      {!isUnmoveable && <Box sx={{ display: 'flex', alignItems: 'center', borderBottom: '1px solid', borderColor: 'divider' }}>
         <MenuItem
           disabled={props.isPositionFirst}
           onClick={handleMoveUp}
@@ -118,7 +121,10 @@ export function LLMAttachmentMenu(props: {
           <KeyboardArrowRightIcon />
         </MenuItem>
       </Box>}
-      {!isUnmoveable && <ListDivider sx={{ my: 0 }} />}
+
+      {/*{(showDetails && canHaveDetails) && <ListItem variant='soft' sx={{ fontSize: 'sm', borderBottom: '1px solid', borderColor: 'divider' }}>*/}
+      {/*  {draft.ref}*/}
+      {/*</ListItem>}*/}
 
       {/* Render Converters as menu items */}
       {!isUnconvertible && (
@@ -178,6 +184,14 @@ export function LLMAttachmentMenu(props: {
           </Typography>
         ) : (
           <Box sx={{ my: 0.5 }}>
+            {/* LiveFile notice */}
+            {hasLiveFile && !!draftInput && (
+              <Typography level='body-sm' sx={{ mb: 1 }} startDecorator={<LiveFileIcon sx={{ width: 16, height: 16 }} />}>
+                LiveFile is supported
+              </Typography>
+            )}
+
+            {/* <- inputs */}
             {!!draftInput && (
               <Typography level='body-sm'>
                 🡐 {draftInput.mimeType}{typeof draftInput.dataSize === 'number' ? ` · ${draftInput.dataSize.toLocaleString()} bytes` : ''}
@@ -201,9 +215,12 @@ export function LLMAttachmentMenu(props: {
                 </Link>
               </Typography>
             )}
+
             {/*<Typography level='body-sm'>*/}
-            {/*  Converters: {aConverters.map(((converter, idx) => ` ${converter.id}${converter.isActive ? '*' : ''}`)).join(', ')}*/}
+            {/*  Converters: {draft.converters.map(((converter, idx) => ` ${converter.id}${converter.isActive ? '*' : ''}`)).join(', ')}*/}
             {/*</Typography>*/}
+
+            {/* -> Outputs */}
             <Box sx={{ mt: 1 }}>
               {isOutputMissing ? (
                 <Typography level='body-sm'>🡒 ...</Typography>

src/apps/chat/components/message/fragments-attachment-doc/DocumentFragmentButton.tsx

@@ -10,12 +10,14 @@ import TelegramIcon from '@mui/icons-material/Telegram';
 import TextFieldsIcon from '@mui/icons-material/TextFields';
 import TextureIcon from '@mui/icons-material/Texture';
 
-import { DMessageAttachmentFragment, DMessageFragmentId, isDocPart } from '~/common/stores/chat/chat.fragments';
 import { ContentScaling, themeScalingMap } from '~/common/app.theme';
+import { DMessageAttachmentFragment, DMessageFragmentId, isDocPart } from '~/common/stores/chat/chat.fragments';
+import { LiveFileIcon } from '~/common/livefile/LiveFileIcon';
 import { ellipsizeMiddle } from '~/common/util/textUtils';
+import { liveFileInAttachmentFragment } from '~/common/livefile/liveFile';
 
 
-function iconForFragment({ part }: DMessageAttachmentFragment): React.ComponentType<any> {
+function buttonIconForFragment({ part }: DMessageAttachmentFragment): React.ComponentType<any> {
   switch (part.pt) {
     case 'doc':
       switch (part.type) {
@@ -86,7 +88,7 @@ export function DocumentFragmentButton(props: {
 
   const buttonText = ellipsizeMiddle(fragment.part.l1Title || fragment.title || 'Document', 28 /* totally arbitrary length */);
 
-  const Icon = iconForFragment(fragment);
+  const Icon = buttonIconForFragment(fragment);
 
   return (
     <Button
@@ -115,6 +117,9 @@ export function DocumentFragmentButton(props: {
         {/*  {fragment.caption}*/}
         {/*</Box>*/}
       </Box>
+      {liveFileInAttachmentFragment(fragment) && (
+        <LiveFileIcon sx={{ mr: '0.5rem' }} />
+      )}
     </Button>
   );
 }

src/apps/chat/components/message/fragments-attachment-doc/DocumentFragmentEditor.tsx

@@ -73,7 +73,7 @@ export function DocumentFragmentEditor(props: {
 
     if (editedText.length > 0) {
       const newData = createDMessageDataInlineText(editedText, fragment.part.data.mimeType);
-      const newAttachment = createDocAttachmentFragment(fragmentTitle, fragment.caption, fragment.part.type, newData, fragment.part.ref, fragment.part.meta);
+      const newAttachment = createDocAttachmentFragment(fragmentTitle, fragment.caption, fragment.part.type, newData, fragment.part.ref, fragment.part.meta, fragment._liveFile);
       // reuse the same fragment ID, which makes the screen not flash (otherwise the whole editor would disappear as the ID does not exist anymore)
       newAttachment.fId = fragmentId;
       onFragmentReplace(fragmentId, newAttachment);
@@ -82,7 +82,7 @@ export function DocumentFragmentEditor(props: {
       // if the user deleted all text, let's remove the part
       handleFragmentDelete();
     }
-  }, [editedText, fragment.caption, fragment.part, fragmentId, fragmentTitle, handleFragmentDelete, onFragmentReplace]);
+  }, [editedText, fragment._liveFile, fragment.caption, fragment.part, fragmentId, fragmentTitle, handleFragmentDelete, onFragmentReplace]);
 
 
   return (

src/common/attachment-drafts/attachment.pipeline.ts

@@ -10,7 +10,7 @@ import { createDMessageDataInlineText, createDocAttachmentFragment, DMessageAtta
 import type { AttachmentsDraftsStore } from './store-attachment-drafts-slice';
 import { AttachmentDraft, AttachmentDraftConverter, AttachmentDraftInput, AttachmentDraftSource, DraftEgoFragmentsInputData, draftInputMimeEgoFragments, draftInputMimeWebpage, DraftWebInputData } from './attachment.types';
 import { imageDataToImageAttachmentFragmentViaDBlob } from './attachment.dblobs';
-import { liveFileAddToAttachmentFragment, liveFileIsSupported } from '../livefile/livefile';
+import { liveFileCreate, liveFileInSource } from '../livefile/liveFile';
 
 
 // configuration
@@ -284,7 +284,7 @@ export function attachmentDefineConverters(source: AttachmentDraftSource, input:
         converters.push({ id: 'rich-text-table', name: 'Markdown Table' });
 
       // p2: Text
-      converters.push({ id: 'text', name: liveFileIsSupported(source) ? 'Text (Live)' : 'Text' });
+      converters.push({ id: 'text', name: liveFileInSource(source) ? 'Text (Live)' : 'Text' });
 
       // p3: Html
       if (textOriginHtml) {
@@ -470,11 +470,13 @@ export async function attachmentPerformConversion(
 
       // text as-is
       case 'text':
-        const textData = createDMessageDataInlineText(inputDataToString(input.data), input.mimeType);
-        const textDocAttachmentFragment = createDocAttachmentFragment(title, caption, 'text/plain', textData, refString, docMeta);
+
         // [LiveFile] if we have a handle for this file, transfer it to the Doc Attachment fragment
-        liveFileAddToAttachmentFragment(source, textDocAttachmentFragment);
-        newFragments.push(textDocAttachmentFragment);
+        const liveFile = (liveFileInSource(source) && source.media === 'file' && converter.id === 'text')
+          ? liveFileCreate(source.fileWithHandle) : undefined;
+
+        const textualInlineData = createDMessageDataInlineText(inputDataToString(input.data), input.mimeType);
+        newFragments.push(createDocAttachmentFragment(title, caption, 'text/plain', textualInlineData, refString, docMeta, liveFile));
         break;
 
       // html as-is

src/common/attachment-drafts/attachment.types.ts

@@ -11,7 +11,7 @@ export type AttachmentDraft = {
   readonly id: AttachmentDraftId;
   readonly source: AttachmentDraftSource,
   label: string;
-  ref: string; // will be used in ```ref\n...``` for instance
+  ref: string; // will be used in ```ref\n...``` for instance - TODO: remove?
 
   inputLoading: boolean;
   inputError: string | null;
@@ -44,7 +44,7 @@ export type AttachmentDraftSource = {
   media: 'file';
   origin: AttachmentDraftSourceOriginFile,
   fileWithHandle: FileWithHandle;
-  refPath: string;
+  refPath: string; // original file name, or path/to/file name
 } | {
   media: 'text';
   method: 'clipboard-read' | AttachmentDraftSourceOriginDTO;

src/common/livefile/LiveFileIcon.ts

@@ -0,0 +1,3 @@
+import MultipleStopRoundedIcon from '@mui/icons-material/MultipleStopRounded';
+
+export { MultipleStopRoundedIcon as LiveFileIcon };

src/common/livefile/liveFile.ts

@@ -0,0 +1,26 @@
+import type { FileWithHandle } from 'browser-fs-access';
+
+import type { AttachmentDraftSource } from '~/common/attachment-drafts/attachment.types';
+import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.fragments';
+
+
+// AttachmentDraft Source
+
+export function liveFileInSource(source: AttachmentDraftSource): boolean {
+  return source.media === 'file' && !!source.fileWithHandle.handle && typeof source.fileWithHandle.handle.getFile === 'function';
+}
+
+
+// DMessageAttachmentFragment
+
+export function liveFileCreate(fileWithHandle: FileWithHandle): DMessageAttachmentFragment['_liveFile'] {
+  return {
+    lft: 'fs',
+    _fsFileHandle: fileWithHandle.handle,
+  };
+}
+
+export function liveFileInAttachmentFragment(attachmentFragment: DMessageAttachmentFragment): boolean {
+  return !!attachmentFragment._liveFile?._fsFileHandle;
+}
+

src/common/livefile/livefile.ts

@@ -1,12 +0,0 @@
-import type { AttachmentDraftSource } from '~/common/attachment-drafts/attachment.types';
-import type { DMessageAttachmentFragment } from '~/common/stores/chat/chat.fragments';
-
-
-export function liveFileIsSupported(source: AttachmentDraftSource): boolean {
-  return source.media === 'file' && !!source.fileWithHandle.handle && typeof source.fileWithHandle.handle.getFile === 'function';
-}
-
-export function liveFileAddToAttachmentFragment(source: AttachmentDraftSource, attachmentFragment: DMessageAttachmentFragment) {
-  if (source.media === 'file' && !!source.fileWithHandle.handle && typeof source.fileWithHandle.handle.getFile === 'function')
-    attachmentFragment._fileSystemFileHandle = source.fileWithHandle.handle;
-}

src/common/stores/chat/chat.fragments.ts

@@ -45,7 +45,10 @@ export type DMessageAttachmentFragment = _DMessageFragmentWrapper<'attachment',
   title: string;                  // label of the attachment (filename, named id, content overview, title..)
   caption: string;                // additional information, such as provenance, content preview, etc.
   created: number;
-  _fileSystemFileHandle?: FileSystemFileHandle; // [LiveFile] Store the handle to mem/DB - TODO: check if it works across sessions
+  _liveFile?: {
+    lft: 'fs';
+    _fsFileHandle?: FileSystemFileHandle; // [LiveFile] Store the handle to mem/DB - TODO: check if it works across sessions
+  }
 };
 
 // Future Examples: up to 1 per message, containing the Rays and Merges that would be used to restore the Beam state - could be volatile (omitted at save)
@@ -201,16 +204,16 @@ export function specialShallowReplaceTextContentFragment(copyFragment: DMessageC
 }
 
 
-function _createAttachmentFragment(title: string, caption: string, part: DMessageAttachmentFragment['part']): DMessageAttachmentFragment {
-  return { ft: 'attachment', fId: agiId('chat-dfragment' /* -attachment */), title, caption, created: Date.now(), part };
+function _createAttachmentFragment(title: string, caption: string, part: DMessageAttachmentFragment['part'], liveFile: DMessageAttachmentFragment['_liveFile']): DMessageAttachmentFragment {
+  return { ft: 'attachment', fId: agiId('chat-dfragment' /* -attachment */), title, caption, created: Date.now(), _liveFile: liveFile, part };
 }
 
-export function createDocAttachmentFragment(l1Title: string, caption: string, type: DMessageDocMimeType, data: DMessageDataInline, ref: string, meta?: DMessageDocMeta): DMessageAttachmentFragment {
-  return _createAttachmentFragment(l1Title, caption, createDMessageDocPart(type, data, ref, l1Title, meta));
+export function createDocAttachmentFragment(l1Title: string, caption: string, type: DMessageDocMimeType, data: DMessageDataInline, ref: string, meta?: DMessageDocMeta, liveFile?: DMessageAttachmentFragment['_liveFile']): DMessageAttachmentFragment {
+  return _createAttachmentFragment(l1Title, caption, createDMessageDocPart(type, data, ref, l1Title, meta), liveFile);
 }
 
 export function createImageAttachmentFragment(title: string, caption: string, dataRef: DMessageDataRef, imgAltText?: string, width?: number, height?: number): DMessageAttachmentFragment {
-  return _createAttachmentFragment(title, caption, createDMessageImageRefPart(dataRef, imgAltText, width, height));
+  return _createAttachmentFragment(title, caption, createDMessageImageRefPart(dataRef, imgAltText, width, height), undefined);
 }
 
 export function specialContentPartToDocAttachmentFragment(title: string, caption: string, contentPart: DMessageContentFragment['part'], ref: string, docMeta?: DMessageDocMeta): DMessageAttachmentFragment {
@@ -297,7 +300,9 @@ function _duplicateFragment(fragment: DMessageFragment): DMessageFragment {
       return _createContentFragment(_duplicatePart(fragment.part));
 
     case 'attachment':
-      return _createAttachmentFragment(fragment.title, fragment.caption, _duplicatePart(fragment.part));
+      // NOTE: check if this is correct - we haven't tested the serialization of FileSystemFileHandle yet
+      const liveFileCopy = fragment._liveFile ? { ...fragment._liveFile } : undefined;
+      return _createAttachmentFragment(fragment.title, fragment.caption, _duplicatePart(fragment.part), liveFileCopy);
 
     case '_ft_sentinel':
       return _createSentinelFragment();