diff --git a/src/apps/chat/components/composer/Composer.tsx b/src/apps/chat/components/composer/Composer.tsx index 89196d382..114fd09e4 100644 --- a/src/apps/chat/components/composer/Composer.tsx +++ b/src/apps/chat/components/composer/Composer.tsx @@ -30,7 +30,7 @@ import { DMessageMetadata, DMetaReferenceItem, messageFragmentsReduceText } from import { ShortcutKey, ShortcutObject, useGlobalShortcuts } from '~/common/components/shortcuts/useGlobalShortcuts'; import { addSnackbar } from '~/common/components/snackbar/useSnackbarsStore'; import { animationEnterBelow } from '~/common/util/animUtils'; -import { browserSpeechRecognitionCapability, PLACEHOLDER_INTERIM_TRANSCRIPT, SpeechResult, useSpeechRecognition, } from '~/common/components/useSpeechRecognition'; +import { browserSpeechRecognitionCapability, PLACEHOLDER_INTERIM_TRANSCRIPT, SpeechResult, useSpeechRecognition } from '~/common/components/useSpeechRecognition'; import { conversationTitle, DConversationId } from '~/common/stores/chat/chat.conversation'; import { copyToClipboard, supportsClipboardRead } from '~/common/util/clipboardUtils'; import { createTextContentFragment, DMessageAttachmentFragment, DMessageContentFragment, duplicateDMessageFragmentsNoPH } from '~/common/stores/chat/chat.fragments'; @@ -108,6 +108,7 @@ export function Composer(props: { const [composeText, debouncedText, setComposeText] = useDebouncer('', 300, 1200, true); const [micContinuation, setMicContinuation] = React.useState(false); const [speechInterimResult, setSpeechInterimResult] = React.useState(null); + const [sendStarted, setSendStarted] = React.useState(false); const { chatExecuteMode, chatExecuteModeSendColor, chatExecuteModeSendLabel, @@ -167,8 +168,8 @@ export function Composer(props: { onConversationsImportFromFiles([file]); onResolve(true); }} - title="Open Conversation or Attach?" - positiveActionText="Open" negativeActionText="Attach" + title='Open Conversation or Attach?' + positiveActionText='Open' negativeActionText='Attach' confirmationText={`Would you like to open the conversation "${file.name}" or attach it to the message?`} /> )), [onConversationsImportFromFiles, showPromisedOverlay]); @@ -252,10 +253,10 @@ export function Composer(props: { open onClose={onUserReject} onPositive={() => onResolve(true)} - confirmationText="Some attached files may not be fully compatible with the current AI model. This could affect processing. Would you like to review or proceed?" - positiveActionText="Proceed" - negativeActionText="Review Attachments" - title="Attachment Compatibility Notice" + confirmationText='Some attached files may not be fully compatible with the current AI model. This could affect processing. Would you like to review or proceed?' + positiveActionText='Proceed' + negativeActionText='Review Attachments' + title='Attachment Compatibility Notice' /> )); }, [llmAttachmentDraftsCollection.canAttachAllFragments, showPromisedOverlay]); @@ -263,14 +264,13 @@ export function Composer(props: { // Primary button - const handleClear = React.useCallback(() => { + const _handleClearText = React.useCallback(() => { setComposeText(''); attachmentsRemoveAll(); handleInReferenceToClear(); }, [attachmentsRemoveAll, handleInReferenceToClear, setComposeText]); - - const handleSendAction = React.useCallback(async (_chatExecuteMode: ChatExecuteMode, composerText: string): Promise => { + const _handleSendActionUnguarded = React.useCallback(async (_chatExecuteMode: ChatExecuteMode, composerText: string): Promise => { if (!isValidConversation(targetConversationId)) return false; // await user confirmation (or rejection) if attachments are not supported @@ -306,10 +306,93 @@ export function Composer(props: { // send the message - NOTE: if successful, the ownership of the fragments is transferred to the receiver, so we just clear them const enqueued = onAction(targetConversationId, _chatExecuteMode, fragments, metadata); if (enqueued) - handleClear(); + _handleClearText(); return enqueued; - }, [attachmentsTakeAllFragments, confirmProceedIfAttachmentsNotSupported, handleClear, inReferenceTo, onAction, targetConversationId]); + }, [attachmentsTakeAllFragments, confirmProceedIfAttachmentsNotSupported, _handleClearText, inReferenceTo, onAction, targetConversationId]); + const handleSendAction = React.useCallback(async (chatExecuteMode: ChatExecuteMode, composerText: string): Promise => { + setSendStarted(true); + const enqueued = await _handleSendActionUnguarded(chatExecuteMode, composerText); + setSendStarted(false); + return enqueued; + }, [_handleSendActionUnguarded, setSendStarted]); + + + // Mic typing & continuation mode - NOTE: this is here because needs the handleSendAction, and provides recognitionState + + const onSpeechResultCallback = React.useCallback((result: SpeechResult) => { + // not done: show interim + if (!result.done) { + setSpeechInterimResult({ ...result }); + return; + } + + // done + setSpeechInterimResult(null); + const transcript = result.transcript.trim(); + let nextText = (composeText || '').trim(); + nextText = nextText ? nextText + ' ' + transcript : transcript; + + // auto-send (mic continuation mode) if requested + const autoSend = (result.flagSendOnDone || micContinuation) && nextText.length >= 1 && !noConversation; //&& assistantAbortible; + const notUserStop = result.doneReason !== 'manual'; + if (autoSend) { + // if (notUserStop) { + void AudioGenerator.chatAutoSend(); + // void AudioPlayer.playUrl('/sounds/mic-off-mid.mp3'); + // } + void handleSendAction(chatExecuteMode, nextText); // fire/forget + } else { + // if scheduled for send but not sent, clear the send state + if (result.flagSendOnDone) + setSendStarted(false); + + // mic off sound + if (!micContinuation && notUserStop) + void AudioPlayer.playUrl('/sounds/mic-off-mid.mp3').catch(() => { + // This happens on Is.Browser.Safari, where the audio is not allowed to play without user interaction + }); + + // update with the spoken text + if (nextText) { + composerTextAreaRef.current?.focus(); + setComposeText(nextText); + } + } + }, [chatExecuteMode, composeText, composerTextAreaRef, handleSendAction, micContinuation, noConversation, setComposeText]); + + const { recognitionState, toggleRecognition } = useSpeechRecognition(onSpeechResultCallback, chatMicTimeoutMs || 2000); + + const micContinuationTrigger = micContinuation && !micIsRunning && !assistantAbortible && !recognitionState.errorMessage; + const micColor: ColorPaletteProp = recognitionState.errorMessage ? 'danger' : recognitionState.isActive ? 'primary' : recognitionState.hasAudio ? 'primary' : 'neutral'; + const micVariant: VariantProp = recognitionState.hasSpeech ? 'solid' : recognitionState.hasAudio ? 'soft' : 'soft'; //(isDesktop ? 'soft' : 'plain'); + + const handleToggleMic = React.useCallback(() => { + if (micIsRunning && micContinuation) + setMicContinuation(false); + toggleRecognition(); + }, [micContinuation, micIsRunning, toggleRecognition]); + + const handleToggleMicContinuation = React.useCallback(() => { + setMicContinuation(continued => !continued); + }, []); + + React.useEffect(() => { + // autostart the microphone if the assistant stopped typing + if (micContinuationTrigger) + toggleRecognition(); + }, [toggleRecognition, micContinuationTrigger]); + + React.useEffect(() => { + // auto-scroll the mic card to the bottom + micCardRef.current?.scrollTo({ + top: micCardRef.current.scrollHeight, + behavior: 'smooth', + }); + }, [speechInterimResult]); + + + // Other send actins const handleAppendTextAndSend = React.useCallback(async (appendText: string) => { const newText = composeText ? `${composeText} ${appendText}` : appendText; @@ -317,13 +400,26 @@ export function Composer(props: { await handleSendAction(chatExecuteMode, newText); }, [chatExecuteMode, composeText, handleSendAction, setComposeText]); + const handleFinishMicAndSend = React.useCallback(() => { + if (!sendStarted) { + setSendStarted(true); + toggleRecognition(true); + } + }, [sendStarted, toggleRecognition]); + const handleSendClicked = React.useCallback(async () => { + // Auto-send as soon as the mic is done + if (recognitionState.isActive) { + handleFinishMicAndSend(); + return; + } + // Safety option if (micIsRunning) { addSnackbar({ key: 'chat-mic-running', message: 'Please wait for the microphone to finish.', type: 'info' }); return; } await handleSendAction(chatExecuteMode, composeText); // 'chat/write/...' button - }, [chatExecuteMode, composeText, handleSendAction, micIsRunning]); + }, [chatExecuteMode, composeText, handleFinishMicAndSend, handleSendAction, micIsRunning, recognitionState.isActive]); const handleSendTextBeamClicked = React.useCallback(async () => { if (micIsRunning) { @@ -446,76 +542,8 @@ export function Composer(props: { // const handleFocusModeOff = React.useCallback(() => setIsFocusedMode(false), [setIsFocusedMode]); - - // Mic typing & continuation mode - - const onSpeechResultCallback = React.useCallback((result: SpeechResult) => { - // not done: show interim - if (!result.done) { - setSpeechInterimResult({ ...result }); - return; - } - - // done - setSpeechInterimResult(null); - const transcript = result.transcript.trim(); - let nextText = (composeText || '').trim(); - nextText = nextText ? nextText + ' ' + transcript : transcript; - - // auto-send (mic continuation mode) if requested - const autoSend = (result.flagSendOnDone || micContinuation) && nextText.length >= 1 && !noConversation; //&& assistantAbortible; - const notUserStop = result.doneReason !== 'manual'; - if (autoSend) { - // if (notUserStop) { - void AudioGenerator.chatAutoSend(); - // void AudioPlayer.playUrl('/sounds/mic-off-mid.mp3'); - // } - void handleSendAction(chatExecuteMode, nextText); // fire/forget - } else { - if (!micContinuation && notUserStop) - void AudioPlayer.playUrl('/sounds/mic-off-mid.mp3').catch(() => { - // This happens on Is.Browser.Safari, where the audio is not allowed to play without user interaction - }); - if (nextText) { - composerTextAreaRef.current?.focus(); - setComposeText(nextText); - } - } - }, [chatExecuteMode, composeText, composerTextAreaRef, handleSendAction, micContinuation, noConversation, setComposeText]); - - const { recognitionState, toggleRecognition } = useSpeechRecognition(onSpeechResultCallback, chatMicTimeoutMs || 2000); - // useMediaSessionCallbacks({ play: toggleRecognition, pause: toggleRecognition }); - const micContinuationTrigger = micContinuation && !micIsRunning && !assistantAbortible && !recognitionState.errorMessage; - const micColor: ColorPaletteProp = recognitionState.errorMessage ? 'danger' : recognitionState.isActive ? 'primary' : recognitionState.hasAudio ? 'primary' : 'neutral'; - const micVariant: VariantProp = recognitionState.hasSpeech ? 'solid' : recognitionState.hasAudio ? 'soft' : 'soft'; //(isDesktop ? 'soft' : 'plain'); - - const handleToggleMic = React.useCallback(() => { - if (micIsRunning && micContinuation) - setMicContinuation(false); - toggleRecognition(); - }, [micContinuation, micIsRunning, toggleRecognition]); - - const handleToggleMicContinuation = React.useCallback(() => { - setMicContinuation(continued => !continued); - }, []); - - React.useEffect(() => { - // autostart the microphone if the assistant stopped typing - if (micContinuationTrigger) - toggleRecognition(); - }, [toggleRecognition, micContinuationTrigger]); - - React.useEffect(() => { - // auto-scroll the mic card to the bottom - micCardRef.current?.scrollTo({ - top: micCardRef.current.scrollHeight, - behavior: 'smooth' - }); - }, [speechInterimResult]); - - // Attachment Up @@ -574,7 +602,7 @@ export function Composer(props: { composerShortcuts.push({ key: 'v', ctrl: true, shift: true, action: attachAppendClipboardItems, description: 'Attach Clipboard' }); } if (recognitionState.isActive) { - composerShortcuts.push({ key: 'm', ctrl: true, action: () => toggleRecognition(true), description: 'Mic · Send', disabled: !recognitionState.hasSpeech, endDecoratorIcon: TelegramIcon as any, level: 4 }); + composerShortcuts.push({ key: 'm', ctrl: true, action: handleFinishMicAndSend, description: 'Mic · Send', disabled: !recognitionState.hasSpeech || sendStarted, endDecoratorIcon: TelegramIcon as any, level: 4 }); composerShortcuts.push({ key: ShortcutKey.Esc, action: () => { setMicContinuation(false); @@ -590,7 +618,7 @@ export function Composer(props: { }, description: 'Microphone', }); return composerShortcuts; - }, [attachAppendClipboardItems, handleAttachFiles, recognitionState.hasSpeech, recognitionState.isActive, showChatAttachments, toggleRecognition])); + }, [attachAppendClipboardItems, handleAttachFiles, handleFinishMicAndSend, recognitionState.hasSpeech, recognitionState.isActive, sendStarted, showChatAttachments, toggleRecognition])); // ... @@ -921,7 +949,10 @@ export function Composer(props: { {!assistantAbortible ? (