Compare commits

..

81 Commits

Author SHA1 Message Date
Enrico Ros e5f674509c 2.0.2 News 2025-11-30 16:54:56 -08:00
Enrico Ros 197a4ae5c0 2.0.2 Package 2025-11-30 16:53:25 -08:00
Enrico Ros 64d2dcf39c AudioLivePlayer: tryfix for the persistent android notification 2025-11-30 15:05:17 -08:00
Enrico Ros caf54c736b Speex: do not stop the playback too early 2025-11-30 14:31:43 -08:00
Enrico Ros 423c2cce28 speakText: port to Speex 2025-11-30 12:51:55 -08:00
Enrico Ros a1af51efcb Call: port to Speex 2025-11-30 06:55:51 -08:00
Enrico Ros ffc1bf9c58 Remove src/modules/elevenlabs 2025-11-30 06:55:51 -08:00
Enrico Ros a54bfdb342 Settings: port to Speex 2025-11-30 06:55:51 -08:00
Enrico Ros 03861d2dbd Speex: map instead of array 2025-11-30 06:38:14 -08:00
Enrico Ros 8c080da6bf Speex: Autoconfig WebSpeech best 2025-11-30 06:38:14 -08:00
Enrico Ros a8c98056b6 Speex: Config UI Done 2025-11-30 06:38:14 -08:00
Enrico Ros 78e663f955 Speex: important fixes 2025-11-30 06:38:14 -08:00
Enrico Ros 70546a5039 Speex: Almost Done 2025-11-30 06:38:14 -08:00
Enrico Ros 30f78b33cb Speex: diable Azure 2025-11-30 06:38:14 -08:00
Enrico Ros 712e8c1f16 Speex: UI update: Selects and Persona Voice changer 2025-11-30 06:38:14 -08:00
Enrico Ros 933dfdfb53 Speex: improve types 2025-11-30 06:38:14 -08:00
Enrico Ros 9ce86b029f Speex: UI settings modal 2025-11-30 06:38:14 -08:00
Enrico Ros 13580cc69d Speex: UI config improvements 2025-11-30 06:38:14 -08:00
Enrico Ros a7dee0002d Speex: debug instrumentation 2025-11-30 06:38:14 -08:00
Enrico Ros c84b2df3fa Speex: fix elevenlabs 2025-11-30 06:38:14 -08:00
Enrico Ros d9471a8684 Speex: fix types 2025-11-30 06:38:14 -08:00
Enrico Ros ef630c2272 Speex: improve UI and errors 2025-11-30 06:38:14 -08:00
Enrico Ros e188c71652 Speex: RPC: shared downstreaming 2025-11-30 06:38:14 -08:00
Enrico Ros 910260c2c8 Speex: UI: credentials edit and add new 2025-11-30 06:38:14 -08:00
Enrico Ros 22752abc38 Speex: relax engine validation 2025-11-30 06:38:14 -08:00
Enrico Ros 92bc3a5d64 Speex: DVoice -> wire_Voice 2025-11-30 06:38:14 -08:00
Enrico Ros 1383752cc1 Speex: reduce logging 2025-11-30 06:38:13 -08:00
Enrico Ros 66af16fb81 Speex: manual refactor 2025-11-30 06:38:13 -08:00
Enrico Ros fc019d7b46 Speex: client cleanups 2025-11-30 06:38:13 -08:00
Enrico Ros ac4f0fcb12 Speex: LocalAI: Preview 2025-11-30 06:38:13 -08:00
Enrico Ros a6c2bc663d Speex: arrange files 2025-11-30 06:38:13 -08:00
Enrico Ros e62ffa02e9 Speex: LocalAI vendor 2025-11-30 06:38:13 -08:00
Enrico Ros a003600839 Speex: some UI 2025-11-30 06:38:13 -08:00
Enrico Ros ea73feb06d Speex: remove elevenlabs, with key migration 2025-11-30 06:38:13 -08:00
Enrico Ros 3bdf69e1b7 Speex: ui: begin 2025-11-30 06:38:13 -08:00
Enrico Ros 590fe78bd1 Speex: client cleanup 2025-11-30 06:38:13 -08:00
Enrico Ros 76187ba0e7 Speex: rpc backend 2025-11-30 06:38:13 -08:00
Enrico Ros 5eba375f4d Speex: add webspeech (with detection) and synthesize-openai 2025-11-30 06:38:13 -08:00
Enrico Ros 8fa6a8251f Speex: vendors, engine store, client, router, skel-synthesize 2025-11-30 06:38:13 -08:00
Enrico Ros 75fa046f30 Speex: centralize capability 2025-11-30 06:38:13 -08:00
Enrico Ros 08a8cd1430 Speex: Types & Client 2025-11-30 06:38:13 -08:00
Enrico Ros 3afbb78a39 Icons: port to PhVoice 2025-11-30 06:38:12 -08:00
Enrico Ros fca6ccd816 Badge: transparent BG to not overlap text. Fixes #889 2025-11-29 14:52:13 -08:00
Enrico Ros 8d351822c1 Niy 2025-11-29 13:25:36 -08:00
Enrico Ros 7d274a31fe AIX: CGR: use shared objectUtils 2025-11-29 12:40:04 -08:00
Enrico Ros e36dde0d25 objectUtils: estimate JSON size, deep clone with string limit, find largestStringPaths 2025-11-29 12:17:28 -08:00
Enrico Ros 51cc6e5ae5 CSF: only show the option for server-side (not client-side) disconnect 2025-11-29 11:12:30 -08:00
Enrico Ros 28d911c617 ElevenLabsIcon: add icon 2025-11-28 05:49:33 -08:00
Enrico Ros b1e9fe58fb objectUtils: add stripUndefined 2025-11-28 04:23:11 -08:00
Enrico Ros 16ba014ade GoodBadge: for 'new' 2025-11-28 04:23:11 -08:00
Enrico Ros e9d5a20c1a FormTextField: support inputSx 2025-11-28 04:23:11 -08:00
Enrico Ros 6e0036f9c4 FormSecretField: crystal clear keys input 2025-11-28 04:23:11 -08:00
Enrico Ros d7e189aa1c FormSliderControl: allow sliderSx 2025-11-28 04:23:11 -08:00
Enrico Ros ea2b444fb2 FormChipControl: alignEnd 2025-11-28 04:23:11 -08:00
Enrico Ros cd1efaf26e FormChipControl: support descriptions 2025-11-28 04:23:11 -08:00
Enrico Ros e47f0e5d43 LanguageSelect: imrove select 2025-11-28 04:23:11 -08:00
Enrico Ros 5284d37984 AudioLivePlayer: ignore a closure error 2025-11-28 04:23:11 -08:00
Enrico Ros 1bf6fa0e4d Browse service: improve error reporting 2025-11-27 19:12:08 -08:00
Enrico Ros fc294c82f1 Pdfjs: lock to 5.4.54
more recent 5.4 have trouble with await import('pdfjs-dist'), throwing.
2025-11-27 18:33:20 -08:00
Enrico Ros 7b1dc49dda Roll pdfjs 2025-11-27 18:19:51 -08:00
Enrico Ros d15ddeea24 Roll react-player 2025-11-27 18:15:19 -08:00
Enrico Ros eaac213859 Ph: add Voice 2025-11-27 18:07:54 -08:00
Enrico Ros 02c1460351 Roll posthog 2025-11-27 18:04:06 -08:00
Enrico Ros 2fff35b7d9 Roll superjson 2025-11-27 18:03:37 -08:00
Enrico Ros c5b9072bde LLMs: LocalAI publish interface 2025-11-26 19:01:44 -08:00
Enrico Ros 8a570e912a CSF: docs 2025-11-26 07:37:56 -08:00
Enrico Ros 1dcc40afb8 CSF: Propagate everywhere 2025-11-26 07:37:09 -08:00
Enrico Ros c2092f8035 BlockPartError: vendor name 2025-11-26 06:50:11 -08:00
Enrico Ros 886c4b411e Revert "Test Edge on node"
This reverts commit 8888fd40cd.
2025-11-26 06:13:28 -08:00
Enrico Ros 8888fd40cd Test Edge on node 2025-11-26 04:56:26 -08:00
Enrico Ros 31cd01bccf BlockPartError: CSF enabled 2025-11-26 04:42:50 -08:00
Enrico Ros c59b221004 BlockPartError: allow retrying disconnected errors too 2025-11-26 04:27:52 -08:00
Enrico Ros cb3cc3e74c PostHog: disable the info level 2025-11-26 04:05:03 -08:00
Enrico Ros 9e90015fcc PostHog: disable the info level 2025-11-26 03:56:55 -08:00
Enrico Ros 95e0517056 60s - disable any maxDuration 2025-11-26 03:56:25 -08:00
Enrico Ros 2b2f47915f AIX: OpenAI: Fix CSF! 2025-11-26 03:11:12 -08:00
Enrico Ros 9acd178ce1 AudioPlayer: safe end of stream 2025-11-26 03:11:08 -08:00
Enrico Ros f381f80184 AIX: Anthropic: add strict to tool defs on wiretypes 2025-11-24 16:44:13 -08:00
Enrico Ros c83be61343 AIX: Anthropic: newlines for text broken by tool calls 2025-11-24 16:05:44 -08:00
Enrico Ros f6e49d31ec PWA-Desktop detect. Fixes #887 2025-11-24 15:48:50 -08:00
Enrico Ros cc0429a362 Update readme 2025-11-24 15:14:49 -08:00
112 changed files with 4490 additions and 1138 deletions
+10 -4
View File
@@ -42,7 +42,8 @@ It comes packed with **world-class features** like Beam, and is praised for its
[![Feature Inspector](https://img.shields.io/badge/Expert_Mode-AI_Inspector-000?style=for-the-badge&labelColor=purple)](https://big-agi.com/inspector)
### What makes Big-AGI different:
**Intelligence**: with [Beam & Merge](https://big-agi.com/beam) for multi-model de-hallucination, native search, and bleeding-edge AI models like Nano Banana, Kimi K2 Thinking or GPT 5.1 -
**Intelligence**: with [Beam & Merge](https://big-agi.com/beam) for multi-model de-hallucination, native search, and bleeding-edge AI models like Opus 4.5, Nano Banana, Kimi K2 or GPT 5.1 -
**Control**: with personas, data ownership, requests inspection, unlimited usage with API keys, and *no vendor lock-in* -
and **Speed**: with a local-first, over-powered, zero-latency, madly optimized web app.
@@ -138,9 +139,14 @@ so you **are not vendor locked-in**, and obsessed over a powerful UI that works,
NOTE: this is a powerful tool - if you need a toy UI or clone, this ain't it.
## What's New in 2.0 · Oct 31, 2025 · Open
---
👉 **[See the full changelog](https://big-agi.com/changes)**
## Release Notes
👉 **[See the Live Release Notes](https://big-agi.com/changes)**
- Open 2.0.1: **Opus 4.5** full support, **Gemini 3 Pro** w/ code exec, **Nano Banana Pro**, **Grok 4.1**, **GPT-5.1**, **Kimi K2 Thinking** + 280 fixes
### What's New in 2.0 · Oct 31, 2025 · Open
- **Big-AGI Open** is ready and more productive and faster than ever, with:
- **Beam 2**: multi-modal, program-based, follow-ups, save presets
@@ -153,7 +159,7 @@ NOTE: this is a powerful tool - if you need a toy UI or clone, this ain't it.
<img width="830" height="385" alt="image" src="https://github.com/user-attachments/assets/ad52761d-7e3f-44d8-b41e-947ce8b4faa1" />
### Open links: 👉 [changelog](https://big-agi.com/changes) 👉 [installation](docs/installation.md) 👉 [roadmap](https://github.com/users/enricoros/projects/4/views/2) 👉 [documentation](docs/README.md)
#### **Open** links: 👉 [changelog](https://big-agi.com/changes) 👉 [installation](docs/installation.md) 👉 [roadmap](https://github.com/users/enricoros/projects/4/views/2) 👉 [documentation](docs/README.md)
**For teams and institutions:** Need shared prompts, SSO, or managed deployments? Reach out at enrico@big-agi.com. We're actively collecting requirements from research groups and IT departments.
+1 -1
View File
@@ -33,7 +33,7 @@ const handlerNodeRoutes = (req: Request) => fetchRequestHandler({
// NOTE: the following statement breaks the build on non-pro deployments, and conditionals don't work either
// so we resorted to raising the timeout from 10s to 60s in the vercel.json file instead
export const maxDuration = 60;
// export const maxDuration = 60;
export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
export { handlerNodeRoutes as GET, handlerNodeRoutes as POST };
+2
View File
@@ -14,5 +14,7 @@ const handlerEdgeRoutes = (req: Request) => fetchRequestHandler({
: undefined,
});
// NOTE: we don't set maxDuration explicitly here - however we set it in the Vercel project settings, raising to the limit of 300s
// export const maxDuration = 60;
export const runtime = 'edge';
export { handlerEdgeRoutes as GET, handlerEdgeRoutes as POST };
+3
View File
@@ -14,6 +14,9 @@ Internal documentation for Big-AGI architecture and systems, for use by AI agent
- **[AIX.md](modules/AIX.md)** - AIX streaming architecture documentation
- **[AIX-callers-analysis.md](modules/AIX-callers-analysis.md)** - Analysis of AIX entry points, call chains, common and different rendering, error handling, etc.
#### CSF - Client-Side Fetch
- **[CSF.md](systems/client-side-fetch.md)** - Direct browser-to-API communication for LLM requests
### Systems Documentation
#### Core Platform Systems
+13
View File
@@ -0,0 +1,13 @@
# CSF - Client-Side Fetch
Client-Side Fetch (CSF) enables direct browser-to-API communication, bypassing the server for LLM requests. When enabled, the browser makes requests directly to vendor APIs (e.g., `api.openai.com`, `api.groq.com`) instead of routing through the Next.js server. This reduces latency, decreases server load, and is particularly useful for local models where the browser can communicate directly with Ollama or LM Studio.
## Implementation
CSF is implemented as an opt-in setting stored as `csf: boolean` in each vendor's service settings. The vendor interface exposes `csfAvailable?: (setup) => boolean` to determine if CSF can be enabled (typically checking if an API key or host is configured). The actual execution happens in `aix.client.direct-chatGenerate.ts` which dynamically imports when CSF is active, making direct fetch calls using the same wire protocols as the server.
All 16 supported vendors (OpenAI, Anthropic, Gemini, Ollama, LocalAI, Deepseek, Groq, Mistral, xAI, OpenRouter, Perplexity, Together AI, Alibaba, Moonshot, OpenPipe, LM Studio) support CSF. Cloud vendors require CORS support from the API provider (all tested vendors return `access-control-allow-origin: *`). Local vendors (Ollama, LocalAI, LM Studio) require CORS to be enabled on the local server.
## UI
The CSF toggle appears in each vendor's setup panel under "Advanced" settings, labeled "Direct Connection". It becomes visible when the prerequisites are met (API key present for cloud vendors, host configured for local vendors). The setting is managed through `useModelServiceClientSideFetch` hook which provides `csfAvailable`, `csfActive`, `csfToggle`, and `csfReset` for UI consumption.
+1 -1
View File
@@ -141,7 +141,7 @@ if (process.env.POSTHOG_API_KEY && process.env.POSTHOG_ENV_ID) {
personalApiKey: process.env.POSTHOG_API_KEY,
envId: process.env.POSTHOG_ENV_ID,
host: 'https://us.i.posthog.com', // backtrace upload host
logLevel: 'info',
logLevel: 'error', // lowered, too noisy
sourcemaps: {
enabled: process.env.NODE_ENV === 'production',
project: 'big-agi',
+102 -102
View File
@@ -1,12 +1,12 @@
{
"name": "big-agi",
"version": "2.0.1",
"version": "2.0.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "big-agi",
"version": "2.0.1",
"version": "2.0.2",
"hasInstallScript": true,
"dependencies": {
"@dnd-kit/core": "^6.3.1",
@@ -42,7 +42,7 @@
"next": "~15.1.8",
"nprogress": "^0.2.0",
"pdfjs-dist": "5.4.54",
"posthog-js": "^1.298.0",
"posthog-js": "^1.298.1",
"posthog-node": "^5.14.0",
"prismjs": "^1.30.0",
"puppeteer-core": "^24.31.0",
@@ -58,7 +58,7 @@
"remark-mark-highlight": "^0.1.1",
"remark-math": "^6.0.0",
"sharp": "^0.34.5",
"superjson": "^2.2.5",
"superjson": "^2.2.6",
"tesseract.js": "^6.0.1",
"tiktoken": "^1.0.22",
"turndown": "^7.2.2",
@@ -1556,25 +1556,25 @@
}
},
"node_modules/@mux/mux-player": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/@mux/mux-player/-/mux-player-3.9.0.tgz",
"integrity": "sha512-OjRXdJFPstCoTipqJCXyC3e3PVoLp8jOheCaWxe2a8qvHkSs/sg+UoYegr++hAoLXXIyy2M7F6vi+tWq0W5bYA==",
"version": "3.9.1",
"resolved": "https://registry.npmjs.org/@mux/mux-player/-/mux-player-3.9.1.tgz",
"integrity": "sha512-buvulWJD9qevaCp02oTMezn6QKmV7OwU1WDt7In8IgOJ8GKn1ZWLvfSbWEd1s3Ea1tC33NastKVG8sYlJ7HNkA==",
"license": "MIT",
"dependencies": {
"@mux/mux-video": "0.28.0",
"@mux/playback-core": "0.31.3",
"@mux/mux-video": "0.28.1",
"@mux/playback-core": "0.31.4",
"media-chrome": "~4.16.0",
"player.style": "^0.3.0"
}
},
"node_modules/@mux/mux-player-react": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/@mux/mux-player-react/-/mux-player-react-3.9.0.tgz",
"integrity": "sha512-AYBX89T02qOJ6rF4X2sB8WmPoHBQIrASvp6rxCf9wWYdp5lYtAjjTwaAM2aTlVEXSzdDPaOwgC7VmR7LhBJn3g==",
"version": "3.9.1",
"resolved": "https://registry.npmjs.org/@mux/mux-player-react/-/mux-player-react-3.9.1.tgz",
"integrity": "sha512-4cumt+5ObUKIZioSajLLxIghCzOn4LBcFz2mNj1OGAZGckEk8jbFB/W4o2HRlOS+4cTCEKPnOQ1l7AgjGuzZ2A==",
"license": "MIT",
"dependencies": {
"@mux/mux-player": "3.9.0",
"@mux/playback-core": "0.31.3",
"@mux/mux-player": "3.9.1",
"@mux/playback-core": "0.31.4",
"prop-types": "^15.8.1"
},
"peerDependencies": {
@@ -1592,32 +1592,32 @@
}
},
"node_modules/@mux/mux-video": {
"version": "0.28.0",
"resolved": "https://registry.npmjs.org/@mux/mux-video/-/mux-video-0.28.0.tgz",
"integrity": "sha512-pqpoaoxHXsGX/l7jOZGZ7jOVBVdct8jq+Be1cQ9+n5N/XrkXIGNvO/liprQET3wRuWs2Xri5lWZFRe3ZkOkYHw==",
"version": "0.28.1",
"resolved": "https://registry.npmjs.org/@mux/mux-video/-/mux-video-0.28.1.tgz",
"integrity": "sha512-lmxPmAPtUITZ+EButcZBoVs2FZbt7UgxYV/sCVOo0kGnOCrnmqjw1G3CW9tEcFST+NzdW/nIdzpRu9kg9FxBeg==",
"license": "MIT",
"dependencies": {
"@mux/mux-data-google-ima": "0.2.8",
"@mux/playback-core": "0.31.3",
"@mux/playback-core": "0.31.4",
"castable-video": "~1.1.11",
"custom-media-element": "~1.4.5",
"media-tracks": "~0.3.4"
}
},
"node_modules/@mux/playback-core": {
"version": "0.31.3",
"resolved": "https://registry.npmjs.org/@mux/playback-core/-/playback-core-0.31.3.tgz",
"integrity": "sha512-IiNnF6LeE8xB5lXzwCUmHUi40q/88ook/YYKTwDsIMG/0zY8b9Ypyzw9ghsELZH0mzuFmv+rvm3ufOIoaIi9eA==",
"version": "0.31.4",
"resolved": "https://registry.npmjs.org/@mux/playback-core/-/playback-core-0.31.4.tgz",
"integrity": "sha512-qQrNAAdJ7vjr1XEObE1hOUmuYngk/fjwmtYhpzkX4jJZwUC8I0rHjeFv7LXuCQD1D/mYJlWEpuyA0gPd6Y2eQw==",
"license": "MIT",
"dependencies": {
"hls.js": "~1.6.13",
"hls.js": "~1.6.15",
"mux-embed": "^5.8.3"
}
},
"node_modules/@napi-rs/canvas": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.82.tgz",
"integrity": "sha512-FGjyUBoF0sl1EenSiE4UV2WYu76q6F9GSYedq5EiOCOyGYoQ/Owulcv6rd7v/tWOpljDDtefXXIaOCJrVKem4w==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.83.tgz",
"integrity": "sha512-f9GVB9VNc9vn/nroc9epXRNkVpvNPZh69+qzLJIm9DfruxFqX0/jsXG46OGWAJgkO4mN0HvFHjRROMXKVmPszg==",
"license": "MIT",
"optional": true,
"workspaces": [
@@ -1627,22 +1627,22 @@
"node": ">= 10"
},
"optionalDependencies": {
"@napi-rs/canvas-android-arm64": "0.1.82",
"@napi-rs/canvas-darwin-arm64": "0.1.82",
"@napi-rs/canvas-darwin-x64": "0.1.82",
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.82",
"@napi-rs/canvas-linux-arm64-gnu": "0.1.82",
"@napi-rs/canvas-linux-arm64-musl": "0.1.82",
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.82",
"@napi-rs/canvas-linux-x64-gnu": "0.1.82",
"@napi-rs/canvas-linux-x64-musl": "0.1.82",
"@napi-rs/canvas-win32-x64-msvc": "0.1.82"
"@napi-rs/canvas-android-arm64": "0.1.83",
"@napi-rs/canvas-darwin-arm64": "0.1.83",
"@napi-rs/canvas-darwin-x64": "0.1.83",
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.83",
"@napi-rs/canvas-linux-arm64-gnu": "0.1.83",
"@napi-rs/canvas-linux-arm64-musl": "0.1.83",
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.83",
"@napi-rs/canvas-linux-x64-gnu": "0.1.83",
"@napi-rs/canvas-linux-x64-musl": "0.1.83",
"@napi-rs/canvas-win32-x64-msvc": "0.1.83"
}
},
"node_modules/@napi-rs/canvas-android-arm64": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.82.tgz",
"integrity": "sha512-bvZhN0iI54ouaQOrgJV96H2q7J3ZoufnHf4E1fUaERwW29Rz4rgicohnAg4venwBJZYjGl5Yl3CGmlAl1LZowQ==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.83.tgz",
"integrity": "sha512-TbKM2fh9zXjqFIU8bgMfzG7rkrIYdLKMafgPhFoPwKrpWk1glGbWP7LEu8Y/WrMDqTGFdRqUmuX89yQEzZbkiw==",
"cpu": [
"arm64"
],
@@ -1656,9 +1656,9 @@
}
},
"node_modules/@napi-rs/canvas-darwin-arm64": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.82.tgz",
"integrity": "sha512-InuBHKCyuFqhNwNr4gpqazo5Xp6ltKflqOLiROn4hqAS8u21xAHyYCJRgHwd+a5NKmutFTaRWeUIT/vxWbU/iw==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.83.tgz",
"integrity": "sha512-gp8IDVUloPUmkepHly4xRUOfUJSFNvA4jR7ZRF5nk3YcGzegSFGeICiT4PnYyPgSKEhYAFe1Y2XNy0Mp6Tu8mQ==",
"cpu": [
"arm64"
],
@@ -1672,9 +1672,9 @@
}
},
"node_modules/@napi-rs/canvas-darwin-x64": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.82.tgz",
"integrity": "sha512-aQGV5Ynn96onSXcuvYb2y7TRXD/t4CL2EGmnGqvLyeJX1JLSNisKQlWN/1bPDDXymZYSdUqbXehj5qzBlOx+RQ==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.83.tgz",
"integrity": "sha512-r4ZJxiP9OgUbdGZhPDEXD3hQ0aIPcVaywtcTXvamYxTU/SWKAbKVhFNTtpRe1J30oQ25gWyxTkUKSBgUkNzdnw==",
"cpu": [
"x64"
],
@@ -1688,9 +1688,9 @@
}
},
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.82.tgz",
"integrity": "sha512-YIUpmHWeHGGRhWitT1KJkgj/JPXPfc9ox8oUoyaGPxolLGPp5AxJkq8wIg8CdFGtutget968dtwmx71m8o3h5g==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.83.tgz",
"integrity": "sha512-Uc6aSB05qH1r+9GUDxIE6F5ZF7L0nTFyyzq8ublWUZhw8fEGK8iy931ff1ByGFT04+xHJad1kBcL4R1ZEV8z7Q==",
"cpu": [
"arm"
],
@@ -1704,9 +1704,9 @@
}
},
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.82.tgz",
"integrity": "sha512-AwLzwLBgmvk7kWeUgItOUor/QyG31xqtD26w1tLpf4yE0hiXTGp23yc669aawjB6FzgIkjh1NKaNS52B7/qEBQ==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.83.tgz",
"integrity": "sha512-eEeaJA7V5KOFq7W0GtoRVbd3ak8UZpK+XLkCgUiFGtlunNw+ZZW9Cr/92MXflGe7o3SqqMUg+f975LPxO/vsOQ==",
"cpu": [
"arm64"
],
@@ -1720,9 +1720,9 @@
}
},
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.82.tgz",
"integrity": "sha512-moZWuqepAwWBffdF4JDadt8TgBD02iMhG6I1FHZf8xO20AsIp9rB+p0B8Zma2h2vAF/YMjeFCDmW5un6+zZz9g==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.83.tgz",
"integrity": "sha512-cAvonp5XpbatVGegF9lMQNchs3z5RH6EtamRVnQvtoRtwbzOMcdzwuLBqDBQxQF79MFbuZNkWj3YRJjZCjHVzw==",
"cpu": [
"arm64"
],
@@ -1736,9 +1736,9 @@
}
},
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.82.tgz",
"integrity": "sha512-w9++2df2kG9eC9LWYIHIlMLuhIrKGQYfUxs97CwgxYjITeFakIRazI9LYWgVzEc98QZ9x9GQvlicFsrROV59MQ==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.83.tgz",
"integrity": "sha512-WFUPQ9qZy31vmLxIJ3MfmHw+R2g/mLCgk8zmh7maJW8snV3vLPA7pZfIS65Dc61EVDp1vaBskwQ2RqPPzwkaew==",
"cpu": [
"riscv64"
],
@@ -1752,9 +1752,9 @@
}
},
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.82.tgz",
"integrity": "sha512-lZulOPwrRi6hEg/17CaqdwWEUfOlIJuhXxincx1aVzsVOCmyHf+xFq4i6liJl1P+x2v6Iz2Z/H5zHvXJCC7Bwg==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.83.tgz",
"integrity": "sha512-X9YwIjsuy50WwOyYeNhEHjKHO8rrfH9M4U8vNqLuGmqsZdKua/GrUhdQGdjq7lTgdY3g4+Ta5jF8MzAa7UAs/g==",
"cpu": [
"x64"
],
@@ -1768,9 +1768,9 @@
}
},
"node_modules/@napi-rs/canvas-linux-x64-musl": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.82.tgz",
"integrity": "sha512-Be9Wf5RTv1w6GXlTph55K3PH3vsAh1Ax4T1FQY1UYM0QfD0yrwGdnJ8/fhqw7dEgMjd59zIbjJQC8C3msbGn5g==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.83.tgz",
"integrity": "sha512-Vv2pLWQS8EnlSM1bstJ7vVhKA+mL4+my4sKUIn/bgIxB5O90dqiDhQjUDLP+5xn9ZMestRWDt3tdQEkGAmzq/A==",
"cpu": [
"x64"
],
@@ -1784,9 +1784,9 @@
}
},
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
"version": "0.1.82",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.82.tgz",
"integrity": "sha512-LN/i8VrvxTDmEEK1c10z2cdOTkWT76LlTGtyZe5Kr1sqoSomKeExAjbilnu1+oee5lZUgS5yfZ2LNlVhCeARuw==",
"version": "0.1.83",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.83.tgz",
"integrity": "sha512-K1TtjbScfRNYhq8dengLLufXGbtEtWdUXPV505uLFPovyGHzDUGXLFP/zUJzj6xWXwgUjHNLgEPIt7mye0zr6Q==",
"cpu": [
"x64"
],
@@ -4958,9 +4958,9 @@
}
},
"node_modules/cloudflare-video-element": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/cloudflare-video-element/-/cloudflare-video-element-1.3.4.tgz",
"integrity": "sha512-F9g+tXzGEXI6v6L48qXxr8vnR8+L6yy7IhpJxK++lpzuVekMHTixxH7/dzLuq6OacVGziU4RB5pzZYJ7/LYtJg==",
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/cloudflare-video-element/-/cloudflare-video-element-1.3.5.tgz",
"integrity": "sha512-zj9gjJa6xW8MNrfc4oKuwgGS0njRLpOlQjdifbuNxvy8k4Y3pKCyKCMG2XIsjd2iQGhgjS57b1P5VWdJlxcXBw==",
"license": "MIT"
},
"node_modules/clsx": {
@@ -5187,9 +5187,9 @@
"license": "BSD-2-Clause"
},
"node_modules/dash-video-element": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/dash-video-element/-/dash-video-element-0.3.0.tgz",
"integrity": "sha512-Pe+BxG153n+CH++3gmWMApVXEUs767YGxsRebdNZRSZdXjbv7OGbsitYbjNMC4QAjCWBvBjIclAYV4hoc7OWSQ==",
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/dash-video-element/-/dash-video-element-0.3.1.tgz",
"integrity": "sha512-KSdCd6lqjum4LizHLtB2EGvaGr7YJU7SZekTTDHixRondaRNcm0t9W2V3I7/itNBzQwdDbC1cKkXryc8I8IViA==",
"license": "MIT",
"dependencies": {
"custom-media-element": "^1.4.5",
@@ -7034,9 +7034,9 @@
}
},
"node_modules/hls-video-element": {
"version": "1.5.9",
"resolved": "https://registry.npmjs.org/hls-video-element/-/hls-video-element-1.5.9.tgz",
"integrity": "sha512-hDXhSI3IpSSODJF8ecNzDHKP5cqsouOuKDMjoTexyFePKr9KpXVCPAnVrXFTTH8VbOim4xkLtPkVJFt7J1Rs6w==",
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/hls-video-element/-/hls-video-element-1.5.10.tgz",
"integrity": "sha512-FruzD03CaQlPlNKfXO1njPbo3jCSImAtFwX1OqgFbMllTQzdYqAHODiWan0q3mr1cYCONOWiAz2/nX+2qHHC+g==",
"license": "MIT",
"dependencies": {
"custom-media-element": "^1.4.5",
@@ -8446,12 +8446,12 @@
}
},
"node_modules/media-chrome": {
"version": "4.16.0",
"resolved": "https://registry.npmjs.org/media-chrome/-/media-chrome-4.16.0.tgz",
"integrity": "sha512-c5xpTYcYo9nYsC/G/C1PyOcPXEL6iIaSR9MH3GncVuj4S90aHqvGbsyUWFDPPBKx5sCwWLxDnbszE/24eMT54g==",
"version": "4.16.1",
"resolved": "https://registry.npmjs.org/media-chrome/-/media-chrome-4.16.1.tgz",
"integrity": "sha512-qtFlsy0lNDVCyVo//ZCAfRPKwgehfOYp6rThZzDUuZ5ypv41yqUfAxK+P9TOs+XSVWXATPTT2WRV0fbW0BH4vQ==",
"license": "MIT",
"dependencies": {
"ce-la-react": "^0.3.0"
"ce-la-react": "^0.3.2"
}
},
"node_modules/media-tracks": {
@@ -10207,9 +10207,9 @@
}
},
"node_modules/posthog-js": {
"version": "1.298.0",
"resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.298.0.tgz",
"integrity": "sha512-Zwzsf7TO8qJ6DFLuUlQSsT/5OIOcxSBZlKOSk3satkEnwKdmnBXUuxgVXRHrvq1kj7OB2PVAPgZiQ8iHHj9DRA==",
"version": "1.298.1",
"resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.298.1.tgz",
"integrity": "sha512-MynFhC2HO6sg5moUfpkd0s6RzAqcqFX75kjIi4Xrj2Gl0/YQWYvFUgvv8FCpWPKPs2mdvNWYhs+oqJg0BVVHPw==",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@posthog/core": "1.6.0",
@@ -11209,9 +11209,9 @@
}
},
"node_modules/spotify-audio-element": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/spotify-audio-element/-/spotify-audio-element-1.0.3.tgz",
"integrity": "sha512-I1/qD8cg/UnTlCIMiKSdZUJTyYfYhaqFK7LIVElc48eOqUUbVCaw1bqL8I6mJzdMJTh3eoNyF/ewvB7NoS/g9A==",
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/spotify-audio-element/-/spotify-audio-element-1.0.4.tgz",
"integrity": "sha512-QdKrJPkYCzaNwwz2vN2eDGyoW0KmQFmnwVprB41mpMzj4qujbqr6pegEchQeTn0b5PceKiLoVu0pp2QDpTcWnw==",
"license": "MIT"
},
"node_modules/sprintf-js": {
@@ -11502,9 +11502,9 @@
"license": "MIT"
},
"node_modules/superjson": {
"version": "2.2.5",
"resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.5.tgz",
"integrity": "sha512-zWPTX96LVsA/eVYnqOM2+ofcdPqdS1dAF1LN4TS2/MWuUpfitd9ctTa87wt4xrYnZnkLtS69xpBdSxVBP5Rm6w==",
"version": "2.2.6",
"resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.6.tgz",
"integrity": "sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==",
"license": "MIT",
"dependencies": {
"copy-anything": "^4"
@@ -11641,9 +11641,9 @@
}
},
"node_modules/tiktok-video-element": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/tiktok-video-element/-/tiktok-video-element-0.1.1.tgz",
"integrity": "sha512-BaiVzvNz2UXDKTdSrXzrNf4q6Ecc+/utYUh7zdEu2jzYcJVDoqYbVfUl0bCfMoOeeAqg28vD/yN63Y3E9jOrlA==",
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/tiktok-video-element/-/tiktok-video-element-0.1.2.tgz",
"integrity": "sha512-w6TboLm236XJKKiIXIhCbYCnUxbixBbaAoty0etaEAZ/2kHkVIdfZdv2oouMU/HGMsWCHI/VjQ3wU3MJ+s192Q==",
"license": "MIT"
},
"node_modules/tiktoken": {
@@ -11790,9 +11790,9 @@
}
},
"node_modules/twitch-video-element": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/twitch-video-element/-/twitch-video-element-0.1.5.tgz",
"integrity": "sha512-3UdWMa5ytWFdpgJAM6XEqqRK/1FvWdJVcKDOw4IHBPt4p52E+4fXT42fBdRZFfoxBPXQNZUDDNHFW8wIopD7Og==",
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/twitch-video-element/-/twitch-video-element-0.1.6.tgz",
"integrity": "sha512-X7l8gy+DEFKJ/EztUwaVnAYwQN9fUJxPkOVJj2sE62sGvGU4DNLyvmOsmVulM+8Plc5dMg6hYIMNRAPaH+39Uw==",
"license": "MIT"
},
"node_modules/type-check": {
@@ -12181,9 +12181,9 @@
}
},
"node_modules/vimeo-video-element": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/vimeo-video-element/-/vimeo-video-element-1.6.1.tgz",
"integrity": "sha512-UwDLzhgg98pct1xb6799I1vRDXIzaAX6rs1TG/QOf6y+VrXpTFrI7mYz2gnj9QCtBcGK68f4z64A+MRYRsLJaQ==",
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/vimeo-video-element/-/vimeo-video-element-1.6.2.tgz",
"integrity": "sha512-/l+/ugpFdCMLUDfUanzP8aSxiaNNsWvecnoIi1ziWd1shS0q7UAG3dqV9mFwO6l2GMOKtpl9uPHNHpV6RloSQA==",
"license": "MIT",
"dependencies": {
"@vimeo/player": "2.29.0"
@@ -12432,9 +12432,9 @@
}
},
"node_modules/wistia-video-element": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/wistia-video-element/-/wistia-video-element-1.3.5.tgz",
"integrity": "sha512-aIG0xEtclPb9xfklAkOwHFv/BMiH3Ql0yWWKQ1XyUCoSDaF3sOD+JNLmakOChvn2LLUX7FqH/mYb8bXT4ACnMw==",
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/wistia-video-element/-/wistia-video-element-1.3.6.tgz",
"integrity": "sha512-wPizIpXDaCs6fvDzhU3MBtEpxIqhgXlu00kSrKgmjPb5DRqZt927xZZjE1qm81Df40d445u4a/mRKX5I66zaYA==",
"license": "MIT",
"dependencies": {
"super-media-element": "~1.4.2"
@@ -12589,9 +12589,9 @@
}
},
"node_modules/youtube-video-element": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/youtube-video-element/-/youtube-video-element-1.8.0.tgz",
"integrity": "sha512-u3M0MgO+KUtVwIyKJXZXXJ0As0k6d5NflOrh1GjyG8NNOp+liW2nFU29hpXeUcxUWbVKhudIYd39hMVeEgCilQ==",
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/youtube-video-element/-/youtube-video-element-1.8.1.tgz",
"integrity": "sha512-+5UuAGaj+5AnBf39huLVpy/4dLtR0rmJP1TxOHVZ81bac4ZHFpTtQ4Dz2FAn2GPnfXISezvUEaQoAdFW4hH9Xg==",
"license": "MIT"
},
"node_modules/zlibjs": {
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "big-agi",
"version": "2.0.1",
"version": "2.0.2",
"private": true,
"author": "Enrico Ros <enrico.ros@gmail.com>",
"repository": "https://github.com/enricoros/big-agi",
@@ -54,7 +54,7 @@
"next": "~15.1.8",
"nprogress": "^0.2.0",
"pdfjs-dist": "5.4.54",
"posthog-js": "^1.298.0",
"posthog-js": "^1.298.1",
"posthog-node": "^5.14.0",
"prismjs": "^1.30.0",
"puppeteer-core": "^24.31.0",
@@ -70,7 +70,7 @@
"remark-mark-highlight": "^0.1.1",
"remark-math": "^6.0.0",
"sharp": "^0.34.5",
"superjson": "^2.2.5",
"superjson": "^2.2.6",
"tesseract.js": "^6.0.1",
"tiktoken": "^1.0.22",
"turndown": "^7.2.2",
+1 -2
View File
@@ -18,7 +18,7 @@ import { ROUTE_APP_CHAT, ROUTE_INDEX } from '~/common/app.routes';
import { Release } from '~/common/app.release';
// capabilities access
import { useCapabilityBrowserSpeechRecognition, useCapabilityElevenLabs, useCapabilityTextToImage } from '~/common/components/useCapabilities';
import { useCapabilityBrowserSpeechRecognition, useCapabilityTextToImage } from '~/common/components/useCapabilities';
// stores access
import { getLLMsDebugInfo } from '~/common/stores/llms/store-llms';
@@ -95,7 +95,6 @@ function AppDebug() {
const cProduct = {
capabilities: {
mic: useCapabilityBrowserSpeechRecognition(),
elevenLabs: useCapabilityElevenLabs(),
textToImage: useCapabilityTextToImage(),
},
models: getLLMsDebugInfo(),
+15 -12
View File
@@ -6,13 +6,15 @@ import ChatIcon from '@mui/icons-material/Chat';
import CheckRoundedIcon from '@mui/icons-material/CheckRounded';
import CloseRoundedIcon from '@mui/icons-material/CloseRounded';
import MicIcon from '@mui/icons-material/Mic';
import RecordVoiceOverTwoToneIcon from '@mui/icons-material/RecordVoiceOverTwoTone';
import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
import { useSpeexGlobalEngine } from '~/modules/speex/store-module-speex';
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
import { animationColorRainbow } from '~/common/util/animUtils';
import { navigateBack } from '~/common/app.routes';
import { optimaOpenPreferences } from '~/common/layout/optima/useOptima';
import { useCapabilityBrowserSpeechRecognition, useCapabilityElevenLabs } from '~/common/components/useCapabilities';
import { useCapabilityBrowserSpeechRecognition } from '~/common/components/useCapabilities';
import { useChatStore } from '~/common/stores/chat/store-chats';
import { useUICounter } from '~/common/stores/store-ui';
@@ -45,7 +47,7 @@ export function CallWizard(props: { strict?: boolean, conversationId: string | n
// external state
const recognition = useCapabilityBrowserSpeechRecognition();
const synthesis = useCapabilityElevenLabs();
const speexGlobalEngine = useSpeexGlobalEngine();
const chatIsEmpty = useChatStore(state => {
if (!props.conversationId)
return false;
@@ -58,15 +60,16 @@ export function CallWizard(props: { strict?: boolean, conversationId: string | n
const outOfTheBlue = !props.conversationId;
const overriddenEmptyChat = chatEmptyOverride || !chatIsEmpty;
const overriddenRecognition = recognitionOverride || recognition.mayWork;
const allGood = overriddenEmptyChat && overriddenRecognition && synthesis.mayWork;
const fatalGood = overriddenRecognition && synthesis.mayWork;
const synthesisShallWork = !!speexGlobalEngine;
const allGood = overriddenEmptyChat && overriddenRecognition && synthesisShallWork;
const fatalGood = overriddenRecognition && synthesisShallWork;
const handleOverrideChatEmpty = React.useCallback(() => setChatEmptyOverride(true), []);
const handleOverrideRecognition = React.useCallback(() => setRecognitionOverride(true), []);
const handleConfigureElevenLabs = React.useCallback(() => optimaOpenPreferences('voice'), []);
const handleConfigureVoice = React.useCallback(() => optimaOpenPreferences('voice'), []);
const handleFinishButton = React.useCallback(() => {
if (!allGood)
@@ -128,17 +131,17 @@ export function CallWizard(props: { strict?: boolean, conversationId: string | n
{/* Text to Speech status */}
<StatusCard
icon={<RecordVoiceOverTwoToneIcon />}
icon={<PhVoice />}
text={
(synthesis.mayWork ? 'Voice synthesis should be ready.' : 'There might be an issue with ElevenLabs voice synthesis.')
+ (synthesis.isConfiguredServerSide ? '' : (synthesis.isConfiguredClientSide ? '' : ' Please add your API key in the settings.'))
(synthesisShallWork ? 'Voice synthesis should be ready.' : 'There might be an issue with voice synthesis.')
// + (synthesis.isConfiguredServerSide ? '' : (synthesis.isConfiguredClientSide ? '' : ' Please add your API key in the settings.'))
}
button={synthesis.mayWork ? undefined : (
<Button variant='outlined' onClick={handleConfigureElevenLabs} sx={{ mx: 1 }}>
button={synthesisShallWork ? undefined : (
<Button variant='outlined' onClick={handleConfigureVoice} sx={{ mx: 1 }}>
Configure
</Button>
)}
hasIssue={!synthesis.mayWork}
hasIssue={!synthesisShallWork}
/>
{/*<Typography>*/}
+1 -1
View File
@@ -317,7 +317,7 @@ export function Contacts(props: { setCallIntent: (intent: AppCallIntent) => void
issue={354}
text='Call App: Support thread and compatibility matrix'
note={<>
Voice input uses the HTML Web Speech API, and speech output requires an ElevenLabs API Key.
Voice input uses the HTML Web Speech API.
</>}
// note2='Please report any issues you encounter'
sx={{
+17 -30
View File
@@ -7,22 +7,22 @@ import CallEndIcon from '@mui/icons-material/CallEnd';
import CallIcon from '@mui/icons-material/Call';
import MicIcon from '@mui/icons-material/Mic';
import MicNoneIcon from '@mui/icons-material/MicNone';
import RecordVoiceOverTwoToneIcon from '@mui/icons-material/RecordVoiceOverTwoTone';
import { ScrollToBottom } from '~/common/scroll-to-bottom/ScrollToBottom';
import { ScrollToBottomButton } from '~/common/scroll-to-bottom/ScrollToBottomButton';
import { useChatLLMDropdown } from '../chat/components/layout-bar/useLLMDropdown';
import { SystemPurposeId, SystemPurposes } from '../../data';
import { elevenLabsSpeakText } from '~/modules/elevenlabs/elevenlabs.client';
import { AixChatGenerateContent_DMessageGuts, aixChatGenerateContent_DMessage_FromConversation } from '~/modules/aix/client/aix.client';
import { useElevenLabsVoiceDropdown } from '~/modules/elevenlabs/useElevenLabsVoiceDropdown';
import { aixChatGenerateContent_DMessage_FromConversation, AixChatGenerateContent_DMessageGuts } from '~/modules/aix/client/aix.client';
import { speakText } from '~/modules/speex/speex.client';
import type { OptimaBarControlMethods } from '~/common/layout/optima/bar/OptimaBarDropdown';
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
import { Link } from '~/common/components/Link';
import { OptimaPanelGroupedList } from '~/common/layout/optima/panel/OptimaPanelGroupedList';
import { OptimaPanelIn, OptimaToolbarIn } from '~/common/layout/optima/portals/OptimaPortalsIn';
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
import { SpeechResult, useSpeechRecognition } from '~/common/components/speechrecognition/useSpeechRecognition';
import { conversationTitle, remapMessagesSysToUsr } from '~/common/stores/chat/chat.conversation';
import { createDMessageFromFragments, createDMessageTextContent, DMessage, messageFragmentsReduceText, messageWasInterruptedAtStart } from '~/common/stores/chat/chat.message';
@@ -43,18 +43,13 @@ import { useAppCallStore } from './state/store-app-call';
function CallMenu(props: {
pushToTalk: boolean,
setPushToTalk: (pushToTalk: boolean) => void,
override: boolean,
setOverride: (overridePersonaVoice: boolean) => void,
}) {
// external state
const { grayUI, toggleGrayUI } = useAppCallStore();
const { voicesDropdown } = useElevenLabsVoiceDropdown(false, !props.override);
const handlePushToTalkToggle = () => props.setPushToTalk(!props.pushToTalk);
const handleChangeVoiceToggle = () => props.setOverride(!props.override);
return <OptimaPanelGroupedList title='Call'>
<MenuItem onClick={handlePushToTalkToggle}>
@@ -63,17 +58,6 @@ function CallMenu(props: {
<Switch checked={props.pushToTalk} onChange={handlePushToTalkToggle} sx={{ ml: 'auto' }} />
</MenuItem>
<MenuItem onClick={handleChangeVoiceToggle}>
<ListItemDecorator><RecordVoiceOverTwoToneIcon /></ListItemDecorator>
Change Voice
<Switch checked={props.override} onChange={handleChangeVoiceToggle} sx={{ ml: 'auto' }} />
</MenuItem>
<MenuItem>
<ListItemDecorator>{' '}</ListItemDecorator>
{voicesDropdown}
</MenuItem>
<ListDivider />
<MenuItem onClick={toggleGrayUI}>
@@ -98,7 +82,6 @@ export function Telephone(props: {
const [avatarClickCount, setAvatarClickCount] = React.useState<number>(0);// const [micMuted, setMicMuted] = React.useState(false);
const [callElapsedTime, setCallElapsedTime] = React.useState<string>('00:00');
const [callMessages, setCallMessages] = React.useState<DMessage[]>([]);
const [overridePersonaVoice, setOverridePersonaVoice] = React.useState<boolean>(false);
const [personaTextInterim, setPersonaTextInterim] = React.useState<string | null>(null);
const [pushToTalk, setPushToTalk] = React.useState(true);
const [stage, setStage] = React.useState<'ring' | 'declined' | 'connected' | 'ended'>('ring');
@@ -118,7 +101,7 @@ export function Telephone(props: {
}));
const persona = SystemPurposes[props.callIntent.personaId as SystemPurposeId] ?? undefined;
const personaCallStarters = persona?.call?.starters ?? undefined;
const personaVoiceId = overridePersonaVoice ? undefined : (persona?.voices?.elevenLabs?.voiceId ?? undefined);
// const personaVoiceSelector = React.useMemo(() => personaGetVoiceSelector(persona), [persona]);
const personaSystemMessage = persona?.systemMessage ?? undefined;
// hooks and speech
@@ -165,7 +148,6 @@ export function Telephone(props: {
};
// [E] pickup -> seed message and call timer
// FIXME: Overriding the voice will reset the call - not a desired behavior
React.useEffect(() => {
if (!isConnected) return;
@@ -185,11 +167,14 @@ export function Telephone(props: {
setCallMessages([createDMessageTextContent('assistant', firstMessage)]); // [state] set assistant:hello message
// fire/forget
void elevenLabsSpeakText(firstMessage, personaVoiceId, true, true);
// fire/forget - use 'fast' priority for real-time conversation
void speakText(firstMessage,
undefined,
{ label: 'Call', priority: 'fast' },
);
return () => clearInterval(interval);
}, [isConnected, personaCallStarters, personaVoiceId]);
}, [isConnected, personaCallStarters]);
// [E] persona streaming response - upon new user message
React.useEffect(() => {
@@ -270,9 +255,12 @@ export function Telephone(props: {
fullMessage.generator = status.lastDMessage.generator;
setCallMessages(messages => [...messages, fullMessage]); // [state] append assistant:call_response
// fire/forget
// fire/forget - use 'fast' priority for real-time conversation
if (status.outcome === 'success' && finalText?.length >= 1)
void elevenLabsSpeakText(finalText, personaVoiceId, true, true);
void speakText(finalText,
undefined,
{ label: 'Call', priority: 'fast' },
);
}).catch((err: DOMException) => {
if (err?.name !== 'AbortError') {
@@ -288,7 +276,7 @@ export function Telephone(props: {
responseAbortController.current?.abort();
responseAbortController.current = null;
};
}, [isConnected, callMessages, modelId, personaVoiceId, personaSystemMessage, reMessages]);
}, [callMessages, isConnected, modelId, personaSystemMessage, reMessages]);
// [E] Message interrupter
const abortTrigger = isConnected && recognitionState.hasSpeech;
@@ -325,7 +313,6 @@ export function Telephone(props: {
<OptimaPanelIn>
<CallMenu
pushToTalk={pushToTalk} setPushToTalk={setPushToTalk}
override={overridePersonaVoice} setOverride={setOverridePersonaVoice}
/>
</OptimaPanelIn>
-7
View File
@@ -10,7 +10,6 @@ import type { DiagramConfig } from '~/modules/aifn/digrams/DiagramsModal';
import type { TradeConfig } from '~/modules/trade/TradeModal';
import { downloadSingleChat, importConversationsFromFilesAtRest, openConversationsAtRestPicker } from '~/modules/trade/trade.client';
import { imaginePromptFromTextOrThrow } from '~/modules/aifn/imagine/imaginePromptFromText';
import { elevenLabsSpeakText } from '~/modules/elevenlabs/elevenlabs.client';
import { useAreBeamsOpen } from '~/modules/beam/store-beam.hooks';
import { useCapabilityTextToImage } from '~/modules/t2i/t2i.client';
@@ -346,11 +345,6 @@ export function AppChat() {
});
}, [handleExecuteAndOutcome]);
const handleTextSpeak = React.useCallback(async (text: string): Promise<void> => {
await elevenLabsSpeakText(text, undefined, true, true);
}, []);
// Chat actions
const handleConversationNewInFocusedPane = React.useCallback((forceNoRecycle: boolean, isIncognito: boolean) => {
@@ -725,7 +719,6 @@ export function AppChat() {
onConversationNew={handleConversationNewInFocusedPane}
onTextDiagram={handleTextDiagram}
onTextImagine={handleImagineFromText}
onTextSpeak={handleTextSpeak}
sx={chatMessageListSx}
/>
)}
+10 -9
View File
@@ -7,6 +7,7 @@ import { Box, List } from '@mui/joy';
import type { SystemPurposeExample } from '../../../data';
import type { DiagramConfig } from '~/modules/aifn/digrams/DiagramsModal';
import { speakText } from '~/modules/speex/speex.client';
import type { ConversationHandler } from '~/common/chat-overlay/ConversationHandler';
import type { DLLMContextTokens } from '~/common/stores/llms/llms.types';
@@ -17,7 +18,6 @@ import { createDMessageFromFragments, createDMessageTextContent, DMessage, DMess
import { createTextContentFragment, DMessageFragment, DMessageFragmentId } from '~/common/stores/chat/chat.fragments';
import { openFileForAttaching } from '~/common/components/ButtonAttachFiles';
import { optimaOpenPreferences } from '~/common/layout/optima/useOptima';
import { useCapabilityElevenLabs } from '~/common/components/useCapabilities';
import { useChatOverlayStore } from '~/common/chat-overlay/store-perchat_vanilla';
import { useChatStore } from '~/common/stores/chat/store-chats';
import { useScrollToBottom } from '~/common/scroll-to-bottom/useScrollToBottom';
@@ -50,7 +50,6 @@ export function ChatMessageList(props: {
onConversationNew: (forceNoRecycle: boolean, isIncognito: boolean) => void,
onTextDiagram: (diagramConfig: DiagramConfig | null) => void,
onTextImagine: (conversationId: DConversationId, selectedText: string) => Promise<void>,
onTextSpeak: (selectedText: string) => Promise<void>,
setIsMessageSelectionMode: (isMessageSelectionMode: boolean) => void,
sx?: SxProps,
}) {
@@ -75,10 +74,9 @@ export function ChatMessageList(props: {
_composerInReferenceToCount: state.inReferenceTo?.length ?? 0,
ephemerals: state.ephemerals?.length ? state.ephemerals : null,
})));
const { mayWork: isSpeakable } = useCapabilityElevenLabs();
// derived state
const { conversationHandler, conversationId, capabilityHasT2I, onConversationBranch, onConversationExecuteHistory, onTextDiagram, onTextImagine, onTextSpeak } = props;
const { conversationHandler, conversationId, capabilityHasT2I, onConversationBranch, onConversationExecuteHistory, onTextDiagram, onTextImagine } = props;
const composerCanAddInReferenceTo = _composerInReferenceToCount < 5;
const composerHasInReferenceto = _composerInReferenceToCount > 0;
@@ -212,12 +210,15 @@ export function ChatMessageList(props: {
}, [capabilityHasT2I, conversationId, onTextImagine]);
const handleTextSpeak = React.useCallback(async (text: string) => {
if (!isSpeakable)
return optimaOpenPreferences('voice');
// sandwich the speaking with the indicator
setIsSpeaking(true);
await onTextSpeak(text);
const result = await speakText(text, undefined, { label: 'Chat speak' });
setIsSpeaking(false);
}, [isSpeakable, onTextSpeak]);
// open voice preferences
if (!result.success && (result.errorType === 'tts-no-engine' || result.errorType === 'tts-unconfigured'))
optimaOpenPreferences('voice');
}, []);
// operate on the local selection set
@@ -377,7 +378,7 @@ export function ChatMessageList(props: {
onMessageTruncate={handleMessageTruncate}
onTextDiagram={handleTextDiagram}
onTextImagine={capabilityHasT2I ? handleTextImagine : undefined}
onTextSpeak={isSpeakable ? handleTextSpeak : undefined}
onTextSpeak={handleTextSpeak}
/>
);
@@ -905,7 +905,7 @@ export function Composer(props: {
)}
{!showChatInReferenceTo && !isDraw && tokenLimit > 0 && (
<TokenBadgeMemo hideBelowDollars={0.005} chatPricing={tokenChatPricing} direct={tokensComposer} history={tokensHistory} responseMax={tokensResponseMax} limit={tokenLimit} showCost={labsShowCost} enableHover={!isMobile} showExcess absoluteBottomRight />
<TokenBadgeMemo hideBelowDollars={0.01} chatPricing={tokenChatPricing} direct={tokensComposer} history={tokensHistory} responseMax={tokensResponseMax} limit={tokenLimit} showCost={labsShowCost} enableHover={!isMobile} showExcess absoluteBottomRight />
)}
</Box>
@@ -47,9 +47,9 @@ function TokenBadge(props: {
const showAltCosts = !!props.showCost && !!costMax && costMin !== undefined;
if (showAltCosts) {
// Note: switched to 'min cost (>= ...)' on mobile as well, to restore the former behavior, just uncomment the !props.enableHover (a proxy for isMobile)
badgeValue = (/*!props.enableHover ||*/ isHovering)
? '< ' + formatModelsCost(costMax)
: '> ' + formatModelsCost(costMin);
badgeValue =
// (/*!props.enableHover ||*/ isHovering) ? '< ' + formatModelsCost(costMax) :
'> ' + formatModelsCost(costMin);
} else {
// show the direct tokens, unless we exceed the limit and 'showExcess' is enabled
@@ -77,7 +77,7 @@ function TokenBadge(props: {
slotProps={{
root: {
sx: {
...((props.absoluteBottomRight) && { position: 'absolute', bottom: 8, right: 8 }),
...((props.absoluteBottomRight) && { position: 'absolute', bottom: 8, right: '1rem' }),
cursor: 'help',
...(shallInvisible && {
opacity: 0,
@@ -92,6 +92,13 @@ function TokenBadge(props: {
fontFamily: 'code',
fontSize: 'xs',
...((props.absoluteBottomRight || props.inline) && { position: 'static', transform: 'none' }),
// make it transparent over text
// backgroundColor: `rgb(var(--joy-palette-${color}-lightChannel) / 15%)`, // similar to success.50
background: 'transparent',
boxShadow: 'none', // outline
'&:hover': {
backgroundColor: `${color}.softHoverBg`,
},
},
},
}}
@@ -21,7 +21,6 @@ import InsertLinkIcon from '@mui/icons-material/InsertLink';
import MoreVertIcon from '@mui/icons-material/MoreVert';
import NotificationsActiveIcon from '@mui/icons-material/NotificationsActive';
import NotificationsOutlinedIcon from '@mui/icons-material/NotificationsOutlined';
import RecordVoiceOverOutlinedIcon from '@mui/icons-material/RecordVoiceOverOutlined';
import ReplayIcon from '@mui/icons-material/Replay';
import ReplyAllRoundedIcon from '@mui/icons-material/ReplyAllRounded';
import ReplyRoundedIcon from '@mui/icons-material/ReplyRounded';
@@ -40,6 +39,7 @@ import { CloseablePopup } from '~/common/components/CloseablePopup';
import { DMessage, DMessageId, DMessageUserFlag, DMetaReferenceItem, MESSAGE_FLAG_AIX_SKIP, MESSAGE_FLAG_NOTIFY_COMPLETE, MESSAGE_FLAG_STARRED, MESSAGE_FLAG_VND_ANT_CACHE_AUTO, MESSAGE_FLAG_VND_ANT_CACHE_USER, messageFragmentsReduceText, messageHasUserFlag } from '~/common/stores/chat/chat.message';
import { KeyStroke } from '~/common/components/KeyStroke';
import { MarkHighlightIcon } from '~/common/components/icons/MarkHighlightIcon';
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
import { Release } from '~/common/app.release';
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
import { adjustContentScaling, themeScalingMap, themeZIndexChatBubble } from '~/common/app.theme';
@@ -1027,7 +1027,7 @@ export function ChatMessage(props: {
)}
{!!props.onTextSpeak && (
<MenuItem onClick={handleOpsSpeak} disabled={!couldSpeak || props.isSpeaking}>
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <RecordVoiceOverOutlinedIcon />}</ListItemDecorator>
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <PhVoice />}</ListItemDecorator>
Speak
</MenuItem>
)}
@@ -1155,7 +1155,7 @@ export function ChatMessage(props: {
</Tooltip>}
{!!props.onTextSpeak && <Tooltip disableInteractive arrow placement='top' title='Speak'>
<IconButton color='success' onClick={handleOpsSpeak} disabled={!couldSpeak || props.isSpeaking}>
{!props.isSpeaking ? <RecordVoiceOverOutlinedIcon /> : <CircularProgress sx={{ '--CircularProgress-size': '16px' }} />}
{!props.isSpeaking ? <PhVoice /> : <CircularProgress sx={{ '--CircularProgress-size': '16px' }} />}
</IconButton>
</Tooltip>}
{(!!props.onTextDiagram || !!props.onTextImagine || !!props.onTextSpeak) && <Divider />}
@@ -1195,7 +1195,7 @@ export function ChatMessage(props: {
Auto-Draw
</MenuItem>}
{!!props.onTextSpeak && <MenuItem onClick={handleOpsSpeak} disabled={!couldSpeak || props.isSpeaking}>
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <RecordVoiceOverOutlinedIcon />}</ListItemDecorator>
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <PhVoice />}</ListItemDecorator>
Speak
</MenuItem>}
</CloseablePopup>
@@ -7,13 +7,13 @@ import CodeIcon from '@mui/icons-material/Code';
import EditRoundedIcon from '@mui/icons-material/EditRounded';
import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined';
import PictureAsPdfIcon from '@mui/icons-material/PictureAsPdf';
import RecordVoiceOverOutlinedIcon from '@mui/icons-material/RecordVoiceOverOutlined';
import TextFieldsIcon from '@mui/icons-material/TextFields';
import TextureIcon from '@mui/icons-material/Texture';
import { ContentScaling, themeScalingMap } from '~/common/app.theme';
import { DMessageAttachmentFragment, DMessageFragmentId, DVMimeType, isDocPart } from '~/common/stores/chat/chat.fragments';
import { LiveFileIcon } from '~/common/livefile/liveFile.icons';
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
import { ellipsizeMiddle } from '~/common/util/textUtils';
import { useLiveFileMetadata } from '~/common/livefile/useLiveFileMetadata';
@@ -41,7 +41,7 @@ export function buttonIconForFragment(part: DMessageAttachmentFragment['part']):
case 'image':
return ImageOutlinedIcon;
case 'audio':
return RecordVoiceOverOutlinedIcon;
return PhVoice;
default:
const _exhaustiveCheck: never = assetType;
return TextureIcon; // missing zync asset type
@@ -6,6 +6,7 @@ import type { ContentScaling } from '~/common/app.theme';
import type { DMessageErrorPart } from '~/common/stores/chat/chat.fragments';
import type { DMessageRole } from '~/common/stores/chat/chat.message';
import { BlockPartError_NetDisconnected } from './BlockPartError_NetDisconnected';
import { BlockPartError_RequestExceeded } from './BlockPartError_RequestExceeded';
@@ -19,6 +20,19 @@ export function BlockPartError(props: {
// special error presentation, based on hints
switch (props.errorHint) {
case 'aix-net-disconnected':
// determine the 2 'kinds' of disconnection errors in aix.client.ts
const kind =
props.errorText.includes('**network error**') ? 'net-client-closed'
: props.errorText.includes('**connection terminated**') ? 'net-server-closed'
: 'net-unknown-closed';
// For client-side error, we don't show the _NetDisconnected component
if (kind === 'net-client-closed')
break;
return <BlockPartError_NetDisconnected disconnectionKind={kind} messageGeneratorLlmId={props.messageGeneratorLlmId} contentScaling={props.contentScaling} />;
case 'aix-request-exceeded':
return <BlockPartError_RequestExceeded messageGeneratorLlmId={props.messageGeneratorLlmId} contentScaling={props.contentScaling} />;
@@ -0,0 +1,103 @@
import * as React from 'react';
import { Alert, Box, FormHelperText, Switch } from '@mui/joy';
import WifiOffRoundedIcon from '@mui/icons-material/WifiOffRounded';
import type { ContentScaling } from '~/common/app.theme';
import { useLLM } from '~/common/stores/llms/llms.hooks';
import { useModelServiceClientSideFetch } from '~/common/stores/llms/hooks/useModelServiceClientSideFetch';
/**
* Error recovery component for "Connection terminated" errors.
*/
export function BlockPartError_NetDisconnected(props: {
disconnectionKind: 'net-client-closed' | 'net-server-closed' | 'net-unknown-closed';
messageGeneratorLlmId?: string | null;
contentScaling: ContentScaling;
}) {
// external state
const model = useLLM(props.messageGeneratorLlmId) ?? null;
const isServerSideClosed = props.disconnectionKind === 'net-server-closed'; // do not show CSF option for non-server-side
const { csfAvailable, csfActive, csfToggle, vendorName } = useModelServiceClientSideFetch(isServerSideClosed, model);
return (
<Alert
size={props.contentScaling === 'xs' ? 'sm' : 'md'}
color='danger'
variant='plain'
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1 }}
>
<Box sx={{ flex: 1, display: 'flex', flexDirection: 'column', gap: 0.5, alignItems: 'flex-start' }}>
{/* Header */}
<Box sx={{ display: 'flex', gap: 2 }}>
<WifiOffRoundedIcon sx={{ flexShrink: 0, mt: 0.5 }} />
<div>
<Box fontSize='larger'>
Connection Terminated
</Box>
<div>
The connection was unexpectedly closed before the response completed.
</div>
</div>
</Box>
{/* Recovery options */}
{csfAvailable ? <>
{/* Explanation */}
<Box color='text.tertiary' fontSize='sm' my={2}>
<strong>Experimental:</strong> enable direct connection to {vendorName} to bypass server timeouts - then try again.
</Box>
{/* Toggle */}
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 2,
p: 2,
borderRadius: 'sm',
bgcolor: 'background.popup',
boxShadow: 'md',
// border: '1px solid',
// borderColor: 'divider',
}}
>
<Box sx={{ flex: 1 }}>
<Box color={!csfActive ? undefined : 'primary.solidBg'} fontWeight='lg' mb={0.5}>
Direct Connection {csfActive && '- Now Try Again'}
</Box>
<FormHelperText>
Connect directly from this client -&gt; {vendorName || 'AI service'}
</FormHelperText>
</Box>
<Switch
checked={csfActive}
onChange={(e) => csfToggle(e.target.checked)}
/>
</Box>
</> : (
<div>
<Box sx={{ color: 'text.secondary', my: 1 }}>
Suggestions:
</Box>
<Box component='ul' sx={{ color: 'text.secondary' }}>
<li>Check your internet connection and try again</li>
<li>The AI service may be experiencing issues - wait a moment and retry</li>
<li>If the issue persists, please let us know promptly on Discord or GitHib</li>
</Box>
</div>
)}
</Box>
</Alert>
);
}
@@ -1,6 +1,6 @@
import * as React from 'react';
import { Alert, Box, FormHelperText, Switch, Typography } from '@mui/joy';
import { Alert, Box, FormHelperText, Switch } from '@mui/joy';
import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
import type { ContentScaling } from '~/common/app.theme';
@@ -19,13 +19,13 @@ export function BlockPartError_RequestExceeded(props: {
// external state
const model = useLLM(props.messageGeneratorLlmId) ?? null;
const { csfAvailable, csfActive, csfToggle } = useModelServiceClientSideFetch(true, model);
const { csfAvailable, csfActive, csfToggle, vendorName } = useModelServiceClientSideFetch(true, model);
return (
<Alert
size={props.contentScaling === 'xs' ? 'sm' : 'md'}
color='danger'
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1 }}
color='warning'
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1, border: '1px solid', borderColor: 'warning.outlinedBorder' }}
>
<WarningRoundedIcon sx={{ flexShrink: 0, mt: 0.25 }} />
@@ -36,70 +36,69 @@ export function BlockPartError_RequestExceeded(props: {
Request Too Large
</Box>
<div>
Your message or attachments exceed the limit of the Vercel edge network.
Your message or attachments exceed the limit of the Vercel edge network
</div>
{/* Recovery options */}
{csfAvailable ? (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
{csfAvailable ? <>
{/* Explanation */}
<Box color='text.secondary' fontSize='sm'>
<strong>Experimental:</strong> enable direct connection to the AI services, and try again.
</Box>
{/* Toggle */}
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 2,
p: 2,
borderRadius: 'sm',
bgcolor: 'background.popup',
boxShadow: 'md',
}}
>
<Box sx={{ flex: 1 }}>
<Box color={!csfActive ? undefined : 'primary.solidBg'} fontWeight='lg' mb={0.5}>
Direct Connection {csfActive && '- Now Try Again'}
</Box>
<FormHelperText>
Bypassing servers and connect directly from this client -&gt; AI provider
</FormHelperText>
</Box>
<Switch
checked={csfActive}
onChange={(e) => csfToggle(e.target.checked)}
/>
</Box>
{/* Regenerate button */}
{/*{props.onRegenerate && (*/}
{/* <Button*/}
{/* size='sm'*/}
{/* variant={csfActive ? 'solid' : 'outlined'}*/}
{/* color={csfActive ? 'success' : 'neutral'}*/}
{/* startDecorator={<RefreshIcon />}*/}
{/* onClick={props.onRegenerate}*/}
{/* sx={{ alignSelf: 'flex-start' }}*/}
{/* >*/}
{/* {csfActive ? 'Regenerate with Direct Connection' : 'Regenerate'}*/}
{/* </Button>*/}
{/*)}*/}
{/* Explanation */}
<Box color='text.secondary' fontSize='sm' my={2}>
<strong>Experimental:</strong> enable Direct Connection to {vendorName} to work around size limitations.
</Box>
) : (
{/* Toggle */}
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 2,
p: 2,
borderRadius: 'sm',
bgcolor: 'background.popup',
boxShadow: 'md',
}}
>
<Box sx={{ flex: 1 }}>
<Box color={!csfActive ? undefined : 'primary.solidBg'} fontWeight='lg' mb={0.5}>
Direct Connection {csfActive && '- Now Try Again'}
</Box>
<FormHelperText>
Connect directly from this client -&gt; {vendorName || 'AI service'}
</FormHelperText>
</Box>
<Switch
checked={csfActive}
onChange={(e) => csfToggle(e.target.checked)}
/>
</Box>
{/* Regenerate button */}
{/*{props.onRegenerate && (*/}
{/* <Button*/}
{/* size='sm'*/}
{/* variant={csfActive ? 'solid' : 'outlined'}*/}
{/* color={csfActive ? 'success' : 'neutral'}*/}
{/* startDecorator={<RefreshIcon />}*/}
{/* onClick={props.onRegenerate}*/}
{/* sx={{ alignSelf: 'flex-start' }}*/}
{/* >*/}
{/* {csfActive ? 'Regenerate with Direct Connection' : 'Regenerate'}*/}
{/* </Button>*/}
{/*)}*/}
</> : (
<Box>
<Typography level='body-sm' sx={{ mb: 1 }}>
<strong>Suggestions:</strong>
</Typography>
<Typography component='ul' level='body-sm' sx={{ pl: 2, m: 0 }}>
<Box sx={{ color: 'text.secondary', my: 1 }}>
Suggestions:
</Box>
<Box component='ul' sx={{ color: 'text.secondary' }}>
<li>Use the cleanup button in the right pane to hide old messages</li>
<li>Remove large attachments from the conversation</li>
{/*<li>Reduce conversation length before sending</li>*/}
</Typography>
</Box>
</Box>
)}
</Box>
@@ -1,9 +1,8 @@
import { elevenLabsSpeakText } from '~/modules/elevenlabs/elevenlabs.client';
import type { AixChatGenerateContent_DMessageGuts } from '~/modules/aix/client/aix.client';
import { speakText } from '~/modules/speex/speex.client';
import { isTextContentFragment } from '~/common/stores/chat/chat.fragments';
import type { AixChatGenerateContent_DMessageGuts } from '~/modules/aix/client/aix.client';
import type { PersonaProcessorInterface } from '../chat-persona';
@@ -58,7 +57,7 @@ export class PersonaChatMessageSpeak implements PersonaProcessorInterface {
#speak(text: string) {
console.log('📢 TTS:', text);
this.spokenLine = true;
// fire/forget: we don't want to stall this loop
void elevenLabsSpeakText(text, undefined, false, true);
// fire/forget: we don't want to stall streaming
void speakText(text, undefined, { label: 'Chat message' });
}
}
+5 -3
View File
@@ -72,11 +72,13 @@ export const DevNewsItem: NewsItem = {
// news and feature surfaces
export const NewsItems: NewsItem[] = [
{
versionCode: '2.0.1',
versionCode: '2.0.2',
versionName: 'Heavy Critters',
versionDate: new Date('2025-11-24T23:30:00Z'),
versionDate: new Date('2025-12-01T06:00:00Z'), // 2.0.2
// versionDate: new Date('2025-11-24T23:30:00Z'), // 2.0.1
items: [
{ text: <>New: <B>Opus 4.5</B>, <B>Gemini 3 Pro</B>, <B>Nano Banana Pro</B>, <B>Grok 4.1</B>, <B>GPT-5.1</B>, <B>Kimi K2</B></> },
{ text: <><B>New in 2.0.2</B> Speech synthesis with Web Speech, LocalAI, OpenAI and more</> },
{ text: <><B>Opus 4.5</B>, <B>Gemini 3 Pro</B>, <B>Nano Banana Pro</B>, <B>Grok 4.1</B>, <B>GPT-5.1</B>, <B>Kimi K2</B></> },
{ text: <><B>Image Generation</B> with Azure and LocalAI providers, in addition to OpenAI</> },
{ text: <>Enhanced <B>OpenRouter</B> integration with auto-capabilities and reasoning</> },
{ text: <>Call transcripts, generate persona images, search button in beams</> },
+46 -8
View File
@@ -1,18 +1,17 @@
import * as React from 'react';
import { Accordion, AccordionDetails, accordionDetailsClasses, AccordionGroup, AccordionSummary, accordionSummaryClasses, Avatar, Box, Button, ListItemContent, styled, Tab, TabList, TabPanel, Tabs } from '@mui/joy';
import { Accordion, AccordionDetails, AccordionGroup, AccordionSummary, accordionSummaryClasses, Avatar, Box, Button, ListItemContent, styled, Tab, TabList, TabPanel, Tabs, Typography } from '@mui/joy';
import AddIcon from '@mui/icons-material/Add';
import AutoAwesomeIcon from '@mui/icons-material/AutoAwesome';
import KeyboardCommandKeyOutlinedIcon from '@mui/icons-material/KeyboardCommandKeyOutlined';
import LanguageRoundedIcon from '@mui/icons-material/LanguageRounded';
import MicIcon from '@mui/icons-material/Mic';
import RecordVoiceOverIcon from '@mui/icons-material/RecordVoiceOver';
import ScienceIcon from '@mui/icons-material/Science';
import SearchIcon from '@mui/icons-material/Search';
import TerminalOutlinedIcon from '@mui/icons-material/TerminalOutlined';
import { BrowseSettings } from '~/modules/browse/BrowseSettings';
import { DallESettings } from '~/modules/t2i/dalle/DallESettings';
import { ElevenlabsSettings } from '~/modules/elevenlabs/ElevenlabsSettings';
import { GoogleSearchSettings } from '~/modules/google/GoogleSearchSettings';
import { T2ISettings } from '~/modules/t2i/T2ISettings';
@@ -20,14 +19,15 @@ import type { PreferencesTabId } from '~/common/layout/optima/store-layout-optim
import { AppBreadcrumbs } from '~/common/components/AppBreadcrumbs';
import { DarkModeToggleButton, darkModeToggleButtonSx } from '~/common/components/DarkModeToggleButton';
import { GoodModal } from '~/common/components/modals/GoodModal';
import { Is } from '~/common/util/pwaUtils';
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
import { optimaActions } from '~/common/layout/optima/useOptima';
import { useIsMobile } from '~/common/components/useMatchMedia';
import { AppChatSettingsAI } from './AppChatSettingsAI';
import { AppChatSettingsUI } from './settings-ui/AppChatSettingsUI';
import { UxLabsSettings } from './UxLabsSettings';
import { VoiceSettings } from './VoiceSettings';
import { VoiceInSettings } from './VoiceInSettings';
import { VoiceOutSettings } from './VoiceOutSettings';
// configuration
@@ -271,10 +271,10 @@ export function SettingsModal(props: {
<TabPanel value='voice' color='primary' variant='outlined' sx={_styles.tabPanel}>
<Topics>
<Topic icon={/*'🎙️'*/ <MicIcon />} title='Microphone'>
<VoiceSettings />
<VoiceInSettings isMobile={isMobile} />
</Topic>
<Topic icon={/*'📢'*/ <RecordVoiceOverIcon />} title='ElevenLabs API'>
<ElevenlabsSettings />
<Topic icon={/*'📢'*/ <PhVoice />} title={'Speech'/*<>Voices <GoodBadge badge='New' /></>*/}>
<VoiceOutSettings isMobile={isMobile} />
</Topic>
</Topics>
</TabPanel>
@@ -291,6 +291,44 @@ export function SettingsModal(props: {
</TabPanel>
<TabPanel value='tools' color='primary' variant='outlined' sx={_styles.tabPanel}>
{/* Search Modifier Info */}
<Box sx={{
p: 2,
borderRadius: 'calc(var(--joy-radius-md) - 1px)',
// backgroundColor: 'background.level1',
display: 'flex',
alignItems: 'center',
gap: 2,
}}>
<Button
variant='soft'
color='success'
startDecorator={<SearchIcon />}
sx={{
// this is copied frmo ButtonSearchControl._styles.desktop
minWidth: 100,
justifyContent: 'flex-start',
borderRadius: '18px',
pointerEvents: 'none',
'[data-joy-color-scheme="light"] &': {
bgcolor: '#d5ec31',
},
boxShadow: 'inset 0 2px 4px -1px rgba(0,0,0,0.15)',
textWrap: 'nowrap',
}}
>
Search
</Button>
<Box sx={{ flex: 1 }}>
<Typography level='body-sm' sx={{ fontWeight: 'md', mb: 0.5 }}>
Use the Search button
</Typography>
<Typography level='body-xs' sx={{ color: 'text.secondary' }}>
Modern AI models have native search built-in. Click the Search button when chatting to enable real-time web search.
</Typography>
</Box>
</Box>
<Topics>
<Topic icon={<LanguageRoundedIcon />} title='Load Web Pages (with images)' startCollapsed>
<BrowseSettings />
@@ -0,0 +1,54 @@
import * as React from 'react';
import { FormControl } from '@mui/joy';
import { useChatMicTimeoutMs } from '../chat/store-app-chat';
import type { FormRadioOption } from '~/common/components/forms/FormRadioControl';
import { FormChipControl } from '~/common/components/forms/FormChipControl';
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
import { LanguageSelect } from '~/common/components/LanguageSelect';
const _minTimeouts: ReadonlyArray<FormRadioOption<string>> = [
{ value: '600', label: '0.6s', description: 'Best for quick calls' },
{ value: '2000', label: '2s', description: 'Standard' },
{ value: '5000', label: '5s', description: 'Breathe' },
{ value: '15000', label: '15s', description: 'Best for thinking' },
] as const;
export function VoiceInSettings(props: { isMobile: boolean }) {
// external state
const [chatTimeoutMs, setChatTimeoutMs] = useChatMicTimeoutMs();
// derived - converts from string keys to numbers and vice versa
const chatTimeoutValue: string = '' + chatTimeoutMs;
const setChatTimeoutValue = React.useCallback((value: string) => {
value && setChatTimeoutMs(parseInt(value));
}, [setChatTimeoutMs]);
return <>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center' }}>
<FormLabelStart
title='Language'
description='Mic and voice'
// tooltip='For Microphone input and Voice output. Microphone support varies by browser (iPhone/Safari lacks speech input).'
/>
<LanguageSelect />
</FormControl>
{!props.isMobile && (
<FormChipControl
title='Timeout'
// color='primary'
options={_minTimeouts}
value={chatTimeoutValue}
onChange={setChatTimeoutValue}
/>
)}
</>;
}
@@ -0,0 +1,46 @@
import { SpeexConfigureEngines } from '~/modules/speex/components/SpeexConfigureEngines';
import { useSpeexEngines } from '~/modules/speex/store-module-speex';
import { ChatAutoSpeakType, useChatAutoAI } from '../chat/store-app-chat';
import { FormRadioOption } from '~/common/components/forms/FormRadioControl';
import { FormChipControl } from '~/common/components/forms/FormChipControl';
const _autoSpeakOptions: FormRadioOption<ChatAutoSpeakType>[] = [
{ value: 'off', label: 'No', description: 'Off' },
{ value: 'firstLine', label: 'Start', description: 'First paragraph' },
{ value: 'all', label: 'Full', description: 'Complete response' },
] as const;
/**
* Voice output settings - Auto-speak mode and TTS engine configuration
*/
export function VoiceOutSettings(props: { isMobile: boolean }) {
// external state
const { autoSpeak, setAutoSpeak } = useChatAutoAI();
// external state - module
const hasEngines = useSpeexEngines().length > 0;
return <>
{/* Auto-speak setting */}
<FormChipControl
title='Speak Chats'
size='md'
// color='primary'
tooltip={!hasEngines ? 'No voice engines available. Configure a TTS service or use system voice.' : undefined}
disabled={!hasEngines}
options={_autoSpeakOptions}
value={autoSpeak}
onChange={setAutoSpeak}
/>
{/* Engine configuration */}
<SpeexConfigureEngines isMobile={props.isMobile} />
</>;
}
-47
View File
@@ -1,47 +0,0 @@
import * as React from 'react';
import { FormControl } from '@mui/joy';
import { useChatMicTimeoutMs } from '../chat/store-app-chat';
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
import { FormRadioControl } from '~/common/components/forms/FormRadioControl';
import { LanguageSelect } from '~/common/components/LanguageSelect';
import { useIsMobile } from '~/common/components/useMatchMedia';
export function VoiceSettings() {
// external state
const isMobile = useIsMobile();
const [chatTimeoutMs, setChatTimeoutMs] = useChatMicTimeoutMs();
// this converts from string keys to numbers and vice versa
const chatTimeoutValue: string = '' + chatTimeoutMs;
const setChatTimeoutValue = (value: string) => value && setChatTimeoutMs(parseInt(value));
return <>
{/* LanguageSelect: moved from the UI settings (where it logically belongs), just to group things better from an UX perspective */}
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center' }}>
<FormLabelStart title='Language'
description='ASR and TTS'
tooltip='Currently for Microphone input and Voice output. Microphone support varies by browser (iPhone/Safari lacks speech input). We will use the ElevenLabs MultiLanguage model if a language other than English is selected.' />
<LanguageSelect />
</FormControl>
{!isMobile && <FormRadioControl
title='Mic Timeout'
description={chatTimeoutMs < 1000 ? 'Best for quick calls' : chatTimeoutMs > 5000 ? 'Best for thinking' : 'Standard'}
options={[
{ value: '600', label: '.6s' },
{ value: '2000', label: '2s' },
{ value: '5000', label: '5s' },
{ value: '15000', label: '15s' },
]}
value={chatTimeoutValue} onChange={setChatTimeoutValue}
/>}
</>;
}
+2 -2
View File
@@ -23,8 +23,8 @@ export const Release = {
// this is here to trigger revalidation of data, e.g. models refresh
Monotonics: {
Aix: 42,
NewsVersion: 201,
Aix: 43,
NewsVersion: 202,
},
// Frontend: pretty features
+38
View File
@@ -0,0 +1,38 @@
import * as React from 'react';
import { Chip, ChipProps } from '@mui/joy';
/**
* Simple badge/label component for inline status indicators like "New", "Beta", etc.
*/
export function GoodBadge(props: {
badge: React.ReactNode;
color?: ChipProps['color'];
variant?: ChipProps['variant'];
sx?: ChipProps['sx'];
}) {
return (
<Chip
size='sm'
color={props.color ?? 'success'}
variant={props.variant ?? 'soft'}
sx={{
ml: 1.5,
fontSize: 'xs',
fontWeight: 'md',
borderRadius: 'xs',
px: 1,
py: 0.25,
// default "new" color - lime/yellow-green
...(props.color === undefined && {
bgcolor: '#d5ec31',
color: 'primary.softColor',
}),
...props.sx,
}}
>
{props.badge}
</Chip>
);
}
+22 -8
View File
@@ -1,6 +1,6 @@
import * as React from 'react';
import { Option, Select } from '@mui/joy';
import { Option, optionClasses, Select, SelectSlotsAndSlotProps } from '@mui/joy';
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
import { useUIPreferencesStore } from '~/common/stores/store-ui';
@@ -10,6 +10,20 @@ import { useUIPreferencesStore } from '~/common/stores/store-ui';
import languages from './Languages.json';
// copied from useLLMSelect.tsx - inspired by optimaSelectSlotProps.listbox
const _selectSlotProps: SelectSlotsAndSlotProps<false>['slotProps'] = {
root: { sx: { minWidth: 200 } },
listbox: {
sx: {
boxShadow: 'xl',
[`& .${optionClasses.root}`]: {
maxWidth: 'min(640px, calc(100dvw - 0.25rem))',
},
},
} as const,
} as const;
export function LanguageSelect() {
// external state
@@ -32,19 +46,19 @@ export function LanguageSelect() {
</Option>
) : (
Object.entries(localesOrCode).map(([country, code]) => (
<Option key={code} value={code}>
<Option key={code} value={code} label={language}>
{`${language} (${country})`}
</Option>
))
)), []);
return (
<Select value={preferredLanguage} onChange={handleLanguageChanged}
indicator={<KeyboardArrowDownIcon />}
slotProps={{
root: { sx: { minWidth: 200 } },
indicator: { sx: { opacity: 0.5 } },
}}>
<Select
value={preferredLanguage}
onChange={handleLanguageChanged}
indicator={<KeyboardArrowDownIcon />}
slotProps={_selectSlotProps}
>
{languageOptions}
</Select>
);
@@ -21,6 +21,13 @@ const _styles = {
gap: 1,
} as const,
chipGroupEnd: {
display: 'flex',
flexWrap: 'wrap',
justifyContent: 'flex-end',
gap: 1,
} as const,
chip: {
'--Chip-minHeight': '1.75rem', // this makes it prob better
px: 1.5,
@@ -36,6 +43,7 @@ export const FormChipControl = <TValue extends string>(props: {
// specific
size?: 'sm' | 'md' | 'lg',
color?: ColorPaletteProp,
alignEnd?: boolean,
// =FormRadioControl
title: string | React.JSX.Element;
description?: string | React.JSX.Element;
@@ -48,6 +56,9 @@ export const FormChipControl = <TValue extends string>(props: {
const { onChange } = props;
const selectedOption = props.options.find(option => option.value === props.value);
const description = selectedOption?.description ?? props.description;
const handleChipClick = React.useCallback((value: Immutable<TValue>) => {
if (!props.disabled)
onChange(value);
@@ -55,8 +66,8 @@ export const FormChipControl = <TValue extends string>(props: {
return (
<FormControl orientation='horizontal' disabled={props.disabled} sx={_styles.control}>
{(!!props.title || !!props.description) && <FormLabelStart title={props.title} description={props.description} tooltip={props.tooltip} />}
<Box sx={_styles.chipGroup}>
{(!!props.title || !!description) && <FormLabelStart title={props.title} description={description} tooltip={props.tooltip} />}
<Box sx={props.alignEnd ? _styles.chipGroupEnd : _styles.chipGroup}>
{props.options.map((option) => (
<Chip
key={'opt-' + option.value}
@@ -0,0 +1,95 @@
import * as React from 'react';
import type { SxProps } from '@mui/joy/styles/types';
import { FormControl, IconButton, Input } from '@mui/joy';
import KeyIcon from '@mui/icons-material/Key';
import VisibilityIcon from '@mui/icons-material/Visibility';
import VisibilityOffIcon from '@mui/icons-material/VisibilityOff';
import { FormLabelStart } from './FormLabelStart';
const _styles = {
formControl: {
flexWrap: 'wrap',
justifyContent: 'space-between',
alignItems: 'center',
},
inputDefault: {
flexGrow: 1,
},
} as const satisfies Record<string, SxProps>;
/**
* Secret/API key form field with visibility toggle.
* Same inline layout as FormTextField but with secret-specific features:
* - Password masking with visibility toggle
* - Key icon (customizable)
* - Password manager integration
*/
export function FormSecretField(props: {
autoCompleteId: string;
title: string | React.JSX.Element;
description?: string | React.JSX.Element;
tooltip?: string | React.JSX.Element;
placeholder?: string;
value: string;
onChange: (text: string) => void;
// Behavior
required?: boolean;
disabled?: boolean;
isError?: boolean;
// Appearance
inputSx?: SxProps;
/** Custom start decorator, or false to hide. Default: KeyIcon */
startDecorator?: React.ReactNode | false;
}) {
// state
const [isVisible, setIsVisible] = React.useState(false);
// derived
const acId = 'secret-' + props.autoCompleteId;
// password manager username
const ghost = props.autoCompleteId.replace(/-key$/, '').replace(/-/g, ' ');
const endDecorator = React.useMemo(() => !!props.value && (
<IconButton size='sm' onClick={() => setIsVisible(on => !on)}>
{isVisible ? <VisibilityIcon sx={{ fontSize: 'md' }} /> : <VisibilityOffIcon sx={{ fontSize: 'md' }} />}
</IconButton>
), [props.value, isVisible]);
return (
<FormControl
id={acId}
orientation='horizontal'
disabled={props.disabled}
sx={_styles.formControl}
>
<FormLabelStart title={props.title} description={props.description} tooltip={props.tooltip} />
{/* Hidden username field for password manager association */}
<input
type='text'
autoComplete='username'
value={ghost}
readOnly
tabIndex={-1}
style={{ display: 'none' }}
/>
<Input
name={acId}
type={isVisible ? 'text' : 'password'}
autoComplete='new-password'
variant='outlined'
placeholder={props.required && !props.placeholder ? 'required' : props.placeholder}
error={props.isError}
value={props.value}
onChange={event => props.onChange(event.target.value)}
startDecorator={props.startDecorator ?? <KeyIcon sx={{ fontSize: 'md' }} />}
endDecorator={endDecorator}
sx={props.inputSx ?? _styles.inputDefault}
/>
</FormControl>
);
}
@@ -24,6 +24,7 @@ export function FormSliderControl(props: {
startAdornment?: React.ReactNode,
endAdornment?: React.ReactNode,
styleNoTrack?: boolean,
sliderSx?: SxProps,
}) {
@@ -66,8 +67,7 @@ export function FormSliderControl(props: {
onChange={handleChange}
onChangeCommitted={handleChangeCommitted}
valueLabelDisplay={props.valueLabelDisplay}
sx={props.styleNoTrack ? _styleNoTrack : undefined}
// sx={{ py: 1, mt: 1.1 }}
sx={props.styleNoTrack ? _styleNoTrack : props.sliderSx}
/>
{props.endAdornment}
</FormControl>
+13 -7
View File
@@ -6,11 +6,16 @@ import { FormControl, Input } from '@mui/joy';
import { FormLabelStart } from './FormLabelStart';
const formControlSx: SxProps = {
flexWrap: 'wrap',
justifyContent: 'space-between',
alignItems: 'center',
};
const _styles = {
formControl: {
flexWrap: 'wrap',
justifyContent: 'space-between',
alignItems: 'center',
},
inputDefault: {
flexGrow: 1,
},
} as const satisfies Record<string, SxProps>;
/**
@@ -23,6 +28,7 @@ export function FormTextField(props: {
tooltip?: string | React.JSX.Element,
placeholder?: string, isError?: boolean, disabled?: boolean,
value: string | undefined, onChange: (text: string) => void,
inputSx?: SxProps,
}) {
const acId = 'text-' + props.autoCompleteId;
return (
@@ -30,7 +36,7 @@ export function FormTextField(props: {
id={acId}
orientation='horizontal'
disabled={props.disabled}
sx={formControlSx}
sx={_styles.formControl}
>
<FormLabelStart title={props.title} description={props.description} tooltip={props.tooltip} />
<Input
@@ -39,7 +45,7 @@ export function FormTextField(props: {
autoComplete='off'
variant='outlined' placeholder={props.placeholder} error={props.isError}
value={props.value} onChange={event => props.onChange(event.target.value)}
sx={{ flexGrow: 1 }}
sx={props.inputSx ?? _styles.inputDefault}
/>
</FormControl>
);
@@ -0,0 +1,14 @@
import * as React from 'react';
import { SvgIcon, SvgIconProps } from '@mui/joy';
/*
* Source: 'https://phosphoricons.com/' - user-sound
*/
export function PhVoice(props: SvgIconProps) {
return (
<SvgIcon viewBox='0 0 256 256' stroke='none' fill='currentColor' width='24' height='24' {...props}>
<path d='M144,165.68a68,68,0,1,0-71.9,0c-20.65,6.76-39.23,19.39-54.17,37.17a8,8,0,0,0,12.25,10.3C50.25,189.19,77.91,176,108,176s57.75,13.19,77.88,37.15a8,8,0,1,0,12.25-10.3C183.18,185.07,164.6,172.44,144,165.68ZM56,108a52,52,0,1,1,52,52A52.06,52.06,0,0,1,56,108ZM207.36,65.6a108.36,108.36,0,0,1,0,84.8,8,8,0,0,1-7.36,4.86,8,8,0,0,1-7.36-11.15,92.26,92.26,0,0,0,0-72.22,8,8,0,0,1,14.72-6.29ZM248,108a139,139,0,0,1-11.29,55.15,8,8,0,0,1-14.7-6.3,124.43,124.43,0,0,0,0-97.7,8,8,0,1,1,14.7-6.3A139,139,0,0,1,248,108Z' />
</SvgIcon>
);
}
@@ -0,0 +1,9 @@
import * as React from 'react';
import { SvgIcon, SvgIconProps } from '@mui/joy';
export function ElevenLabsIcon(props: SvgIconProps) {
return <SvgIcon viewBox='0 0 24 24' width='24' height='24' fill='currentColor' {...props}>
<path d='M7 4h3v16H7V4zm7 0h3v16h-3V4z' />
</SvgIcon>;
}
-11
View File
@@ -25,17 +25,6 @@ export interface CapabilityBrowserSpeechRecognition {
export { browserSpeechRecognitionCapability as useCapabilityBrowserSpeechRecognition } from './speechrecognition/useSpeechRecognition';
/// Speech Synthesis: ElevenLabs
export interface CapabilityElevenLabsSpeechSynthesis {
mayWork: boolean;
isConfiguredServerSide: boolean;
isConfiguredClientSide: boolean;
}
export { useCapability as useCapabilityElevenLabs } from '~/modules/elevenlabs/elevenlabs.client';
/// Image Generation
export interface TextToImageProvider {
@@ -4,7 +4,7 @@ import { Alert, IconButton } from '@mui/joy';
import CloseRoundedIcon from '@mui/icons-material/CloseRounded';
import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
import { Is, isBrowser, isPwa } from '~/common/util/pwaUtils';
import { isBrowser, isPwa } from '~/common/util/pwaUtils';
import { useUICounter } from '~/common/stores/store-ui';
@@ -31,15 +31,14 @@ export function usePWADesktopModeWarning() {
// if PWA
const isInPwaMode = isPwa();
if (!isInPwaMode) return false;
// if Mobile device (detected using touch points), while desktop have 0 touch points
const isTouchDevice = navigator?.maxTouchPoints > 0;
// if Physical Screen is small - typical mobile screen size - e.g. 412 for SGS24 Ultra
const isSmallScreen = window.screen?.width < 600;
// if Desktop mode - e.g. "Desktop Site" reports a large viewport width, typically 9xx+
const isDesktopWidth = window.matchMedia('(min-width: 900px)').matches;
// if OS is mobile
const isMobileOS = Is.OS.iOS || Is.OS.Android;
if (!isMobileOS) return false;
// Check if viewport width suggests desktop mode (>= 900px)
// This matches the mobile breakpoint used in useMatchMedia.ts
return window.matchMedia('(min-width: 900px)').matches;
return isInPwaMode && isTouchDevice && isSmallScreen && isDesktopWidth;
}, []);
const showWarning = isInDesktopMode && !hideWarning && lessThanFive;
@@ -7,36 +7,38 @@ import type { DModelsService } from '../llms.service.types';
import { llmsStoreActions, useModelsStore } from '../store-llms';
const CSF_KEY = 'csf';
/**
* Hook to manage client-side fetch setting for a model's service
* The CSF setting is stored as 'csf' in service settings for all vendors
*/
export function useModelServiceClientSideFetch(enabled: boolean, model: DLLM | null) {
// memo vendor
const { vendor, csfKey } = React.useMemo(() => {
if (!enabled) return { vendor: null, csfKey: '' };
const vendor = findModelVendor(model?.vId);
const csfKey = vendor?.csfKey || '';
return { vendor, csfKey };
const vendor = React.useMemo(() => {
if (!enabled) return null;
return findModelVendor(model?.vId);
}, [enabled, model?.vId]);
// external state
const service: null | DModelsService = useModelsStore(state => !model?.sId ? null : state.sources.find(s => s.id === model.sId) ?? null);
// actual state
const csfAvailable: boolean | undefined = !!csfKey && vendor?.csfAvailable?.(service?.setup);
const csfActive: boolean | undefined = csfAvailable && (service?.setup as any)?.[csfKey];
const csfAvailable: boolean | undefined = !!vendor?.csfAvailable && vendor?.csfAvailable?.(service?.setup);
const csfActive: boolean | undefined = csfAvailable && (service?.setup as any)?.[CSF_KEY];
const serviceId = service?.id || '';
const csfToggle = React.useCallback((value: boolean) => {
if (csfKey && serviceId)
llmsStoreActions().updateServiceSettings(serviceId, { [csfKey]: value });
}, [csfKey, serviceId]);
if (serviceId)
llmsStoreActions().updateServiceSettings(serviceId, { [CSF_KEY]: value });
}, [serviceId]);
const csfReset = React.useCallback(() => {
if (csfKey && serviceId)
llmsStoreActions().updateServiceSettings(serviceId, { [csfKey]: false });
}, [csfKey, serviceId]);
if (serviceId)
llmsStoreActions().updateServiceSettings(serviceId, { [CSF_KEY]: false });
}, [serviceId]);
return { csfAvailable, csfActive, csfToggle, csfReset };
return { csfAvailable, csfActive, csfToggle, csfReset, vendorName: vendor?.name || vendor?.id || 'AI Service' };
}
+26 -18
View File
@@ -19,6 +19,10 @@ export class AudioLivePlayer {
this.audioElement.src = URL.createObjectURL(this.mediaSource);
this.audioElement.autoplay = true;
// Suppress Android media notification by clearing media session metadata
if ('mediaSession' in navigator)
navigator.mediaSession.metadata = null;
// Connect the audio element to the audio context
const sourceNode = this.audioContext.createMediaElementSource(this.audioElement);
sourceNode.connect(this.audioContext.destination);
@@ -60,14 +64,12 @@ export class AudioLivePlayer {
private onSourceBufferUpdateEnd = () => {
this.isSourceBufferUpdating = false;
// Continue appending if there's more data
if (!this.isMediaSourceEnded) {
// Always continue appending if there's more data in the queue
if (this.chunkQueue.length > 0) {
this.appendNextChunk();
} else {
// End the stream if all data has been appended
if (this.sourceBuffer && !this.sourceBuffer.updating && this.chunkQueue.length === 0) {
this.mediaSource.endOfStream();
}
} else if (this.isMediaSourceEnded) {
// Only end the stream when queue is fully drained
this._safeEndOfStream();
}
};
@@ -86,9 +88,17 @@ export class AudioLivePlayer {
}
}
} else if (this.isMediaSourceEnded) {
if (this.sourceBuffer && !this.sourceBuffer.updating) {
this.mediaSource.endOfStream();
}
if (this.sourceBuffer && !this.sourceBuffer.updating)
this._safeEndOfStream();
}
}
private _safeEndOfStream() {
if (this.mediaSource.readyState !== 'open') return;
try {
this.mediaSource.endOfStream();
} catch (e) {
// Ignore - MediaSource may have already ended
}
}
@@ -106,9 +116,8 @@ export class AudioLivePlayer {
public endPlayback() {
this.isMediaSourceEnded = true;
// If the sourceBuffer is not updating, we can end the stream
if (this.sourceBuffer && !this.sourceBuffer.updating && this.chunkQueue.length === 0) {
this.mediaSource.endOfStream();
}
if (this.sourceBuffer && !this.sourceBuffer.updating && this.chunkQueue.length === 0)
this._safeEndOfStream();
}
/**
@@ -119,14 +128,13 @@ export class AudioLivePlayer {
this.chunkQueue = [];
this.isMediaSourceEnded = true;
if (this.sourceBuffer) {
// only abort SourceBuffer when MediaSource is 'open'
if (this.sourceBuffer && this.mediaSource.readyState === 'open') {
try {
if (this.mediaSource.readyState === 'open') {
this.mediaSource.endOfStream();
}
this.sourceBuffer.abort();
this.mediaSource.endOfStream();
} catch (e) {
console.warn('Error stopping playback:', e);
// Ignore - may race with natural stream end
}
}
+1
View File
@@ -65,6 +65,7 @@ export function agiCustomId(digits: number) {
type UuidV4Scope =
| 'conversation-2'
| 'persona-2'
| 'speex.engine.instance'
;
+179
View File
@@ -28,3 +28,182 @@ export function countKeys(obj: object | null | undefined): number {
for (const _ in obj) count++;
return count;
}
/**
* Strip undefined fields from an object.
*
* Useful to prevent undefined becoming null over the wire (JSON serialization).
*/
export function stripUndefined<T extends object>(obj: T): T;
export function stripUndefined<T extends object>(obj: T | null): T | null;
export function stripUndefined<T extends object>(obj: T | null): T | null {
if (!obj) return null;
return Object.fromEntries(Object.entries(obj).filter(([_, v]) => v !== undefined)) as T;
}
// === Size Estimation ===
/**
* Estimates JSON serialized size without actually stringifying.
*
* Avoids memory allocation spike on large objects. Useful for progress tracking,
* batching decisions, or debug output sizing.
*
* Note: This is an ESTIMATE. It doesn't account for:
* - UTF-8 multi-byte characters (assumes 1 byte per char)
* - JSON escape sequences (\n, \t, unicode escapes)
* - Floating point precision differences
*
* Returns 0 for the cyclic portion if circular references are detected.
*/
export function objectEstimateJsonSize(value: unknown, debugCaller: string): number {
const seen = new WeakSet<object>();
function estimate(val: unknown): number {
if (val === null) return 4; // "null"
if (val === undefined) return 0; // omitted in JSON
switch (typeof val) {
case 'string':
return val.length + 2; // quotes
case 'number':
return String(val).length;
case 'boolean':
return val ? 4 : 5; // "true" or "false"
case 'object': {
// cycle detection
if (seen.has(val as object)) {
console.warn(`[estimateJsonSize (${debugCaller})] Circular reference detected, returning 0 for this branch`);
return 0;
}
seen.add(val as object);
if (Array.isArray(val)) {
let size = 2; // []
for (let i = 0; i < val.length; i++) {
size += estimate(val[i]);
if (i < val.length - 1) size += 1; // comma
}
return size;
}
// plain object
let size = 2; // {}
const keys = Object.keys(val);
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
size += key.length + 3; // "key":
size += estimate((val as Record<string, unknown>)[key]);
if (i < keys.length - 1) size += 1; // comma
}
return size;
}
default:
return 0;
}
}
return estimate(value);
}
// === Object Traversal ===
/**
* Deep clones an object while truncating strings that exceed maxBytes.
*
* Useful for debug logging of large objects (e.g., requests with base64 images).
* Truncates strings in the middle, preserving start and end with a byte count.
*
* @returns Deep clone with truncated strings, or "[Circular]" for cyclic refs
*/
export function objectDeepCloneWithStringLimit(value: unknown, debugCaller: string, maxBytes: number = 2048): unknown {
const seen = new WeakSet<object>();
function clone(val: unknown): unknown {
// handle primitives first
if (val === null || val === undefined) return val;
// handle strings - truncate if too long
if (typeof val === 'string') {
if (val.length <= maxBytes) return val;
const ellipsis = `...[${(val.length - maxBytes).toLocaleString()} bytes]...`;
const half = Math.floor((maxBytes - ellipsis.length) / 2);
return val.slice(0, half) + ellipsis + val.slice(-half);
}
// handle other primitives
if (typeof val !== 'object') return val;
// cycle detection
if (seen.has(val)) return '[Circular]';
seen.add(val);
// handle arrays - recurse
if (Array.isArray(val))
return val.map(item => clone(item));
// handle objects - recurse
const result: Record<string, unknown> = {};
for (const key in val)
if (Object.prototype.hasOwnProperty.call(val, key))
result[key] = clone((val as Record<string, unknown>)[key]);
return result;
}
return clone(value);
}
/**
* Find the largest string values in an object tree
*
* Recursively traverses an object to find the top N largest string values,
* returning their paths, lengths, and preview snippets.
*
* @returns Array of {path, length, preview} sorted by length (descending)
*/
export function objectFindLargestStringPaths(obj: unknown, debugCaller: string, topN: number = 5, maxDepth: number = 20): Array<{ path: string; length: number; preview: string }> {
const results: Array<{ path: string; length: number; preview: string }> = [];
const seen = new WeakSet<object>();
function traverse(current: unknown, path: string, depth: number) {
// prevent infinite recursion
if (depth > maxDepth) return;
// handle strings
if (typeof current === 'string') {
results.push({
path,
length: current.length,
preview: current.substring(0, 100) + (current.length > 100 ? '...' : ''),
});
return;
}
// handle non-objects
if (current === null || typeof current !== 'object') return;
// cycle detection
if (seen.has(current)) {
console.warn(`[findLargestStringPaths (${debugCaller})] Circular reference at path: ${path}`);
return;
}
seen.add(current);
// handle arrays
if (Array.isArray(current))
return current.forEach((item, index) => traverse(item, `${path}[${index}]`, depth + 1));
// handle objects
for (const [key, value] of Object.entries(current))
traverse(value, path ? `${path}.${key}` : key, depth + 1);
}
traverse(obj, '', 0);
// sort by length descending and return top N
return results
.sort((a, b) => b.length - a.length)
.slice(0, topN);
}
+3 -1
View File
@@ -604,6 +604,8 @@ export class ContentReassembler {
this.accumulator.fragments.unshift(placeholderFragment); // Add to beginning
// Placeholders don't affect text fragment indexing
// NOTE: we could have placeholders breaking text accumulation into new fragments with `this.currentTextFragmentIndex = null;`, however
// since placeholders are used a lot with hosted tool calls, this could lead to way too many fragments being created
}
private onSetVendorState(vs: Extract<AixWire_Particles.PartParticleOp, { p: 'svs' }>): void {
@@ -619,7 +621,7 @@ export class ContentReassembler {
lastFragment.vendorState = {
...lastFragment.vendorState,
[vendor]: state,
}
};
}
// Helper to remove placeholder when real content arrives
+3 -2
View File
@@ -36,13 +36,14 @@ export function aixClassifyStreamingError(error: any, isUserAbort: boolean, hasF
// Browser-level network connection drops (TypeError, happens below tRPC error wrapping layer)
// Network errors - when the client is disconnected (Vercel 5min timeout, Mobile timeout / disconnect, etc) - they show up as TypeErrors
// IMPORTANT: we will differentiate between the 2 'net-disconnected' cases in the UI, checking for the errorMessage '**network error**' vs '**connection terminated**'
if (error instanceof TypeError && error.message === 'network error')
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **network error**.' };
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **network error**.' /* DO NOT CHANGE '**network error**' - usually client-side broken */ };
// tRPC <= 11.5.1 - Vercel Edge network disconnects are thrown form tRPC as 'Stream closed'
// NOTE The behavior changed in 11.6+ for which we have an open upstream ticket: #6989
if (error instanceof Error && error.message === 'Stream closed')
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **connection terminated**.' };
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **connection terminated**.' /* DO NOT CHANGE '**connection terminated**' - usually server (Vercel) side broken */ };
// tRPC-level protocol errors (wrapped by tRPC client)
@@ -1,6 +1,8 @@
import { SERVER_DEBUG_WIRE } from '~/server/wire';
import { serverSideId } from '~/server/trpc/trpc.nanoid';
import { objectDeepCloneWithStringLimit, objectEstimateJsonSize } from '~/common/util/objectUtils';
import type { AixWire_Particles } from '../../api/aix.wiretypes';
import type { IParticleTransmitter, ParticleServerLogLevel } from './parsers/IParticleTransmitter';
@@ -25,70 +27,6 @@ export const IssueSymbols = {
};
/** Estimates JSON size without stringifying (avoids memory spike on large objects). */
function _fastEstimateJsonSize(value: any): number {
if (value === null) return 4; // "null"
if (value === undefined) return 0; // omitted in JSON
if (typeof value === 'string')
return value.length + 2; // quotes
if (typeof value === 'number')
return String(value).length;
if (typeof value === 'boolean')
return value ? 4 : 5; // "true" or "false"
if (Array.isArray(value)) {
let size = 2; // []
for (let i = 0; i < value.length; i++) {
size += _fastEstimateJsonSize(value[i]);
if (i < value.length - 1) size += 1; // comma
}
return size;
}
if (typeof value === 'object') {
let size = 2; // {}
const keys = Object.keys(value);
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
size += key.length + 3; // "key":
size += _fastEstimateJsonSize(value[key]);
if (i < keys.length - 1) size += 1; // comma
}
return size;
}
return 0;
}
/** Deep-clones an object while ellipsizing any string exceeding maxBytes in the middle. */
function _fastEllipsizeStringsInObject(value: any, maxBytes: number = DEBUG_REQUEST_MAX_STRING_BYTES): any {
// handle primitives first
if (value === null || value === undefined)
return value;
// handle strings - ellipsize if too long
if (typeof value === 'string') {
if (value.length <= maxBytes)
return value;
const ellipsis = `...[${(value.length - maxBytes).toLocaleString()} bytes]...`;
const half = Math.floor((maxBytes - ellipsis.length) / 2);
return value.slice(0, half) + ellipsis + value.slice(-half);
}
// handle other primitives (number, boolean)
if (typeof value !== 'object')
return value;
// handle arrays - recurse
if (Array.isArray(value))
return value.map(item => _fastEllipsizeStringsInObject(item, maxBytes));
// handle objects - recurse
const result: any = {};
for (const key in value)
if (value.hasOwnProperty(key))
result[key] = _fastEllipsizeStringsInObject(value[key], maxBytes);
return result;
}
/**
* Queues up and emits small messages (particles) to the client, for the purpose of a stateful
* full reconstruction of the AixWire_Parts[] objects.
@@ -194,7 +132,7 @@ export class ChatGenerateTransmitter implements IParticleTransmitter {
addDebugRequest(hideSensitiveData: boolean, url: string, headers: HeadersInit, body?: object) {
// Ellipsize individual strings in the body object (e.g., base64 images) to reduce debug packet size
const ellipsizedBody = body ? _fastEllipsizeStringsInObject(body) : undefined;
const ellipsizedBody = body ? objectDeepCloneWithStringLimit(body, 'aix.addDebugRequest', DEBUG_REQUEST_MAX_STRING_BYTES) : undefined;
const processedBody = ellipsizedBody ? JSON.stringify(ellipsizedBody, null, 2) : '';
this.transmissionQueue.push({
@@ -204,7 +142,7 @@ export class ChatGenerateTransmitter implements IParticleTransmitter {
url: url,
headers: hideSensitiveData ? '(hidden sensitive data)' : JSON.stringify(headers, null, 2),
body: processedBody,
bodySize: body ? _fastEstimateJsonSize(body) : 0,
bodySize: body ? objectEstimateJsonSize(body, 'aix.addDebugRequest') : 0,
},
});
}
@@ -61,6 +61,7 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
let timeToFirstEvent: number;
let messageStartTime: number | undefined = undefined;
let chatInTokens: number | undefined = undefined;
let needsTextSeparator = false; // insert text separator when text follows server tool
return function(pt: IParticleTransmitter, eventData: string, eventName?: string, context?: { retriesAvailable: boolean }): void {
@@ -155,7 +156,9 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
switch (content_block.type) { // .content_block_start.type
case 'text':
pt.appendText(content_block.text);
// add separator when text follows server tool execution
pt.appendText(!needsTextSeparator ? content_block.text : '\n\n' + content_block.text);
needsTextSeparator = false;
// Note: In streaming mode, citations arrive via citations_delta events, not on content_block_start
break;
@@ -379,6 +382,11 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
const _exhaustiveCheck: never = content_block;
throw new Error(`Unexpected content block type: ${(content_block as any).type}`);
}
// set separator flag when server tools complete (text after tools needs visual separation)
if (content_block.type.includes('tool_use') || content_block.type.includes('tool_result'))
needsTextSeparator = true;
break;
}
@@ -576,6 +584,7 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
export function createAnthropicMessageParserNS(): ChatGenerateParseFunction {
const parserCreationTimestamp = Date.now();
let needsTextSeparator = false; // insert text separator when text follows server tool
return function(pt: IParticleTransmitter, fullData: string /*, eventName?: string, context?: { retriesAvailable: boolean } */): void {
@@ -597,7 +606,9 @@ export function createAnthropicMessageParserNS(): ChatGenerateParseFunction {
const isLastBlock = i === content.length - 1;
switch (contentBlock.type) { // .content_block (non-streaming)
case 'text':
pt.appendText(contentBlock.text);
// add separator when text follows server tool execution
pt.appendText(!needsTextSeparator ? contentBlock.text : '\n\n' + contentBlock.text);
needsTextSeparator = false;
// Handle citations if present (non-streaming mode has all citations attached)
if (contentBlock.citations && Array.isArray(contentBlock.citations)) {
for (const citation of contentBlock.citations) {
@@ -824,6 +835,10 @@ export function createAnthropicMessageParserNS(): ChatGenerateParseFunction {
const _exhaustiveCheck: never = contentBlock;
throw new Error(`Unexpected content block type: ${(contentBlock as any).type}`);
}
// set separator flag when server tools complete (text after tools needs visual separation)
if (contentBlock.type.includes('tool_use') || contentBlock.type.includes('tool_result'))
needsTextSeparator = true;
}
// -> Token Stop Reason
@@ -586,6 +586,11 @@ export namespace AnthropicWire_Tools {
required: z.array(z.string()).optional(), // 2025-02-24: seems to be removed; we may still have this, but it may also be within the 'properties' object
}),
/**
* [Anthropic, 2025-11-13] Structured Outputs - guarantees tool inputs to match `input_schema` exactly.
*/
strict: z.boolean().optional(),
/**
* [Anthropic, 2025-11-24] Tool Search Tool - when true, this tool is not loaded into context initially and can be discovered via the tool search tool when needed.
*/
+25 -7
View File
@@ -143,13 +143,31 @@ async function workerPuppeteer(
};
// [puppeteer] start the remote session
const browser: Browser = await puppeteer.connect({
browserWSEndpoint,
// Add default options for better stability
// defaultViewport: { width: 1024, height: 768 },
// acceptInsecureCerts: true,
protocolTimeout: WORKER_TIMEOUT + 2000, // 2s extra for taking the screenshot?
});
let browser: Browser;
try {
browser = await puppeteer.connect({
browserWSEndpoint,
// Add default options for better stability
// defaultViewport: { width: 1024, height: 768 },
// acceptInsecureCerts: true,
protocolTimeout: WORKER_TIMEOUT + 2000, // 2s extra for taking the screenshot?
});
} catch (connectError: any) {
// Transform connection errors into user-friendly messages
const errorMessage = connectError?.message || '';
if (errorMessage.includes('403'))
throw new Error('Browse service authentication failed (403). Please check your browser endpoint credentials.');
if (errorMessage.includes('401'))
throw new Error('Browse service unauthorized (401). Invalid credentials for the browser endpoint.');
if (errorMessage.includes('429'))
throw new Error('Browse service rate limited (429). Too many requests, please try again later.');
if (errorMessage.includes('502') || errorMessage.includes('503') || errorMessage.includes('504'))
throw new Error('Browse service temporarily unavailable. Please try again later.');
if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('ENOTFOUND'))
throw new Error('Browse service unreachable. The browser endpoint is not accessible.');
// Re-throw with a cleaner message for other connection errors
throw new Error(`Browse service connection failed: ${errorMessage || 'Unknown error'}`);
}
// for local testing, open an incognito context, to separate cookies
let incognitoContext: BrowserContext | null = null;
@@ -1,67 +0,0 @@
import * as React from 'react';
import { FormControl } from '@mui/joy';
import { useChatAutoAI } from '../../apps/chat/store-app-chat';
import { AlreadySet } from '~/common/components/AlreadySet';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
import { FormRadioControl } from '~/common/components/forms/FormRadioControl';
import { useCapabilityElevenLabs } from '~/common/components/useCapabilities';
import { isElevenLabsEnabled } from './elevenlabs.client';
import { useElevenLabsVoiceDropdown, useElevenLabsVoices } from './useElevenLabsVoiceDropdown';
import { useElevenLabsApiKey } from './store-module-elevenlabs';
export function ElevenlabsSettings() {
// state
const [apiKey, setApiKey] = useElevenLabsApiKey();
// external state
const { autoSpeak, setAutoSpeak } = useChatAutoAI();
const { hasVoices } = useElevenLabsVoices();
const { isConfiguredServerSide } = useCapabilityElevenLabs();
const { voicesDropdown } = useElevenLabsVoiceDropdown(true);
// derived state
const isValidKey = isElevenLabsEnabled(apiKey);
return <>
{/*<FormHelperText>*/}
{/* 📢 Hear AI responses, even in your own voice*/}
{/*</FormHelperText>*/}
<FormRadioControl
title='Speak Responses'
description={autoSpeak === 'off' ? 'Off' : 'First paragraph'}
tooltip={!hasVoices ? 'No voices available, please configure a voice synthesis service' : undefined}
disabled={!hasVoices}
options={[
{ value: 'off', label: 'Off' },
{ value: 'firstLine', label: 'Start' },
{ value: 'all', label: 'Full' },
]}
value={autoSpeak} onChange={setAutoSpeak}
/>
{!isConfiguredServerSide && <FormInputKey
autoCompleteId='elevenlabs-key' label='ElevenLabs API Key'
rightLabel={<AlreadySet required={!isConfiguredServerSide} />}
value={apiKey} onChange={setApiKey}
required={!isConfiguredServerSide} isError={!isValidKey}
/>}
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center' }}>
<FormLabelStart title='Assistant Voice' />
{voicesDropdown}
</FormControl>
</>;
}
-126
View File
@@ -1,126 +0,0 @@
import { getBackendCapabilities } from '~/modules/backend/store-backend-capabilities';
import { AudioLivePlayer } from '~/common/util/audio/AudioLivePlayer';
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
import { CapabilityElevenLabsSpeechSynthesis } from '~/common/components/useCapabilities';
import { apiStream } from '~/common/util/trpc.client';
import { convert_Base64_To_UInt8Array } from '~/common/util/blobUtils';
import { useUIPreferencesStore } from '~/common/stores/store-ui';
import { getElevenLabsData, useElevenLabsData } from './store-module-elevenlabs';
export const isValidElevenLabsApiKey = (apiKey?: string) => !!apiKey && apiKey.trim()?.length >= 32;
export const isElevenLabsEnabled = (apiKey?: string) =>
apiKey ? isValidElevenLabsApiKey(apiKey)
: getBackendCapabilities().hasVoiceElevenLabs;
export function useCapability(): CapabilityElevenLabsSpeechSynthesis {
const [clientApiKey, voiceId] = useElevenLabsData();
const isConfiguredServerSide = getBackendCapabilities().hasVoiceElevenLabs;
const isConfiguredClientSide = clientApiKey ? isValidElevenLabsApiKey(clientApiKey) : false;
const mayWork = isConfiguredServerSide || isConfiguredClientSide || !!voiceId;
return { mayWork, isConfiguredServerSide, isConfiguredClientSide };
}
interface ElevenLabsSpeakResult {
success: boolean;
audioBase64?: string; // Available when not streaming
}
/**
* Speaks text using ElevenLabs TTS
* @returns Object with success status and optionally the audio base64 (when not streaming)
*/
export async function elevenLabsSpeakText(text: string, voiceId: string | undefined, audioStreaming: boolean, audioTurbo: boolean): Promise<ElevenLabsSpeakResult> {
// Early validation
if (!(text?.trim())) {
// console.log('ElevenLabs: No text to speak');
return { success: false };
}
const { elevenLabsApiKey, elevenLabsVoiceId } = getElevenLabsData();
if (!isElevenLabsEnabled(elevenLabsApiKey)) {
// console.warn('ElevenLabs: Service not enabled or configured');
return { success: false };
}
const { preferredLanguage } = useUIPreferencesStore.getState();
const nonEnglish = !(preferredLanguage?.toLowerCase()?.startsWith('en'));
// audio live player instance, if needed
let liveAudioPlayer: AudioLivePlayer | undefined;
let playbackStarted = false;
let audioBase64: string | undefined;
try {
const stream = await apiStream.elevenlabs.speech.mutate({
xiKey: elevenLabsApiKey,
voiceId: voiceId || elevenLabsVoiceId,
text: text,
nonEnglish,
audioStreaming,
audioTurbo,
});
for await (const piece of stream) {
// ElevenLabs stream buffer
if (piece.audioChunk) {
try {
// create the live audio player as needed
// NOTE: in the future we can have a centralized audio playing system
if (!liveAudioPlayer)
liveAudioPlayer = new AudioLivePlayer();
// enqueue a decoded audio chunk - this will throw on malformed base64 data
const chunkArray = convert_Base64_To_UInt8Array(piece.audioChunk.base64, 'elevenLabsSpeakText (chunk)');
liveAudioPlayer.enqueueChunk(chunkArray.buffer);
playbackStarted = true;
} catch (audioError) {
console.error('ElevenLabs audio chunk error:', audioError);
return { success: false };
}
}
// ElevenLabs full audio buffer
else if (piece.audio) {
try {
// return base64 for potential reuse
if (!audioStreaming)
audioBase64 = piece.audio.base64;
// also consider merging LiveAudioPlayer into AudioPlayer - note this will throw on malformed base64 data
const audioArray = convert_Base64_To_UInt8Array(piece.audio.base64, 'elevenLabsSpeakText');
void AudioPlayer.playBuffer(audioArray.buffer); // fire/forget - it's a single piece of audio (could be long tho)
playbackStarted = true;
} catch (audioError) {
console.error('ElevenLabs audio buffer error:', audioError);
return { success: false };
}
}
// Errors
else if (piece.errorMessage) {
console.error('ElevenLabs error:', piece.errorMessage);
return { success: false };
} else if (piece.warningMessage) {
console.warn('ElevenLabs warning:', piece.warningMessage);
// Continue processing warnings
} else if (piece.control === 'start' || piece.control === 'end') {
// Control messages - continue processing
} else {
console.log('ElevenLabs unknown piece:', piece);
}
}
return { success: playbackStarted, audioBase64 };
} catch (error) {
console.error('ElevenLabs playback error:', error);
return { success: false };
}
}
-277
View File
@@ -1,277 +0,0 @@
import * as z from 'zod/v4';
import { createTRPCRouter, publicProcedure } from '~/server/trpc/trpc.server';
import { env } from '~/server/env.server';
import { fetchJsonOrTRPCThrow, fetchResponseOrTRPCThrow } from '~/server/trpc/trpc.router.fetchers';
// configuration
const SAFETY_TEXT_LENGTH = 1000;
const MIN_CHUNK_SIZE = 4096; // Minimum chunk size in bytes
// Schema definitions
export type SpeechInputSchema = z.infer<typeof speechInputSchema>;
export const speechInputSchema = z.object({
xiKey: z.string().optional(),
voiceId: z.string().optional(),
text: z.string(),
nonEnglish: z.boolean(),
audioStreaming: z.boolean(),
audioTurbo: z.boolean(),
});
export type VoiceSchema = z.infer<typeof voiceSchema>;
const voiceSchema = z.object({
id: z.string(),
name: z.string(),
description: z.string().nullable(),
previewUrl: z.string().nullable(),
category: z.string(),
default: z.boolean(),
});
export const elevenlabsRouter = createTRPCRouter({
/**
* List Voices available to this API key
*/
listVoices: publicProcedure
.input(z.object({
elevenKey: z.string().optional(),
}))
.output(z.object({
voices: z.array(voiceSchema),
}))
.query(async ({ input }) => {
const { elevenKey } = input;
const { headers, url } = elevenlabsAccess(elevenKey, '/v1/voices');
const voicesList = await fetchJsonOrTRPCThrow<ElevenlabsWire.VoicesList>({
url,
headers,
name: 'ElevenLabs',
});
// bring category != 'premade' to the top
voicesList.voices.sort((a, b) => {
if (a.category === 'premade' && b.category !== 'premade') return 1;
if (a.category !== 'premade' && b.category === 'premade') return -1;
return 0;
});
return {
voices: voicesList.voices.map((voice, idx) => ({
id: voice.voice_id,
name: voice.name,
description: voice.description,
previewUrl: voice.preview_url,
category: voice.category,
default: idx === 0,
})),
};
}),
/**
* Speech synthesis procedure using tRPC streaming
*/
speech: publicProcedure
.input(speechInputSchema)
.mutation(async function* ({ input: { xiKey, text, voiceId, nonEnglish, audioStreaming, audioTurbo }, ctx }) {
// start streaming back
yield { control: 'start' };
// Safety check: trim text that's too long
if (text.length > SAFETY_TEXT_LENGTH) {
text = text.slice(0, SAFETY_TEXT_LENGTH);
yield { warningMessage: 'text was truncated to maximum length' };
}
let response: Response;
try {
// Prepare the upstream request
const path = `/v1/text-to-speech/${elevenlabsVoiceId(voiceId)}${audioStreaming ? '/stream' : ''}`;
const { headers, url } = elevenlabsAccess(xiKey, path);
const body: ElevenlabsWire.TTSRequest = {
text: text,
model_id:
audioTurbo ? 'eleven_turbo_v2_5'
: nonEnglish ? 'eleven_multilingual_v2'
: 'eleven_multilingual_v2', // even for english, use the latest multilingual model
};
// Blocking fetch
response = await fetchResponseOrTRPCThrow({ url, method: 'POST', headers, body, signal: ctx.reqSignal, name: 'ElevenLabs' });
} catch (error: any) {
yield { errorMessage: `fetch issue: ${error.message || 'Unknown error'}` };
return;
}
// Parse headers
const responseHeaders = _safeParseTTSResponseHeaders(response.headers);
// If not streaming, return the entire audio
if (!audioStreaming) {
const audioArrayBuffer = await response.arrayBuffer();
yield {
audio: {
base64: Buffer.from(audioArrayBuffer).toString('base64'),
contentType: responseHeaders.contentType,
characterCost: responseHeaders.characterCost,
ttsLatencyMs: responseHeaders.ttsLatencyMs,
},
};
yield { control: 'end' };
return;
}
const reader = response.body?.getReader();
if (!reader) {
yield { errorMessage: 'stream issue: No reader' };
return;
}
// STREAM the audio chunks back to the client
try {
// Initialize a buffer to accumulate chunks
const accumulatedChunks: Uint8Array[] = [];
let accumulatedSize = 0;
// Read loop
while (true) {
const { value, done: readerDone } = await reader.read();
if (readerDone) break;
if (!value) continue;
// Accumulate chunks
accumulatedChunks.push(value);
accumulatedSize += value.length;
// When accumulated size reaches or exceeds MIN_CHUNK_SIZE, yield the chunk
if (accumulatedSize >= MIN_CHUNK_SIZE) {
yield {
audioChunk: {
base64: Buffer.concat(accumulatedChunks).toString('base64'),
},
};
// Reset the accumulation
accumulatedChunks.length = 0;
accumulatedSize = 0;
}
}
// If there's any remaining data, yield it as well
if (accumulatedSize) {
yield {
audioChunk: {
base64: Buffer.concat(accumulatedChunks).toString('base64'),
},
};
}
} catch (error: any) {
yield { errorMessage: `stream issue: ${error.message || 'Unknown error'}` };
return;
}
// end streaming (if a control error wasn't thrown)
yield { control: 'end' };
}),
});
/**
* Helper function to construct ElevenLabs API access details
*/
export function elevenlabsAccess(elevenKey: string | undefined, apiPath: string): { headers: HeadersInit; url: string } {
// API key
elevenKey = (elevenKey || env.ELEVENLABS_API_KEY || '').trim();
if (!elevenKey)
throw new Error('Missing ElevenLabs API key.');
// API host
let host = (env.ELEVENLABS_API_HOST || 'api.elevenlabs.io').trim();
if (!host.startsWith('http'))
host = `https://${host}`;
if (host.endsWith('/') && apiPath.startsWith('/'))
host = host.slice(0, -1);
return {
headers: {
'Accept': 'audio/mpeg',
'Content-Type': 'application/json',
'xi-api-key': elevenKey,
},
url: host + apiPath,
};
}
export function elevenlabsVoiceId(voiceId?: string): string {
return voiceId?.trim() || env.ELEVENLABS_VOICE_ID || '21m00Tcm4TlvDq8ikWAM';
}
function _safeParseTTSResponseHeaders(headers: Headers): ElevenlabsWire.TTSResponseHeaders {
return {
contentType: headers.get('content-type') || 'audio/mpeg',
characterCost: parseInt(headers.get('character-cost') || '0'),
currentConcurrentRequests: parseInt(headers.get('current-concurrent-requests') || '0'),
maximumConcurrentRequests: parseInt(headers.get('maximum-concurrent-requests') || '0'),
ttsLatencyMs: parseInt(headers.get('tts-latency-ms') || '0'),
};
}
/// This is the upstream API [rev-eng on 2023-04-12]
export namespace ElevenlabsWire {
export interface TTSRequest {
text: string;
model_id?:
| 'eleven_monolingual_v1'
| 'eleven_multilingual_v1'
| 'eleven_multilingual_v2'
| 'eleven_turbo_v2'
| 'eleven_turbo_v2_5';
voice_settings?: {
stability: number;
similarity_boost: number;
};
}
export interface TTSResponseHeaders {
// Response metadata
contentType: string; // Should be 'audio/mpeg'
// Cost and usage metrics
characterCost: number; // Cost in characters for this generation
currentConcurrentRequests: number; // Current number of concurrent requests
maximumConcurrentRequests: number; // Maximum allowed concurrent requests
ttsLatencyMs?: number; // Time taken to generate speech (not in streaming mode)
}
export interface VoicesList {
voices: Voice[];
}
interface Voice {
voice_id: string;
name: string;
//samples: Sample[];
category: string;
// fine_tuning: FineTuning;
labels: Record<string, string>;
description: string;
preview_url: string;
// available_for_tiers: string[];
settings: {
stability: number;
similarity_boost: number;
};
}
}
@@ -1,50 +0,0 @@
import { create } from 'zustand';
import { persist } from 'zustand/middleware';
import { useShallow } from 'zustand/react/shallow';
interface ModuleElevenlabsStore {
// ElevenLabs Text to Speech settings
elevenLabsApiKey: string;
setElevenLabsApiKey: (apiKey: string) => void;
elevenLabsVoiceId: string;
setElevenLabsVoiceId: (voiceId: string) => void;
}
const useElevenlabsStore = create<ModuleElevenlabsStore>()(
persist(
(set) => ({
// ElevenLabs Text to Speech settings
elevenLabsApiKey: '',
setElevenLabsApiKey: (elevenLabsApiKey: string) => set({ elevenLabsApiKey }),
elevenLabsVoiceId: '',
setElevenLabsVoiceId: (elevenLabsVoiceId: string) => set({ elevenLabsVoiceId }),
}),
{
name: 'app-module-elevenlabs',
}),
);
export const useElevenLabsApiKey = (): [string, (apiKey: string) => void] => {
const apiKey = useElevenlabsStore(state => state.elevenLabsApiKey);
return [apiKey, useElevenlabsStore.getState().setElevenLabsApiKey];
};
export const useElevenLabsVoiceId = (): [string, (voiceId: string) => void] => {
const voiceId = useElevenlabsStore(state => state.elevenLabsVoiceId);
return [voiceId, useElevenlabsStore.getState().setElevenLabsVoiceId];
};
export const useElevenLabsData = (): [string, string] =>
useElevenlabsStore(useShallow(state => [state.elevenLabsApiKey, state.elevenLabsVoiceId]));
export const getElevenLabsData = (): { elevenLabsApiKey: string, elevenLabsVoiceId: string } =>
useElevenlabsStore.getState();
@@ -1,102 +0,0 @@
import * as React from 'react';
import { CircularProgress, Option, Select } from '@mui/joy';
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
import RecordVoiceOverTwoToneIcon from '@mui/icons-material/RecordVoiceOverTwoTone';
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
import { apiQuery } from '~/common/util/trpc.client';
import { VoiceSchema } from './elevenlabs.router';
import { isElevenLabsEnabled } from './elevenlabs.client';
import { useElevenLabsApiKey, useElevenLabsVoiceId } from './store-module-elevenlabs';
function VoicesDropdown(props: {
isValidKey: boolean,
isFetchingVoices: boolean,
isErrorVoices: boolean,
disabled?: boolean,
voices: VoiceSchema[],
voiceId: string | null,
setVoiceId: (voiceId: string) => void,
}) {
const handleVoiceChange = (_event: any, value: string | null) => props.setVoiceId(value || '');
return (
<Select
value={props.voiceId} onChange={handleVoiceChange}
variant='outlined' disabled={props.disabled || !props.voices.length}
// color={props.isErrorVoices ? 'danger' : undefined}
placeholder={props.isErrorVoices ? 'Issue loading voices' : props.isValidKey ? 'Select a voice' : 'Missing API Key'}
startDecorator={<RecordVoiceOverTwoToneIcon />}
endDecorator={props.isValidKey && props.isFetchingVoices && <CircularProgress size='sm' />}
indicator={<KeyboardArrowDownIcon />}
slotProps={{
root: { sx: { width: '100%' } },
indicator: { sx: { opacity: 0.5 } },
}}
>
{props.voices.map(voice => (
<Option key={voice.id} value={voice.id}>
{voice.name}
</Option>
))}
</Select>
);
}
export function useElevenLabsVoices() {
const [apiKey] = useElevenLabsApiKey();
const isConfigured = isElevenLabsEnabled(apiKey);
const { data, isError, isFetching, isPending } = apiQuery.elevenlabs.listVoices.useQuery({ elevenKey: apiKey }, {
enabled: isConfigured,
staleTime: 1000 * 60 * 5, // 5 minutes
});
return {
isConfigured,
isError,
isFetching,
hasVoices: !isPending && !!data?.voices.length,
voices: data?.voices || [],
};
}
export function useElevenLabsVoiceDropdown(autoSpeak: boolean, disabled?: boolean) {
// external state
const { isConfigured, isError, isFetching, hasVoices, voices } = useElevenLabsVoices();
const [voiceId, setVoiceId] = useElevenLabsVoiceId();
// derived state
const voice: VoiceSchema | undefined = voices.find(voice => voice.id === voiceId);
// [E] autoSpeak
const previewUrl = (autoSpeak && voice?.previewUrl) || null;
React.useEffect(() => {
if (previewUrl)
void AudioPlayer.playUrl(previewUrl);
}, [previewUrl]);
const voicesDropdown = React.useMemo(() =>
<VoicesDropdown
isValidKey={isConfigured} isFetchingVoices={isFetching} isErrorVoices={isError} disabled={disabled}
voices={voices}
voiceId={voiceId} setVoiceId={setVoiceId}
/>,
[disabled, isConfigured, isError, isFetching, setVoiceId, voiceId, voices],
);
return {
hasVoices,
voiceId,
voiceName: voice?.name,
voicesDropdown,
};
}
+1 -2
View File
@@ -20,8 +20,7 @@ export interface IModelVendor<TServiceSettings extends Record<string, any> = {},
readonly hasServerConfigKey?: keyof BackendCapabilities;
/// client-side-fetch ///
readonly csfKey?: string; // was keyof TServiceSettings, but caused TS troubles
readonly csfAvailable?: (setup?: Partial<TServiceSettings>) => boolean; // undefined: not even, false: conditions not met
readonly csfAvailable?: (setup?: Partial<TServiceSettings>) => boolean; // undefined: not supported, false: conditions not met
/// abstraction interface ///
+11 -2
View File
@@ -9,6 +9,7 @@ import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { FormTextField } from '~/common/components/forms/FormTextField';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
@@ -36,10 +37,11 @@ export function AlibabaServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorAlibaba);
// derived state
const { oaiKey: alibabaOaiKey, oaiHost: alibabaOaiHost } = serviceAccess;
const { clientSideFetch, oaiKey: alibabaOaiKey, oaiHost: alibabaOaiHost } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const shallFetchSucceed = !needsUserKey || (!!alibabaOaiKey && serviceSetupValid);
const showKeyError = !!alibabaOaiKey && !serviceSetupValid;
const showAdvanced = advanced.on || !!clientSideFetch;
// fetch models
const { isFetching, refetch, isError, error } =
@@ -73,7 +75,7 @@ export function AlibabaServiceSetup(props: { serviceId: DModelsServiceId }) {
{/* See the <ExternalLink href={ALIBABA_REG_LINK}>Alibaba Cloud Model Studio</ExternalLink> for more information.*/}
{/*</Typography>*/}
{advanced.on && <FormTextField
{showAdvanced && <FormTextField
autoCompleteId='alibaba-host'
title='API Endpoint'
tooltip={`The API endpoint for the Alibaba Cloud OpenAI service, to be used instead of the default endpoint.`}
@@ -82,6 +84,13 @@ export function AlibabaServiceSetup(props: { serviceId: DModelsServiceId }) {
onChange={text => updateSettings({ alibabaOaiHost: text })}
/>}
{showAdvanced && <SetupFormClientSideToggle
visible={!!alibabaOaiKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Alibaba Cloud API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+9
View File
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
interface DAlibabaServiceSettings {
alibabaOaiKey: string;
alibabaOaiHost: string;
csf?: boolean;
}
export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAccessSchema> = {
@@ -17,6 +18,9 @@ export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAcc
instanceLimit: 1,
hasServerConfigKey: 'hasLlmAlibaba',
/// client-side-fetch ///
csfAvailable: _csfAlibabaAvailable,
// functions
initializeSetup: () => ({
alibabaOaiKey: '',
@@ -27,6 +31,7 @@ export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAcc
},
getTransportAccess: (partialSetup) => ({
dialect: 'alibaba',
clientSideFetch: _csfAlibabaAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.alibabaOaiKey || '',
oaiOrg: '',
oaiHost: partialSetup?.alibabaOaiHost || '',
@@ -37,3 +42,7 @@ export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAcc
// OpenAI transport ('alibaba' dialect in 'access')
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfAlibabaAvailable(s?: Partial<DAlibabaServiceSettings>) {
return !!s?.alibabaOaiKey;
}
@@ -6,7 +6,6 @@ import { useChatAutoAI } from '../../../../apps/chat/store-app-chat';
import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types';
import { AlreadySet } from '~/common/components/AlreadySet';
import { ExternalLink } from '~/common/components/ExternalLink';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
import { FormSwitchControl } from '~/common/components/forms/FormSwitchControl';
@@ -110,7 +109,7 @@ export function AnthropicServiceSetup(props: { serviceId: DModelsServiceId }) {
{showAdvanced && <SetupFormClientSideToggle
visible={!!anthropicKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ anthropicCSF: on })}
onChange={on => updateSettings({ csf: on })}
helpText="Fetch models and make requests directly to Anthropic's API using your browser instead of through the server. Useful for bypassing server limitations or ensuring requests use your API key directly."
/>}
+2 -3
View File
@@ -10,7 +10,7 @@ export const isValidAnthropicApiKey = (apiKey?: string) => !!apiKey && (apiKey.s
interface DAnthropicServiceSettings {
anthropicKey: string;
anthropicHost: string;
anthropicCSF?: boolean;
csf?: boolean;
heliconeKey: string;
}
@@ -25,13 +25,12 @@ export const ModelVendorAnthropic: IModelVendor<DAnthropicServiceSettings, Anthr
hasServerConfigKey: 'hasLlmAnthropic',
/// client-side-fetch ///
csfKey: 'anthropicCSF',
csfAvailable: _csfAnthropicAvailable,
// functions
getTransportAccess: (partialSetup): AnthropicAccessSchema => ({
dialect: 'anthropic',
clientSideFetch: _csfAnthropicAvailable(partialSetup) && !!partialSetup?.anthropicCSF,
clientSideFetch: _csfAnthropicAvailable(partialSetup) && !!partialSetup?.csf,
anthropicKey: partialSetup?.anthropicKey || '',
anthropicHost: partialSetup?.anthropicHost || null,
heliconeKey: partialSetup?.heliconeKey || null,
+13 -2
View File
@@ -9,8 +9,10 @@ import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { FormTextField } from '~/common/components/forms/FormTextField';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { asValidURL } from '~/common/util/urlUtils';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { useLlmUpdateModels } from '../../llm.client.hooks';
@@ -22,6 +24,7 @@ import { isValidAzureApiKey, ModelVendorAzure } from './azure.vendor';
export function AzureServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
const [checkboxExpanded, setCheckboxExpanded] = React.useState(false);
// external state
@@ -29,8 +32,9 @@ export function AzureServiceSetup(props: { serviceId: DModelsServiceId }) {
useServiceSetup(props.serviceId, ModelVendorAzure);
// derived state
const { oaiKey: azureKey, oaiHost: azureEndpoint } = serviceAccess;
const { clientSideFetch, oaiKey: azureKey, oaiHost: azureEndpoint } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
const keyValid = isValidAzureApiKey(azureKey);
const keyError = (/*needsUserKey ||*/ !!azureKey) && !keyValid;
@@ -81,7 +85,14 @@ export function AzureServiceSetup(props: { serviceId: DModelsServiceId }) {
placeholder='...'
/>
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!(azureKey && azureEndpoint)}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Azure OpenAI API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+10 -1
View File
@@ -10,6 +10,7 @@ export const isValidAzureApiKey = (apiKey?: string) => !!apiKey && apiKey.length
interface DAzureServiceSettings {
azureEndpoint: string;
azureKey: string;
csf?: boolean;
}
/** Implementation Notes for the Azure Vendor
@@ -37,9 +38,13 @@ export const ModelVendorAzure: IModelVendor<DAzureServiceSettings, OpenAIAccessS
instanceLimit: 2,
hasServerConfigKey: 'hasLlmAzureOpenAI',
/// client-side-fetch ///
csfAvailable: _csfAzureAvailable,
// functions
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
dialect: 'azure',
clientSideFetch: _csfAzureAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.azureKey || '',
oaiOrg: '',
oaiHost: partialSetup?.azureEndpoint || '',
@@ -50,4 +55,8 @@ export const ModelVendorAzure: IModelVendor<DAzureServiceSettings, OpenAIAccessS
// OpenAI transport ('azure' dialect in 'access')
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
};
function _csfAzureAvailable(s?: Partial<DAzureServiceSettings>) {
return !!(s?.azureKey && s?.azureEndpoint);
}
@@ -5,6 +5,7 @@ import { AlreadySet } from '~/common/components/AlreadySet';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
@@ -30,8 +31,9 @@ export function DeepseekAIServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorDeepseek);
// derived state
const { oaiKey: deepseekKey } = serviceAccess;
const { clientSideFetch, oaiKey: deepseekKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// validate if url is a well formed proper url with zod
const shallFetchSucceed = !needsUserKey || (!!deepseekKey && serviceSetupValid);
@@ -57,6 +59,13 @@ export function DeepseekAIServiceSetup(props: { serviceId: DModelsServiceId }) {
placeholder='...'
/>
{showAdvanced && <SetupFormClientSideToggle
visible={!!deepseekKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Deepseek API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
export interface DDeepseekServiceSettings {
deepseekKey: string;
csf?: boolean;
}
export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIAccessSchema> = {
@@ -17,6 +18,9 @@ export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIA
instanceLimit: 1,
hasServerConfigKey: 'hasLlmDeepseek',
/// client-side-fetch ///
csfAvailable: _csfDeepseekAvailable,
// functions
initializeSetup: () => ({
deepseekKey: '',
@@ -26,6 +30,7 @@ export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIA
},
getTransportAccess: (partialSetup) => ({
dialect: 'deepseek',
clientSideFetch: _csfDeepseekAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.deepseekKey || '',
oaiOrg: '',
oaiHost: '',
@@ -37,3 +42,7 @@ export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIA
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfDeepseekAvailable(s?: Partial<DDeepseekServiceSettings>) {
return !!s?.deepseekKey;
}
+1 -1
View File
@@ -109,7 +109,7 @@ export function GeminiServiceSetup(props: { serviceId: DModelsServiceId }) {
{showAdvanced && <SetupFormClientSideToggle
visible={!!geminiKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ geminiCSF: on })}
onChange={on => updateSettings({ csf: on })}
helpText="Fetch models and make requests directly to Google's Gemini API using your browser instead of through the server."
/>}
+2 -3
View File
@@ -9,7 +9,7 @@ import type { IModelVendor } from '../IModelVendor';
interface DGeminiServiceSettings {
geminiKey: string;
geminiHost: string;
geminiCSF?: boolean;
csf?: boolean;
minSafetyLevel: GeminiWire_Safety.HarmBlockThreshold;
}
@@ -34,7 +34,6 @@ export const ModelVendorGemini: IModelVendor<DGeminiServiceSettings, GeminiAcces
hasServerConfigKey: 'hasLlmGemini',
/// client-side-fetch ///
csfKey: 'geminiCSF',
csfAvailable: _csfGeminiAvailable,
// functions
@@ -48,7 +47,7 @@ export const ModelVendorGemini: IModelVendor<DGeminiServiceSettings, GeminiAcces
},
getTransportAccess: (partialSetup): GeminiAccessSchema => ({
dialect: 'gemini',
clientSideFetch: _csfGeminiAvailable(partialSetup) && !!partialSetup?.geminiCSF,
clientSideFetch: _csfGeminiAvailable(partialSetup) && !!partialSetup?.csf,
geminiKey: partialSetup?.geminiKey || '',
geminiHost: partialSetup?.geminiHost || '',
minSafetyLevel: partialSetup?.minSafetyLevel || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+15 -4
View File
@@ -1,13 +1,13 @@
import * as React from 'react';
import { Typography } from '@mui/joy';
import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types';
import { AlreadySet } from '~/common/components/AlreadySet';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { ModelVendorGroq } from './groq.vendor';
@@ -20,6 +20,9 @@ const GROQ_REG_LINK = 'https://console.groq.com/keys';
export function GroqServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const {
service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs,
@@ -27,8 +30,9 @@ export function GroqServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorGroq);
// derived state
const { oaiKey: groqKey } = serviceAccess;
const { clientSideFetch, oaiKey: groqKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// key validation
const shallFetchSucceed = !needsUserKey || (!!groqKey && serviceSetupValid);
@@ -54,7 +58,14 @@ export function GroqServiceSetup(props: { serviceId: DModelsServiceId }) {
placeholder='...'
/>
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!groqKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Groq API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+9
View File
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
interface DGroqServiceSettings {
groqKey: string;
csf?: boolean;
}
export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSchema> = {
@@ -17,6 +18,9 @@ export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSch
instanceLimit: 1,
hasServerConfigKey: 'hasLlmGroq',
/// client-side-fetch ///
csfAvailable: _csfGroqAvailable,
// functions
initializeSetup: () => ({
groqKey: '',
@@ -26,6 +30,7 @@ export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSch
},
getTransportAccess: (partialSetup) => ({
dialect: 'groq',
clientSideFetch: _csfGroqAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.groqKey || '',
oaiOrg: '',
oaiHost: '',
@@ -37,3 +42,7 @@ export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSch
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfGroqAvailable(s?: Partial<DGroqServiceSettings>) {
return !!s?.groqKey;
}
+15 -2
View File
@@ -9,8 +9,10 @@ import { ExpanderAccordion } from '~/common/components/ExpanderAccordion';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { VideoPlayerYouTube } from '~/common/components/VideoPlayerYouTube';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { useLlmUpdateModels } from '../../llm.client.hooks';
import { useServiceSetup } from '../useServiceSetup';
@@ -20,12 +22,16 @@ import { ModelVendorLMStudio } from './lmstudio.vendor';
export function LMStudioServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const { service, serviceAccess, updateSettings } =
useServiceSetup(props.serviceId, ModelVendorLMStudio);
// derived state
const { oaiHost } = serviceAccess;
const { clientSideFetch, oaiHost } = serviceAccess;
const showAdvanced = advanced.on || !!clientSideFetch;
// validate if url is a well formed proper url with zod
const urlSchema = z.url().startsWith('http');
@@ -62,7 +68,14 @@ export function LMStudioServiceSetup(props: { serviceId: DModelsServiceId }) {
value={oaiHost} onChange={value => updateSettings({ oaiHost: value })}
/>
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!oaiHost}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to LM Studio from your browser. Requires CORS to be enabled in LM Studio.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+9
View File
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
interface DLMStudioServiceSettings {
oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path)
csf?: boolean;
}
export const ModelVendorLMStudio: IModelVendor<DLMStudioServiceSettings, OpenAIAccessSchema> = {
@@ -16,12 +17,16 @@ export const ModelVendorLMStudio: IModelVendor<DLMStudioServiceSettings, OpenAIA
location: 'local',
instanceLimit: 1,
/// client-side-fetch ///
csfAvailable: _csfLMStudioAvailable,
// functions
initializeSetup: () => ({
oaiHost: 'http://localhost:1234',
}),
getTransportAccess: (partialSetup) => ({
dialect: 'lmstudio',
clientSideFetch: _csfLMStudioAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: '',
oaiOrg: '',
oaiHost: partialSetup?.oaiHost || '',
@@ -33,3 +38,7 @@ export const ModelVendorLMStudio: IModelVendor<DLMStudioServiceSettings, OpenAIA
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfLMStudioAvailable(s?: Partial<DLMStudioServiceSettings>) {
return !!s?.oaiHost;
}
+1 -1
View File
@@ -101,7 +101,7 @@ export function LocalAIServiceSetup(props: { serviceId: DModelsServiceId }) {
<SetupFormClientSideToggle
visible={true}
checked={!!clientSideFetch}
onChange={on => updateSettings({ localAICSF: on })}
onChange={on => updateSettings({ csf: on })}
helpText="Fetch models and make requests directly from your LocalAI instance using the browser. Recommended for local setups."
/>
+4 -5
View File
@@ -4,10 +4,10 @@ import type { OpenAIAccessSchema } from '../../server/openai/openai.access';
import { ModelVendorOpenAI } from '../openai/openai.vendor';
interface DLocalAIServiceSettings {
export interface DLocalAIServiceSettings {
localAIHost: string; // use OpenAI-compatible non-default hosts (full origin path)
localAIKey: string; // use OpenAI-compatible API keys
localAICSF?: boolean;
csf?: boolean;
}
export const ModelVendorLocalAI: IModelVendor<DLocalAIServiceSettings, OpenAIAccessSchema> = {
@@ -24,18 +24,17 @@ export const ModelVendorLocalAI: IModelVendor<DLocalAIServiceSettings, OpenAIAcc
},
/// client-side-fetch ///
csfKey: 'localAICSF',
csfAvailable: _csfLocalAIAvailable,
// functions
initializeSetup: () => ({
localAIHost: '',
localAIKey: '',
// localAICSF: true, // eventually, but requires CORS support on the server: -e CORS=true -e CORS_ALLOW_ORIGINS="*"
// csf: true, // eventually, but requires CORS support on the server: -e CORS=true -e CORS_ALLOW_ORIGINS="*"
}),
getTransportAccess: (partialSetup) => ({
dialect: 'localai',
clientSideFetch: _csfLocalAIAvailable(partialSetup) && !!partialSetup?.localAICSF,
clientSideFetch: _csfLocalAIAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.localAIKey || '',
oaiOrg: '',
oaiHost: partialSetup?.localAIHost || '',
+15 -2
View File
@@ -7,7 +7,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { useLlmUpdateModels } from '../../llm.client.hooks';
@@ -21,13 +23,17 @@ const MISTRAL_REG_LINK = 'https://console.mistral.ai/';
export function MistralServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, serviceSetupValid, updateSettings } =
useServiceSetup(props.serviceId, ModelVendorMistral);
// derived state
const { oaiKey: mistralKey } = serviceAccess;
const { clientSideFetch, oaiKey: mistralKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
const shallFetchSucceed = !needsUserKey || (!!mistralKey && serviceSetupValid);
const showKeyError = !!mistralKey && !serviceSetupValid;
@@ -56,7 +62,14 @@ export function MistralServiceSetup(props: { serviceId: DModelsServiceId }) {
{/* Note the elegance of the numbers, representing the Year and Month or release (YYMM).*/}
{/*</Typography>*/}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!mistralKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Mistral API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+10 -2
View File
@@ -6,7 +6,7 @@ import { DOpenAIServiceSettings, ModelVendorOpenAI } from '../openai/openai.vend
// special symbols
type DMistralServiceSettings = Pick<DOpenAIServiceSettings, 'oaiKey' | 'oaiHost'>;
type DMistralServiceSettings = Pick<DOpenAIServiceSettings, 'oaiKey' | 'oaiHost' | 'csf'>;
/** Implementation Notes for the Mistral vendor
@@ -20,6 +20,9 @@ export const ModelVendorMistral: IModelVendor<DMistralServiceSettings, OpenAIAcc
instanceLimit: 1,
hasServerConfigKey: 'hasLlmMistral',
/// client-side-fetch ///
csfAvailable: _csfMistralAvailable,
// functions
initializeSetup: () => ({
oaiHost: 'https://api.mistral.ai/',
@@ -30,6 +33,7 @@ export const ModelVendorMistral: IModelVendor<DMistralServiceSettings, OpenAIAcc
},
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
dialect: 'mistral',
clientSideFetch: _csfMistralAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.oaiKey || '',
oaiOrg: '',
oaiHost: partialSetup?.oaiHost || '',
@@ -40,4 +44,8 @@ export const ModelVendorMistral: IModelVendor<DMistralServiceSettings, OpenAIAcc
// OpenAI transport ('mistral' dialect in 'access')
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
};
function _csfMistralAvailable(s?: Partial<DMistralServiceSettings>) {
return !!s?.oaiKey;
}
+15 -2
View File
@@ -5,7 +5,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { ModelVendorMoonshot } from './moonshot.vendor';
@@ -18,6 +20,9 @@ const MOONSHOT_API_LINK = 'https://platform.moonshot.ai/console/api-keys';
export function MoonshotServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const {
service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs,
@@ -25,8 +30,9 @@ export function MoonshotServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorMoonshot);
// derived state
const { oaiKey: moonshotKey } = serviceAccess;
const { clientSideFetch, oaiKey: moonshotKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// key validation
const shallFetchSucceed = !needsUserKey || (!!moonshotKey && serviceSetupValid);
@@ -52,7 +58,14 @@ export function MoonshotServiceSetup(props: { serviceId: DModelsServiceId }) {
placeholder='...'
/>
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!moonshotKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Moonshot API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+9
View File
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
interface DMoonshotServiceSettings {
moonshotKey: string;
csf?: boolean;
}
export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIAccessSchema> = {
@@ -17,6 +18,9 @@ export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIA
instanceLimit: 1,
hasServerConfigKey: 'hasLlmMoonshot',
/// client-side-fetch ///
csfAvailable: _csfMoonshotAvailable,
// functions
initializeSetup: () => ({
moonshotKey: '',
@@ -26,6 +30,7 @@ export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIA
},
getTransportAccess: (partialSetup) => ({
dialect: 'moonshot',
clientSideFetch: _csfMoonshotAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.moonshotKey || '',
oaiOrg: '',
oaiHost: '',
@@ -37,3 +42,7 @@ export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIA
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfMoonshotAvailable(s?: Partial<DMoonshotServiceSettings>) {
return !!s?.moonshotKey;
}
+1 -1
View File
@@ -62,7 +62,7 @@ export function OllamaServiceSetup(props: { serviceId: DModelsServiceId }) {
<SetupFormClientSideToggle
visible={true}
checked={!!clientSideFetch}
onChange={on => updateSettings({ ollamaCSF: on })}
onChange={on => updateSettings({ csf: on })}
helpText="Fetch models and make requests directly from your local Ollama instance using the browser. Recommended for local setups."
/>
+3 -4
View File
@@ -6,7 +6,7 @@ import type { OllamaAccessSchema } from '../../server/ollama/ollama.access';
interface DOllamaServiceSettings {
ollamaHost: string;
ollamaCSF?: boolean;
csf?: boolean;
}
@@ -20,17 +20,16 @@ export const ModelVendorOllama: IModelVendor<DOllamaServiceSettings, OllamaAcces
hasServerConfigKey: 'hasLlmOllama',
/// client-side-fetch ///
csfKey: 'ollamaCSF',
csfAvailable: _csfOllamaAvailable,
// functions
initializeSetup: () => ({
ollamaHost: '',
// ollamaCSF: true, // eventually
// csf: true, // eventually
}),
getTransportAccess: (partialSetup): OllamaAccessSchema => ({
dialect: 'ollama',
clientSideFetch: _csfOllamaAvailable(partialSetup) && !!partialSetup?.ollamaCSF,
clientSideFetch: _csfOllamaAvailable(partialSetup) && !!partialSetup?.csf,
ollamaHost: partialSetup?.ollamaHost || '',
}),
+1 -1
View File
@@ -110,7 +110,7 @@ export function OpenAIServiceSetup(props: { serviceId: DModelsServiceId }) {
{showAdvanced && <SetupFormClientSideToggle
visible={!!oaiHost || !!oaiKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ oaiCSF: on })}
onChange={on => updateSettings({ csf: on })}
helpText="Fetch models and make requests directly to OpenAI's Responses / Completions and List Models API using your browser instead of through the server."
/>}
+2 -3
View File
@@ -11,7 +11,7 @@ export interface DOpenAIServiceSettings {
oaiKey: string;
oaiOrg: string;
oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path)
oaiCSF?: boolean;
csf?: boolean;
heliKey: string; // helicone key (works in conjunction with oaiHost)
moderationCheck: boolean;
}
@@ -26,13 +26,12 @@ export const ModelVendorOpenAI: IModelVendor<DOpenAIServiceSettings, OpenAIAcces
hasServerConfigKey: 'hasLlmOpenAI',
/// client-side-fetch ///
csfKey: 'oaiCSF',
csfAvailable: _csfOpenAIAvailable,
// functions
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
dialect: 'openai',
clientSideFetch: _csfOpenAIAvailable(partialSetup) && !partialSetup?.oaiCSF,
clientSideFetch: _csfOpenAIAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: '',
oaiOrg: '',
oaiHost: '',
+13 -3
View File
@@ -8,7 +8,9 @@ import { ExternalLink } from '~/common/components/ExternalLink';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { useLlmUpdateModels } from '../../llm.client.hooks';
@@ -23,7 +25,7 @@ const OPENPIPE_API_KEY_LINK = 'https://app.openpipe.ai/settings';
export function OpenPipeServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
// const advanced = useToggleableBoolean();
const advanced = useToggleableBoolean();
// external state
const {
@@ -32,8 +34,9 @@ export function OpenPipeServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorOpenPipe);
// derived state
const { oaiKey: openPipeKey, oaiOrg: openPipeTags } = serviceAccess;
const { clientSideFetch, oaiKey: openPipeKey, oaiOrg: openPipeTags } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// validate if url is a well formed proper url with zod
const shallFetchSucceed = !needsUserKey || (!!openPipeKey && serviceSetupValid);
@@ -80,7 +83,14 @@ export function OpenPipeServiceSetup(props: { serviceId: DModelsServiceId }) {
and <strong>fine-tune</strong> and deploy custom models cost-effectively for specific tasks.
</Typography>
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!openPipeKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to OpenPipe API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
+9
View File
@@ -7,6 +7,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
export interface DOpenPipeServiceSettings {
openPipeKey: string;
openPipeTags: string; // hack: this will travel as 'oaiOrg' in the access schema - then interpreted in the openAIAccess() function
csf?: boolean;
}
export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIAccessSchema> = {
@@ -18,6 +19,9 @@ export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIA
instanceLimit: 1,
hasServerConfigKey: 'hasLlmOpenPipe',
/// client-side-fetch ///
csfAvailable: _csfOpenPipeAvailable,
// functions
initializeSetup: () => ({
openPipeKey: '',
@@ -28,6 +32,7 @@ export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIA
},
getTransportAccess: (partialSetup) => ({
dialect: 'openpipe',
clientSideFetch: _csfOpenPipeAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.openPipeKey || '',
oaiOrg: partialSetup?.openPipeTags || '', // HACK: use tags for org - should use type discrimination
oaiHost: '',
@@ -39,3 +44,7 @@ export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIA
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfOpenPipeAvailable(s?: Partial<DOpenPipeServiceSettings>) {
return !!s?.openPipeKey;
}
@@ -1,6 +1,6 @@
import * as React from 'react';
import { Box, Button, Chip, Typography } from '@mui/joy';
import { Box, Button, Typography } from '@mui/joy';
import LaunchIcon from '@mui/icons-material/Launch';
import VisibilityOffOutlinedIcon from '@mui/icons-material/VisibilityOffOutlined';
import VisibilityOutlinedIcon from '@mui/icons-material/VisibilityOutlined';
@@ -12,9 +12,11 @@ import { getLLMPricing } from '~/common/stores/llms/llms.types';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { PhGift } from '~/common/components/icons/phosphor/PhGift';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { getCallbackUrl } from '~/common/app.routes';
import { llmsStoreActions, llmsStoreState } from '~/common/stores/llms/store-llms';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { useLlmUpdateModels } from '../../llm.client.hooks';
@@ -25,13 +27,17 @@ import { isValidOpenRouterKey, ModelVendorOpenRouter } from './openrouter.vendor
export function OpenRouterServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, serviceHasVisibleLLMs, updateSettings } =
useServiceSetup(props.serviceId, ModelVendorOpenRouter);
// derived state
const { oaiKey } = serviceAccess;
const { clientSideFetch, oaiKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
const keyValid = isValidOpenRouterKey(oaiKey);
const keyError = (/*needsUserKey ||*/ !!oaiKey) && !keyValid;
@@ -124,8 +130,15 @@ export function OpenRouterServiceSetup(props: { serviceId: DModelsServiceId }) {
{/* These are usually moderated by the upstream provider (e.g. OpenAI).*/}
{/*</Typography>*/}
{showAdvanced && <SetupFormClientSideToggle
visible={!!oaiKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to OpenRouter API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton
refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError}
refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} advanced={advanced}
leftButton={
<Box sx={{ display: 'flex', flexWrap: 'wrap', gap: 1 }}>
<Button
@@ -13,6 +13,7 @@ export const isValidOpenRouterKey = (apiKey?: string) => !!apiKey && apiKey.star
export interface DOpenRouterServiceSettings {
oaiKey: string;
oaiHost: string;
csf?: boolean;
}
/**
@@ -36,6 +37,9 @@ export const ModelVendorOpenRouter: IModelVendor<DOpenRouterServiceSettings, Ope
hasFreeModels: true,
hasServerConfigKey: 'hasLlmOpenRouter',
/// client-side-fetch ///
csfAvailable: _csfOpenRouterAvailable,
// functions
initializeSetup: (): DOpenRouterServiceSettings => ({
oaiHost: 'https://openrouter.ai/api',
@@ -43,6 +47,7 @@ export const ModelVendorOpenRouter: IModelVendor<DOpenRouterServiceSettings, Ope
}),
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
dialect: 'openrouter',
clientSideFetch: _csfOpenRouterAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.oaiKey || '',
oaiOrg: '',
oaiHost: partialSetup?.oaiHost || '',
@@ -75,3 +80,7 @@ export const ModelVendorOpenRouter: IModelVendor<DOpenRouterServiceSettings, Ope
// rate limit timestamp
let nextGenerationTs = 0;
function _csfOpenRouterAvailable(s?: Partial<DOpenRouterServiceSettings>) {
return !!s?.oaiKey;
}
@@ -7,7 +7,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { ModelVendorPerplexity } from './perplexity.vendor';
@@ -20,6 +22,9 @@ const PERPLEXITY_REG_LINK = 'https://www.perplexity.ai/settings/api';
export function PerplexityServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const {
service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs,
@@ -27,8 +32,9 @@ export function PerplexityServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorPerplexity);
// derived state
const { oaiKey: perplexityKey } = serviceAccess;
const { clientSideFetch, oaiKey: perplexityKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// key validation
const shallFetchSucceed = !needsUserKey || (!!perplexityKey && serviceSetupValid);
@@ -59,7 +65,14 @@ export function PerplexityServiceSetup(props: { serviceId: DModelsServiceId }) {
as a service for a variety of models. See the <Link href='https://www.perplexity.ai/' target='_blank'>Perplexity AI</Link> website for more information.
</Typography>
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!perplexityKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Perplexity API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
interface DPerpexityServiceSettings {
perplexityKey: string;
csf?: boolean;
}
export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, OpenAIAccessSchema> = {
@@ -17,6 +18,9 @@ export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, Open
instanceLimit: 1,
hasServerConfigKey: 'hasLlmPerplexity',
/// client-side-fetch ///
csfAvailable: _csfPerplexityAvailable,
// functions
initializeSetup: () => ({
perplexityKey: '',
@@ -26,6 +30,7 @@ export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, Open
},
getTransportAccess: (partialSetup) => ({
dialect: 'perplexity',
clientSideFetch: _csfPerplexityAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.perplexityKey || '',
oaiOrg: '',
oaiHost: '',
@@ -37,3 +42,7 @@ export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, Open
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfPerplexityAvailable(s?: Partial<DPerpexityServiceSettings>) {
return !!s?.perplexityKey;
}
@@ -1,6 +1,6 @@
import * as React from 'react';
import { Alert, Typography } from '@mui/joy';
import { Alert } from '@mui/joy';
import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types';
import { AlreadySet } from '~/common/components/AlreadySet';
@@ -8,6 +8,7 @@ import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { FormSwitchControl } from '~/common/components/forms/FormSwitchControl';
import { InlineError } from '~/common/components/InlineError';
import { Link } from '~/common/components/Link';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
@@ -33,8 +34,9 @@ export function TogetherAIServiceSetup(props: { serviceId: DModelsServiceId }) {
} = useServiceSetup(props.serviceId, ModelVendorTogetherAI);
// derived state
const { oaiKey: togetherKey } = serviceAccess;
const { clientSideFetch, oaiKey: togetherKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// validate if url is a well formed proper url with zod
const shallFetchSucceed = !needsUserKey || (!!togetherKey && serviceSetupValid);
@@ -60,17 +62,24 @@ export function TogetherAIServiceSetup(props: { serviceId: DModelsServiceId }) {
placeholder='...'
/>
{advanced.on && <FormSwitchControl
{showAdvanced && <FormSwitchControl
title='Rate Limiter' on='Enabled' off='Disabled'
description={partialSettings?.togetherFreeTrial ? 'Free trial: 2 requests/2s' : 'Disabled'}
checked={partialSettings?.togetherFreeTrial ?? false}
onChange={on => updateSettings({ togetherFreeTrial: on })}
/>}
{advanced.on && !!partialSettings?.togetherFreeTrial && <Alert variant='soft'>
{showAdvanced && !!partialSettings?.togetherFreeTrial && <Alert variant='soft'>
Note: Please refresh the models list if you toggle the rate limiter.
</Alert>}
{showAdvanced && <SetupFormClientSideToggle
visible={!!togetherKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to Together AI API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
{isError && <InlineError error={error} />}
@@ -8,6 +8,7 @@ interface DTogetherAIServiceSettings {
togetherKey: string;
togetherHost: string;
togetherFreeTrial: boolean;
csf?: boolean;
}
export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, OpenAIAccessSchema> = {
@@ -19,6 +20,9 @@ export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, Ope
instanceLimit: 1,
hasServerConfigKey: 'hasLlmTogetherAI',
/// client-side-fetch ///
csfAvailable: _csfTogetherAIAvailable,
// functions
initializeSetup: () => ({
togetherKey: '',
@@ -30,6 +34,7 @@ export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, Ope
},
getTransportAccess: (partialSetup) => ({
dialect: 'togetherai',
clientSideFetch: _csfTogetherAIAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.togetherKey || '',
oaiOrg: '',
oaiHost: partialSetup?.togetherHost || '',
@@ -62,3 +67,7 @@ export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, Ope
// rate limit timestamp
let nextGenerationTs = 0;
function _csfTogetherAIAvailable(s?: Partial<DTogetherAIServiceSettings>) {
return !!s?.togetherKey;
}
+15 -2
View File
@@ -5,7 +5,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
import { ExternalLink } from '~/common/components/ExternalLink';
import { FormInputKey } from '~/common/components/forms/FormInputKey';
import { InlineError } from '~/common/components/InlineError';
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
import { ApproximateCosts } from '../ApproximateCosts';
import { useLlmUpdateModels } from '../../llm.client.hooks';
@@ -20,13 +22,17 @@ const EXTERNAL_LINK_XAI_API_KEYS = 'https://console.x.ai/';
export function XAIServiceSetup(props: { serviceId: DModelsServiceId }) {
// state
const advanced = useToggleableBoolean();
// external state
const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, serviceSetupValid, updateSettings } =
useServiceSetup(props.serviceId, ModelVendorXAI);
// derived state
const { oaiKey: xaiKey } = serviceAccess;
const { clientSideFetch, oaiKey: xaiKey } = serviceAccess;
const needsUserKey = !serviceHasCloudTenantConfig;
const showAdvanced = advanced.on || !!clientSideFetch;
// key validation
const shallFetchSucceed = !needsUserKey || (!!xaiKey && serviceSetupValid);
@@ -62,7 +68,14 @@ export function XAIServiceSetup(props: { serviceId: DModelsServiceId }) {
{/* onChange={(text) => updateSettings({ xaiHost: text })}*/}
{/*/>*/}
<SetupFormRefetchButton refetch={refetch} disabled={isFetching} error={isError} loading={isFetching} />
{showAdvanced && <SetupFormClientSideToggle
visible={!!xaiKey}
checked={!!clientSideFetch}
onChange={on => updateSettings({ csf: on })}
helpText='Connect directly to xAI API from your browser instead of through the server.'
/>}
<SetupFormRefetchButton refetch={refetch} disabled={isFetching} error={isError} loading={isFetching} advanced={advanced} />
{isError && <InlineError error={error} />}
+9
View File
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
export interface DXAIServiceSettings {
xaiKey: string;
csf?: boolean;
}
export const ModelVendorXAI: IModelVendor<DXAIServiceSettings, OpenAIAccessSchema> = {
@@ -17,11 +18,15 @@ export const ModelVendorXAI: IModelVendor<DXAIServiceSettings, OpenAIAccessSchem
instanceLimit: 1,
hasServerConfigKey: 'hasLlmXAI',
/// client-side-fetch ///
csfAvailable: _csfXAIAvailable,
// functions
initializeSetup: () => ({ xaiKey: '' }),
validateSetup: setup => setup.xaiKey?.length >= 80, // we assume all API keys are 80 chars+ - we won't have a strict validation
getTransportAccess: (partialSetup) => ({
dialect: 'xai',
clientSideFetch: _csfXAIAvailable(partialSetup) && !!partialSetup?.csf,
oaiKey: partialSetup?.xaiKey || '',
oaiOrg: '',
oaiHost: '',
@@ -33,3 +38,7 @@ export const ModelVendorXAI: IModelVendor<DXAIServiceSettings, OpenAIAccessSchem
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
};
function _csfXAIAvailable(s?: Partial<DXAIServiceSettings>) {
return !!s?.xaiKey;
}
+38
View File
@@ -0,0 +1,38 @@
import type { ModelVendorId } from '~/modules/llms/vendors/vendors.registry';
import type { DSpeexCredentials, DSpeexVoice, DSpeexVendorType } from './speex.types';
/**
* Descriptions for each Speex TTS Engine Vendor
* - used for DSpeexEngine instances creation, mainly
*
* Configuration including credentials and default voices are in DSpeexEngine instances
* in the speex store.
*/
export interface ISpeexVendor<TVt extends DSpeexVendorType> {
readonly vendorType: TVt;
readonly name: string;
readonly protocol: 'rpc' | 'webspeech';
readonly location: 'browser' | 'local' | 'cloud';
readonly priority: number; // display priority (lower = higher priority): elevenlabs=10, localai=20, openai=30, webspeech=100
// auto-detection info
readonly autoFromLlmVendorIds?: ModelVendorId[];
// capabilities
readonly capabilities: {
streaming: boolean;
voiceListing: boolean; // can list voices via API (vs hardcoded)
speedControl: boolean;
pitchControl: boolean;
};
// defaults for creating new engines
getDefaultCredentials(): DSpeexCredentials<TVt>;
getDefaultVoice(): DSpeexVoice<TVt>;
}
export type ISpeexVendorAny = { [TVt in DSpeexVendorType]: ISpeexVendor<TVt> }[DSpeexVendorType];
@@ -0,0 +1,426 @@
import * as React from 'react';
import { useQuery } from '@tanstack/react-query';
import { Box, Button, Divider, FormControl, Typography } from '@mui/joy';
import PlayArrowRoundedIcon from '@mui/icons-material/PlayArrowRounded';
import StopRoundedIcon from '@mui/icons-material/StopRounded';
import { FormChipControl } from '~/common/components/forms/FormChipControl';
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
import { FormSecretField } from '~/common/components/forms/FormSecretField';
import { FormSliderControl } from '~/common/components/forms/FormSliderControl';
import { FormTextField } from '~/common/components/forms/FormTextField';
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
import type { DCredentialsApiKey, DSpeexEngine, DSpeexEngineAny, DSpeexVendorType, DVoiceElevenLabs, DVoiceLocalAI, DVoiceOpenAI, DVoiceWebSpeech } from '../speex.types';
import { SPEEX_DEFAULTS, SPEEX_PREVIEW_STREAM, SPEEX_PREVIEW_TEXT } from '../speex.config';
import { SpeexVoiceAutocomplete } from './SpeexVoiceAutocomplete';
import { SpeexVoiceSelect } from './SpeexVoiceSelect';
import { speakText } from '../speex.client';
import { speexVendorTypeLabel } from './SpeexEngineSelect';
function CredentialsApiKeyInputs({ credentials, onUpdate, vendorType, showHost, hostRequired, hostPlaceholder }: {
credentials: DCredentialsApiKey;
onUpdate: (credentials: DCredentialsApiKey) => void;
vendorType: DSpeexVendorType;
showHost?: boolean;
hostRequired?: boolean;
hostPlaceholder?: string;
}) {
return <>
<FormSecretField
autoCompleteId={`speex-${vendorType}-key`}
title='API Key'
description={hostRequired ? 'Optional' : speexVendorTypeLabel(vendorType)}
value={credentials.apiKey}
onChange={value => onUpdate({ ...credentials, apiKey: value })}
required={!hostRequired}
startDecorator={credentials.apiKey ? false : undefined}
// placeholder='Required'
inputSx={{ maxWidth: 220 }}
/>
{showHost && (
<FormTextField
autoCompleteId={`speex-${vendorType}-host`}
title='API Host'
description={hostRequired ? 'Required' : 'Optional'}
value={credentials.apiHost ?? ''}
onChange={text => onUpdate({ ...credentials, apiHost: text || undefined })}
placeholder={hostPlaceholder ?? 'https://api.example.com'}
inputSx={{ maxWidth: 220 }}
/>
)}
{showHost && <Divider inset='context' />}
</>;
}
function PreviewButton({ engineId }: { engineId: DSpeexEngineAny['engineId'] }) {
// async + cache
const { isFetching, isError, error, refetch: previewVoice } = useQuery({
enabled: false, // manual trigger only
queryKey: ['speex-preview', engineId],
queryFn: async () => {
const result = await speakText(
SPEEX_PREVIEW_TEXT,
{ engineId: engineId },
{ streaming: SPEEX_PREVIEW_STREAM },
);
if (!result.success) throw new Error(result.error || 'Preview failed');
return result;
},
});
return (
<TooltipOutlined color='danger' title={error?.message ? <pre>{error.message}</pre> : false}>
<Button
variant='outlined'
color={isError ? 'danger' : 'neutral'}
size='sm'
onClick={() => previewVoice()}
disabled={isFetching}
startDecorator={isFetching ? <StopRoundedIcon /> : <PlayArrowRoundedIcon />}
sx={{ ml: 'auto', minWidth: 130 }}
>
{isFetching ? 'Speaking...' : isError ? 'Retry' : 'Preview'}
</Button>
</TooltipOutlined>
);
}
export function SpeexConfigureEngineFull(props: {
engine: DSpeexEngineAny;
isMobile: boolean;
mode?: 'full' | 'voice-only';
bottomStart?: React.ReactNode;
onUpdate: (updates: Partial<DSpeexEngineAny>) => void;
}) {
const { engine, isMobile, mode = 'full', bottomStart, onUpdate } = props;
return <>
{/*<Box mt={2} />*/}
{/*<Divider sx={{ my: 1 }} inset='context' />*/}
<Divider sx={{ my: 1 }} inset='context'>{isMobile ? 'Configuration' : 'App Voice Configuration'}</Divider>
{/* Engine-Specific pane */}
{engine.vendorType === 'elevenlabs' ? (
<ElevenLabsConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
) : engine.vendorType === 'localai' ? (
<LocalAIConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
) : engine.vendorType === 'openai' ? (
<OpenAIConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
) : engine.vendorType === 'webspeech' ? (
<WebSpeechConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
) : (
<Typography level='body-sm' color='warning'>Unknown engine type {(engine as any)?.vendorType}</Typography>
)}
{/* (Delete | Chip) -- Preview */}
<Box sx={{ display: 'flex', alignItems: 'center' }}>
{bottomStart}
<PreviewButton engineId={engine.engineId} />
</Box>
</>;
}
// Vendor-specific configs
function ElevenLabsConfig({ engine, onUpdate, mode, isMobile }: {
engine: DSpeexEngine<'elevenlabs'>,
onUpdate: (updates: Partial<DSpeexEngine<'elevenlabs'>>) => void;
isMobile: boolean;
mode: 'full' | 'voice-only';
}) {
const { credentials, voice } = engine;
const showCredentials = mode === 'full' && !engine.isAutoLinked && credentials.type === 'api-key';
const handleCredentialsUpdate = React.useCallback((newCredentials: DCredentialsApiKey) => {
onUpdate({ credentials: newCredentials });
}, [onUpdate]);
const handleVoiceChange = React.useCallback((ttsVoiceId: DVoiceElevenLabs['ttsVoiceId']) => {
const { ttsVoiceId: _, ...restVoice } = voice;
onUpdate({
voice: {
...restVoice,
...(ttsVoiceId && { ttsVoiceId }),
},
});
}, [onUpdate, voice]);
return <>
{/* Credentials (only for manually added engines in full mode) */}
{showCredentials && (
<CredentialsApiKeyInputs
credentials={credentials}
onUpdate={handleCredentialsUpdate}
vendorType='elevenlabs'
/>
)}
<FormChipControl<Exclude<DVoiceElevenLabs['ttsModel'], undefined>>
title='Model'
alignEnd
options={[
{ value: 'eleven_multilingual_v2', label: 'Multilingual v2', description: 'Default' },
{ value: 'eleven_turbo_v2_5', label: 'Turbo v2.5', description: 'Fast' },
{ value: 'eleven_flash_v2_5', label: 'Flash v2.5', description: 'Fastest' },
{ value: 'eleven_v3', label: 'v3', description: 'Newest' },
]}
value={voice.ttsModel ?? SPEEX_DEFAULTS.ELEVENLABS_MODEL}
onChange={(value) => onUpdate({ voice: { ...voice, ttsModel: value } })}
/>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
<FormLabelStart title='Voice' description={isMobile ? undefined : 'ElevenLabs voice'} />
<SpeexVoiceSelect
autoPreview
engine={engine}
voiceId={voice.ttsVoiceId ?? null}
onVoiceChange={handleVoiceChange}
/>
</FormControl>
{/*{showCredentials && (*/}
{/* <FormHelperText>*/}
{/* Voice listing requires API key. Language auto-detected from preferences.*/}
{/* </FormHelperText>*/}
{/*)}*/}
</>;
}
function LocalAIConfig({ engine, onUpdate, mode, isMobile }: {
engine: DSpeexEngine<'localai'>,
onUpdate: (updates: Partial<DSpeexEngine<'localai'>>) => void;
isMobile: boolean;
mode: 'full' | 'voice-only';
}) {
const { credentials, voice } = engine;
const showCredentials = mode === 'full' && !engine.isAutoLinked && credentials.type === 'api-key';
const handleCredentialsUpdate = React.useCallback((newCredentials: DCredentialsApiKey) => {
onUpdate({ credentials: newCredentials });
}, [onUpdate]);
const handleModelChange = React.useCallback((ttsModel: DVoiceLocalAI['ttsModel']) => {
const { ttsModel: _, ...restVoice } = voice;
onUpdate({
voice: {
...restVoice,
...(ttsModel && { ttsModel }),
},
});
}, [onUpdate, voice]);
return <>
{/* Credentials (only for manually added engines in full mode) */}
{showCredentials && (
<CredentialsApiKeyInputs
credentials={credentials}
onUpdate={handleCredentialsUpdate}
vendorType='localai'
showHost
hostRequired
hostPlaceholder='http://localhost:8080'
/>
)}
{/* Model: autocomplete with suggestions + free-form input */}
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
<FormLabelStart title='Model' description={isMobile ? undefined : 'Select or type'} />
<SpeexVoiceAutocomplete
engine={engine}
value={voice.ttsModel}
onValueChange={handleModelChange}
placeholder='e.g., kokoro'
/>
</FormControl>
<FormTextField
autoCompleteId='speex-localai-backend'
title='Backend'
description='Optional'
placeholder='e.g., coqui, bark, piper'
value={voice.ttsBackend ?? ''}
onChange={(text) => onUpdate({ voice: { ...voice, ttsBackend: text || undefined } })}
inputSx={{ maxWidth: 220 }}
/>
</>;
}
function OpenAIConfig({ engine, onUpdate, isMobile, mode }: {
engine: DSpeexEngine<'openai'>,
onUpdate: (updates: Partial<DSpeexEngineAny>) => void;
isMobile: boolean;
mode: 'full' | 'voice-only';
}) {
const { credentials, voice } = engine;
const showCredentials = mode === 'full' && !engine.isAutoLinked && credentials.type === 'api-key';
const handleCredentialsUpdate = React.useCallback((newCredentials: DCredentialsApiKey) => {
onUpdate({ credentials: newCredentials });
}, [onUpdate]);
const handleVoiceChange = React.useCallback((ttsVoiceId: DVoiceOpenAI['ttsVoiceId']) => {
const { ttsVoiceId: _, ...restVoice } = voice;
onUpdate({
voice: {
...restVoice,
...(ttsVoiceId && { ttsVoiceId }),
},
});
}, [onUpdate, voice]);
const handleSpeedChange = React.useCallback((value: number) => {
onUpdate({ voice: { ...voice, ttsSpeed: value } });
}, [onUpdate, voice]);
return <>
{/* Credentials (only for manually added engines in full mode) */}
{showCredentials && (
<CredentialsApiKeyInputs
credentials={credentials}
onUpdate={handleCredentialsUpdate}
vendorType='openai'
showHost
hostPlaceholder='https://api.openai.com (optional)'
/>
)}
<FormChipControl<DVoiceOpenAI['ttsModel']>
title='Model'
alignEnd
options={[
{ value: 'gpt-4o-mini-tts', label: 'GPT-4o Mini', description: 'Expressive' },
{ value: 'tts-1', label: 'TTS-1', description: 'Fast' },
{ value: 'tts-1-hd', label: 'TTS-1-HD', description: 'Quality' },
]}
value={voice.ttsModel ?? SPEEX_DEFAULTS.OPENAI_MODEL}
onChange={value => onUpdate({
voice: {
...voice,
ttsModel: value,
},
})}
/>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
<FormLabelStart title='Voice' description={isMobile ? undefined : 'OpenAI TTS voice'} />
<SpeexVoiceSelect
engine={engine}
voiceId={voice.ttsVoiceId ?? null}
onVoiceChange={handleVoiceChange}
/>
</FormControl>
<FormSliderControl
title='Speed'
description={`${voice.ttsSpeed ?? 1}x`}
min={0.5}
max={2}
step={0.25}
value={voice.ttsSpeed ?? 1}
onChange={handleSpeedChange}
valueLabelDisplay={voice.ttsSpeed && voice.ttsSpeed !== 1 ? 'on' : 'auto'}
sliderSx={{ maxWidth: 220, my: -0.5 }}
/>
{voice.ttsModel === 'gpt-4o-mini-tts' && (
<FormTextField
autoCompleteId='speex-openai-instruction'
title='Instruction'
description={isMobile ? undefined : '4o Mini only'}
placeholder='e.g., Speak with joy'
value={voice.ttsInstruction ?? ''}
onChange={(text) => onUpdate({ voice: { ...voice, ttsInstruction: text } })}
inputSx={{ flexGrow: 1, maxWidth: 220 }}
/>
)}
</>;
}
function WebSpeechConfig({ engine, onUpdate, isMobile }: {
engine: DSpeexEngine<'webspeech'>
onUpdate: (updates: Partial<DSpeexEngine<'webspeech'>>) => void;
isMobile: boolean;
mode: 'full' | 'voice-only';
}) {
const { voice } = engine;
const handleVoiceChange = React.useCallback((ttsVoiceURI: DVoiceWebSpeech['ttsVoiceURI']) => {
const { ttsVoiceURI: _, ...restVoice } = voice;
onUpdate({
voice: {
...restVoice,
...(ttsVoiceURI && { ttsVoiceURI }),
},
});
}, [onUpdate, voice]);
const handleSpeedChange = React.useCallback((value: number) => {
onUpdate({ voice: { ...voice, ttsSpeed: value } });
}, [onUpdate, voice]);
const handlePitchChange = React.useCallback((value: number) => {
onUpdate({ voice: { ...voice, ttsPitch: value } });
}, [onUpdate, voice]);
return <>
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
<FormLabelStart title='Voice' description={isMobile ? undefined : 'System voice'} />
<SpeexVoiceSelect
engine={engine}
voiceId={voice.ttsVoiceURI ?? null}
onVoiceChange={handleVoiceChange}
/>
</FormControl>
<FormSliderControl
title='Speed'
description={`${(voice.ttsSpeed ?? 1).toFixed(1)}x`}
min={0.5}
max={2}
step={0.1}
value={voice.ttsSpeed ?? 1}
onChange={handleSpeedChange}
valueLabelDisplay={voice.ttsSpeed && voice.ttsSpeed !== 1 ? 'on' : 'auto'}
sliderSx={{ maxWidth: 220, my: -0.5 }}
/>
<FormSliderControl
title='Pitch'
description={`${(voice.ttsPitch ?? 1).toFixed(1)}`}
min={0.5}
max={2}
step={0.1}
value={voice.ttsPitch ?? 1}
onChange={handlePitchChange}
valueLabelDisplay={voice.ttsPitch && voice.ttsPitch !== 1 ? 'on' : 'auto'}
sliderSx={{ maxWidth: 220, my: -0.5 }}
/>
</>;
}
@@ -0,0 +1,282 @@
/**
* SpeexEngineSettings - TTS engine selection and configuration
*
* Provides:
* - Chip-based engine selection with visual status
* - Add Service dropdown menu
* - Per-engine voice configuration in a Card
*/
import * as React from 'react';
import { Box, Button, Chip, Dropdown, ListItemDecorator, Menu, MenuButton, MenuItem, SvgIconProps, Typography } from '@mui/joy';
import AddIcon from '@mui/icons-material/Add';
import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline';
import LinkIcon from '@mui/icons-material/Link';
import { ConfirmationModal } from '~/common/components/modals/ConfirmationModal';
import { ElevenLabsIcon } from '~/common/components/icons/vendors/ElevenLabsIcon';
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
import { LocalAIIcon } from '~/common/components/icons/vendors/LocalAIIcon';
import { OpenAIIcon } from '~/common/components/icons/vendors/OpenAIIcon';
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
import { themeZIndexOverMobileDrawer } from '~/common/app.theme';
import type { DSpeexEngineAny, DSpeexVendorType } from '../speex.types';
import { SpeexConfigureEngineFull } from './SpeexConfigureEngineFull';
import { speexAreCredentialsValid, useSpeexEngines, useSpeexGlobalEngine, useSpeexStore } from '../store-module-speex';
const _styles = {
menu: {
zIndex: themeZIndexOverMobileDrawer,
minWidth: 220,
'--List-padding': '0.75rem',
borderRadius: 'xl',
boxShadow: 'md',
},
menuButton: {
ml: 'auto',
// borderRadius: '1.5rem',
// borderColor: 'neutral.outlinedBorder', // like ModeServiceSelector's Add button
// minWidth: 150,
textWrap: 'nowrap',
// '&[aria-expanded="true"]': {
// borderBottomRightRadius: 0,
// borderBottomLeftRadius: 0,
// // color: 'neutral.softColor',
// // backgroundColor: 'neutral.softHoverBg',
// },
},
menuItem: {
py: 1,
px: 1,
borderRadius: 'md',
minHeight: 56,
},
menuItemContent: {
display: 'flex',
flexDirection: 'column',
gap: 0.125,
},
menuItemName: {
fontWeight: 600,
},
menuItemDescription: {
fontWeight: 400,
},
chipRow: {
display: 'flex',
flexWrap: 'wrap',
gap: 1,
},
chip: {
px: 1.5,
minHeight: '2rem',
// borderRadius: 'md',
// boxShadow: 'sm',
},
chipUnconfigured: {
px: 1.5,
minHeight: '2rem',
// borderRadius: 'md',
// color: 'text.tertiary',
opacity: 0.6,
},
chipSymbol: {
ml: -0.75,
mr: 0.5,
width: 20,
height: 20,
borderRadius: '50%',
backgroundColor: 'background.surface',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
},
} as const;
const ADDABLE_VENDORS: { vendorType: DSpeexVendorType; label: string; description: string, icon?: React.FunctionComponent<SvgIconProps> }[] = [
{ vendorType: 'elevenlabs', label: 'ElevenLabs', description: 'Premium voices', icon: ElevenLabsIcon },
{ vendorType: 'localai', label: 'LocalAI', description: 'Self-hosted TTS', icon: LocalAIIcon },
{ vendorType: 'openai', label: 'OpenAI TTS', description: 'Reliable', icon: OpenAIIcon },
] as const;
export function SpeexConfigureEngines(_props: { isMobile: boolean }) {
// state
const [confirmDeleteEngine, setConfirmDeleteEngine] = React.useState<DSpeexEngineAny | null>(null);
// external state - module
const engines = useSpeexEngines();
const activeEngine = useSpeexGlobalEngine(); // auto-select the highest priority, if the user choice (active engine) is missing
const activeEngineId = activeEngine?.engineId ?? null;
const activeEngineValid = !activeEngine ? false : speexAreCredentialsValid(activeEngine.credentials);
// derived state
const hasEngines = engines.length > 0;
const canDeleteActiveEngine = activeEngine && !activeEngine.isAutoDetected && !activeEngine.isAutoLinked;
// handlers
const handleEngineSelect = React.useCallback((engineId: string | null) => {
useSpeexStore.getState().setActiveEngineId(engineId);
}, []);
const handleEngineUpdate = React.useCallback((updates: Partial<DSpeexEngineAny>) => {
if (activeEngineId)
useSpeexStore.getState().updateEngine(activeEngineId, updates);
}, [activeEngineId]);
const handleAddEngine = React.useCallback((vendorType: DSpeexVendorType) => {
const newEngineId = useSpeexStore.getState().createEngine(vendorType);
useSpeexStore.getState().setActiveEngineId(newEngineId);
}, []);
const handleDeleteClick = React.useCallback((event: React.MouseEvent) => {
if (!activeEngine || !canDeleteActiveEngine) return;
// shift+click skips confirmation
if (event.shiftKey)
return useSpeexStore.getState().deleteEngine(activeEngine.engineId);
setConfirmDeleteEngine(activeEngine);
}, [activeEngine, canDeleteActiveEngine]);
const handleConfirmDelete = React.useCallback(() => {
if (!confirmDeleteEngine) return;
useSpeexStore.getState().deleteEngine(confirmDeleteEngine.engineId);
setConfirmDeleteEngine(null);
}, [confirmDeleteEngine]);
const handleCancelDelete = React.useCallback(() => {
setConfirmDeleteEngine(null);
}, []);
return <>
{/* "Voice Engine" + Add Service dropdown */}
<Box sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<FormLabelStart
// title='Voice Engine'
title='Active Engine'
description={activeEngine ? activeEngine.label : 'Select a voice provider'}
/>
{/* -> Add Service */}
<Dropdown>
<MenuButton size='sm' variant={!activeEngine ? 'solid' : 'outlined'} startDecorator={<AddIcon />} sx={_styles.menuButton}>
Add
{/* Add Service*/}
</MenuButton>
{/*<MenuButton size='sm' color='primary' variant={!activeEngine ? 'solid' : 'outlined'} startDecorator={<AddIcon />} sx={_styles.menuButton}>*/}
{/* Add*/}
{/* /!* Add Service*!/*/}
{/*</MenuButton>*/}
<Menu placement='bottom' popperOptions={{ modifiers: [{ name: 'offset', options: { offset: [-12, -2] } }] }} sx={_styles.menu}>
{ADDABLE_VENDORS.map(vendor => (
<MenuItem key={vendor.vendorType} onClick={() => handleAddEngine(vendor.vendorType)} sx={_styles.menuItem}>
<ListItemDecorator>
{vendor.icon ? <vendor.icon /> : null}
</ListItemDecorator>
<Box sx={_styles.menuItemContent}>
<Typography level='title-md' sx={_styles.menuItemName}>{vendor.label}</Typography>
<Typography level='body-sm' sx={_styles.menuItemDescription}>{vendor.description}</Typography>
</Box>
</MenuItem>
))}
</Menu>
</Dropdown>
</Box>
{/* Engine Chips row */}
{hasEngines && (
<Box sx={_styles.chipRow}>
{engines.map(engine => {
const isActive = engine.engineId === activeEngineId;
const isConfigured = speexAreCredentialsValid(engine.credentials);
return (
<TooltipOutlined key={engine.engineId} title={isActive ? 'Global application voice' : isConfigured ? 'Click to activate' : 'Needs configuration'}>
<Chip
variant={isActive ? 'solid' : 'outlined'}
color={!isActive ? 'neutral' : !isConfigured ? 'danger' : 'neutral'}
// startDecorator={isActive && <Box sx={_styles.chipSymbol}>
// <CheckRoundedIcon sx={{ fontSize: 16, color: 'text.primary' }} />
// </Box>}
endDecorator={engine.isAutoLinked && <LinkIcon sx={{ fontSize: 16 }} />}
onClick={event => handleEngineSelect(event.shiftKey ? null : engine.engineId)}
sx={isConfigured ? _styles.chip : _styles.chipUnconfigured}
>
{engine.label}
</Chip>
</TooltipOutlined>
);
})}
</Box>
)}
{/* Active engine (specific) full configuration */}
{activeEngine && (
<SpeexConfigureEngineFull
engine={activeEngine}
isMobile={_props.isMobile}
mode={activeEngine.isAutoLinked || activeEngine.isAutoDetected ? 'voice-only' : 'full'}
bottomStart={
!canDeleteActiveEngine ? (
<Chip size='sm' color={!activeEngineValid ? 'danger' : undefined} variant='soft' sx={{ px: 1.5, py: 0.5 }}>
{!activeEngineValid ? (activeEngine.isAutoLinked ? 'Linked to AI Service' : 'Invalid Configuration')
: activeEngine.isAutoLinked ? 'Linked to AI Service'
: activeEngine.isAutoDetected ? 'System'
: 'Configured Manually'}
</Chip>
) : (
// <GoodTooltip title='Delete this service'>
<Button
size='sm'
color='neutral'
variant='plain'
onClick={handleDeleteClick}
startDecorator={<DeleteOutlineIcon />}
// sx={{ minWidth: 120 }}
>
Delete
</Button>
// </GoodTooltip>
)}
onUpdate={handleEngineUpdate}
/>
)}
{/* Empty state */}
{!hasEngines && (
<Typography level='body-sm' sx={{ color: 'text.tertiary' }}>
No voice engines available. Configure an OpenAI-compatible AI service to auto-link TTS, or add ElevenLabs for premium voices.
</Typography>
)}
{/* Delete Confirmation Modal */}
{!!confirmDeleteEngine && (
<ConfirmationModal
open
onClose={handleCancelDelete}
onPositive={handleConfirmDelete}
lowStakes
noTitleBar
confirmationText={<>Remove <strong>{confirmDeleteEngine.label}</strong>? This cannot be undone.</>}
positiveActionText='Remove'
/>
)}
</>;
}
@@ -0,0 +1,72 @@
/**
* SpeexEngineSelect - Reusable engine selection dropdown
*/
import * as React from 'react';
import { Option, Select, Typography } from '@mui/joy';
import type { DSpeexVendorType, SpeexEngineId } from '../speex.types';
import { speexAreCredentialsValid, useSpeexEngines } from '../store-module-speex';
export function speexVendorTypeLabel(vendorType: DSpeexVendorType): string {
switch (vendorType) {
case 'elevenlabs':
return 'ElevenLabs';
case 'openai':
return 'OpenAI';
case 'localai':
return 'LocalAI';
case 'webspeech':
return 'System';
}
}
interface SpeexEngineSelectProps {
/** Selected engine ID (null = none selected) */
engineId: string | null;
/** Called when selection changes */
onEngineChange: (engineId: string | null) => void;
/** Disable the select */
disabled?: boolean;
/** Placeholder text (default: 'Select engine...') */
placeholder?: string;
}
export function SpeexEngineSelect(props: SpeexEngineSelectProps) {
const { engineId, onEngineChange, disabled, placeholder = 'Select engine...' } = props;
const engines = useSpeexEngines();
const validEngines = React.useMemo(
() => engines.filter(e => speexAreCredentialsValid(e.credentials)),
[engines],
);
const handleChange = React.useCallback((_event: unknown, value: SpeexEngineId | null) => {
onEngineChange(value);
}, [onEngineChange]);
return (
<Select
value={engineId}
disabled={disabled || !validEngines.length}
placeholder={placeholder}
onChange={handleChange}
sx={{ minWidth: 200 }}
>
{validEngines.map(({ engineId, label, vendorType }) => (
<Option key={engineId} value={engineId} label={label}>
{label}
{!label.toLowerCase().includes(vendorType) && (
<Typography level='body-xs' sx={{ ml: 1, color: 'text.tertiary' }}>
({speexVendorTypeLabel(vendorType)})
</Typography>
)}
</Option>
))}
</Select>
);
}
@@ -0,0 +1,131 @@
/**
* SpeexVoiceAutocomplete - Combined voice/model selector + free-form input
*
* Uses MUI Joy Autocomplete with freeSolo to allow:
* - Selecting from fetched voice/model list (suggestions)
* - Typing custom value (free-form)
*
* Used by LocalAI where models can be selected from list or typed manually.
*/
import * as React from 'react';
import { Autocomplete, AutocompleteOption, Box, CircularProgress, IconButton, Typography } from '@mui/joy';
import RefreshRoundedIcon from '@mui/icons-material/RefreshRounded';
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
import type { DSpeexEngineAny, SpeexListVoiceOption } from '../speex.types';
import { useSpeexVoices } from './useSpeexVoices';
interface SpeexVoiceAutocompleteProps {
engine: DSpeexEngineAny;
/** Current value (can be from list or custom) */
value: string | undefined;
/** Called when value changes (selection or typed). undefined = cleared */
onValueChange: (value: string | undefined) => void;
/** Placeholder text */
placeholder?: string;
disabled?: boolean;
}
export function SpeexVoiceAutocomplete(props: SpeexVoiceAutocompleteProps) {
const { engine, value /* e.g. ttsModel */, onValueChange, placeholder = 'Select or type...', disabled } = props;
// fetch voices/models
const { voices, isLoading, error, refetch } = useSpeexVoices(engine);
// local input state for freeSolo
const [inputValue, setInputValue] = React.useState(value ?? '');
// sync input when value prop changes externally
React.useEffect(() => {
setInputValue(value ?? '');
}, [value]);
// handlers
const handleChange = React.useCallback((_event: unknown, newValue: string | SpeexListVoiceOption | null) => {
// newValue can be: string (typed), SpeexListVoiceOption (selected), or null (cleared)
if (newValue === null)
onValueChange(undefined);
else if (typeof newValue === 'string')
onValueChange(newValue || undefined);
else
onValueChange(newValue.id || undefined);
}, [onValueChange]);
const handleInputChange = React.useCallback((_event: unknown, newInputValue: string, reason: string) => {
// BUGFIX: when re-clicking on the same option on the popup, reason will be 'reset', but the inputValue
// will be the label of the selected option and not the value. This fixes it
if (reason !== 'input')
return;
setInputValue(newInputValue);
// For freeSolo, also update value on input change (typing)
onValueChange(newInputValue || undefined);
}, [onValueChange]);
return (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
{/* Refresh button (only if refetch available) */}
{refetch && (
<TooltipOutlined color={error ? 'danger' : undefined} title={error ? <pre>{error}</pre> : 'Refresh'}>
<IconButton
color={error ? 'danger' : 'neutral'}
variant='plain'
disabled={isLoading}
onClick={() => refetch()}
>
{!isLoading ? <RefreshRoundedIcon /> : <CircularProgress size='sm' />}
</IconButton>
</TooltipOutlined>
)}
{/* Autocomplete */}
<Autocomplete<SpeexListVoiceOption, false, false, true>
freeSolo
openOnFocus
clearOnEscape
disabled={disabled}
placeholder={placeholder}
options={voices}
getOptionKey={(option) => typeof option === 'string' ? option : option.id}
getOptionLabel={(option) => typeof option === 'string' ? option : option.name}
isOptionEqualToValue={(option, val) => option.id === (typeof val === 'string' ? val : val.id)}
value={voices.find(o => o.id === value) ?? (value || null)}
onChange={handleChange}
inputValue={inputValue}
onInputChange={handleInputChange}
loading={isLoading}
renderOption={(optionProps, option) => {
const { key, ...rest } = optionProps as any;
return (
<AutocompleteOption key={key} {...rest} sx={{ display: 'block' }}>
<Typography level='title-sm'>{option.name}</Typography>
{option.description && (
<Typography level='body-xs' sx={{ opacity: 0.6 }}>{option.description}</Typography>
)}
</AutocompleteOption>
);
}}
slotProps={{
root: {
sx: { minWidth: 180, maxWidth: 220, flexGrow: 1 },
},
listbox: {
sx: { maxWidth: 'min(400px, calc(100dvw - 1rem))' },
},
}}
/>
</Box>
);
}
@@ -0,0 +1,121 @@
import * as React from 'react';
import { Box, CircularProgress, IconButton, Option, optionClasses, Select, SelectSlotsAndSlotProps } from '@mui/joy';
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
import RefreshRoundedIcon from '@mui/icons-material/RefreshRounded';
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
import type { DSpeexEngineAny } from '../speex.types';
import { useSpeexVoices } from './useSpeexVoices';
// copied from useLLMSelect.tsx - inspired by optimaSelectSlotProps.listbox
const _selectSlotProps: SelectSlotsAndSlotProps<false>['slotProps'] = {
root: {
sx: {
minWidth: 220, // 180 = 220 - 36 - 4
},
},
button: {
className: 'agi-ellipsize',
sx: {
// these + the ellipsize class will ellipsize the text in the button
display: 'inline-block',
textAlign: 'start',
// maxWidth: 220,
} as const,
},
listbox: {
// size: 'md',
// className: 'agi-ellipsize',
sx: {
boxShadow: 'xl',
// Option: clip width to 200...360px
[`& .${optionClasses.root}`]: {
// minWidth: 300,
maxWidth: 'min(640px, calc(100dvw - 0.25rem))', // the small reduction is to avoid accidental h-scrolling because of the border
},
},
} as const,
} as const;
export function SpeexVoiceSelect(props: {
engine: DSpeexEngineAny;
voiceId: string | null;
onVoiceChange: (voiceId: string) => void;
disabled?: boolean;
autoPreview?: boolean;
}) {
// props
const { engine, voiceId, onVoiceChange, disabled, autoPreview } = props;
// external state - module
const { voices, isLoading, error, refetch } = useSpeexVoices(engine);
// track user-initiated voice changes for preview (not initial load or voice list changes)
const [userSelectedVoiceId, setUserSelectedVoiceId] = React.useState<string | null>(null);
// [effect] auto-preview: play voice sample only when user explicitly selects a voice
const selectedVoice = userSelectedVoiceId ? voices.find(v => v.id === userSelectedVoiceId) : null;
const previewUrl = (autoPreview && selectedVoice?.previewUrl) || null;
React.useEffect(() => {
if (previewUrl)
void AudioPlayer.playUrl(previewUrl);
}, [previewUrl]);
// handlers
const handleVoiceChange = React.useCallback((_event: unknown, value: string | null) => {
setUserSelectedVoiceId(value);
value && onVoiceChange(value);
}, [onVoiceChange]);
return <Box sx={{ display: 'flex', alignItems: 'center' }}>
{refetch && (
<TooltipOutlined color={error ? 'danger' : undefined} title={error ? <pre>{error}</pre> : 'Refresh voices'}>
<IconButton
color={error ? 'danger' : 'neutral'}
variant='plain'
disabled={isLoading}
onClick={() => refetch()}
>
{!isLoading ? <RefreshRoundedIcon /> : <CircularProgress size='sm' />}
</IconButton>
</TooltipOutlined>
)}
<Select
variant='outlined'
disabled={disabled || isLoading || voices.length === 0}
value={voiceId ?? null}
onChange={handleVoiceChange}
placeholder={
error ? 'Error loading voices'
: isLoading ? 'Loading...'
: voices.length === 0 ? 'No voices available'
: voiceId ? `Voice ${voiceId.slice(0, 12)}...`
: 'Select a voice'
}
// startDecorator={<PhVoice />}
// endDecorator={isLoading && <CircularProgress size='sm' />}
indicator={<KeyboardArrowDownIcon />}
slotProps={_selectSlotProps}
>
{voices.map(voice => (
<Option key={voice.id} value={voice.id} label={voice.name.split('-')[0]}>
<div className='agi-ellipsize'>
{voice.name} {voice.description && <span style={{ marginLeft: '0.75rem', opacity: 0.5, fontSize: 'smaller' }}>{voice.description}</span>}
</div>
</Option>
))}
</Select>
</Box>;
}
@@ -0,0 +1,38 @@
import { useQuery } from '@tanstack/react-query';
import type { DSpeexEngineAny, SpeexListVoiceOption, SpeexListVoicesResult } from '../speex.types';
import { speexListVoices_RPC_orThrow } from '../protocols/rpc/rpc.client';
import { useSpeexWebSpeechVoices } from '../protocols/webspeech/webspeech.client';
const _stableEmptyVoices: SpeexListVoiceOption[] = [] as const;
// returns voices given an engine
export function useSpeexVoices(engine: DSpeexEngineAny): SpeexListVoicesResult {
// props
const { vendorType, engineId } = engine;
const isWebspeech = vendorType === 'webspeech';
// use browser voices
const browserVoicesResult = useSpeexWebSpeechVoices(isWebspeech);
// use RPC voices
const { data: cloudVoices, error: cloudError, isFetching: cloudIsFetching, refetch } = useQuery({
enabled: !isWebspeech,
queryKey: ['speex', 'listVoices', engineId, vendorType],
queryFn: () => speexListVoices_RPC_orThrow(engine as any /* will not run for 'webspeech' */),
staleTime: 5 * 60 * 1000, // 5 minutes - voices don't change often
});
// do not refetch openai, voices are hardcoded
const needsRefetch = vendorType !== 'openai';
// switch result
return isWebspeech ? browserVoicesResult : {
voices: cloudVoices?.length ? cloudVoices : _stableEmptyVoices,
isLoading: cloudIsFetching,
error: cloudError instanceof Error ? cloudError.message : null,
refetch: needsRefetch ? refetch : undefined,
};
}
@@ -0,0 +1,244 @@
/**
* Speex RPC Client
*
* Handles communication with speex.router for cloud TTS providers.
* Resolves credentials from engine configuration and calls the streaming API.
*/
import { apiAsync, apiStream } from '~/common/util/trpc.client';
import { convert_Base64_To_UInt8Array, convert_UInt8Array_To_Base64 } from '~/common/util/blobUtils';
import { findModelsServiceOrNull } from '~/common/stores/llms/store-llms';
import { stripUndefined } from '~/common/util/objectUtils';
import type { DLocalAIServiceSettings } from '~/modules/llms/vendors/localai/localai.vendor';
import type { DOpenAIServiceSettings } from '~/modules/llms/vendors/openai/openai.vendor';
import { AudioLivePlayer } from '~/common/util/audio/AudioLivePlayer';
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
import type { DSpeexEngine, SpeexListVoiceOption, SpeexSpeakResult } from '../../speex.types';
import type { SpeexWire_Access, SpeexWire_Voice } from './rpc.wiretypes';
import { SPEEX_DEBUG } from '../../speex.config';
type _DSpeexEngineRPC = DSpeexEngine<'elevenlabs'> | DSpeexEngine<'localai'> | DSpeexEngine<'openai'>;
/**
* Synthesize speech via speex.router (streaming)
*/
export async function speexSynthesize_RPC(
engine: _DSpeexEngineRPC,
text: string,
options: {
streaming: boolean;
languageCode?: string;
priority?: 'fast' | 'balanced' | 'quality';
playback: boolean;
returnAudio: boolean;
},
callbacks?: {
onStart?: () => void;
onChunk?: (chunk: ArrayBuffer) => void;
onComplete?: () => void;
onError?: (error: Error) => void;
},
): Promise<SpeexSpeakResult> {
// engine credentials (DCredentials..) -> wire Access
if (SPEEX_DEBUG) console.log(`[Speex RPC] Synthesize request (engine: ${engine.engineId}, ${text.length} chars) - options:`, options);
const access = stripUndefined(_buildRPCWireAccess(engine));
if (!access) {
const error = new Error(`Failed to resolve credentials for engine ${engine.engineId}`);
callbacks?.onError?.(error);
return { success: false, errorType: 'tts-unconfigured', error: error.message };
}
// engine voice -> wire Voice
// IMPORTANT: TS ensures structural compatibility here between the DVoice* and Voice*_schema types
const voice: SpeexWire_Voice = stripUndefined(engine.voice);
// audio player for streaming playback
let audioPlayer: AudioLivePlayer | null = null;
const audioChunks: ArrayBuffer[] = [];
const abortController = new AbortController();
try {
// call the streaming RPC - whether the backend will stream in chunks or as a whole
const particleStream = await apiStream.speex.synthesize.mutate({
access,
text,
voice,
streaming: options.streaming,
...(options.languageCode && { languageCode: options.languageCode }),
...(options.priority && { priority: options.priority }),
}, {
signal: abortController.signal,
});
// process streaming particles
for await (const particle of particleStream) {
if (SPEEX_DEBUG) console.log('[Speex RPC] <-', particle);
switch (particle.t) {
case 'start':
callbacks?.onStart?.();
if (options.playback && options.streaming)
audioPlayer = new AudioLivePlayer();
break;
case 'audio':
// Decode base64 to ArrayBuffer
const audioData = convert_Base64_To_UInt8Array(particle.base64, 'speex.rpc.client');
// Accumulate for return (copy bytes before playback may transfer/detach the buffer)
if (options.returnAudio)
audioChunks.push(audioData.slice().buffer);
// Playback: streaming uses AudioLivePlayer for chunked playback,
// non-streaming uses AudioPlayer for single-buffer playback
if (options.playback) {
if (particle.chunk) {
// create the player on-demand, however in the near future we'll migrate to
// Northbridge AudioPlayer for all playback needs
if (!audioPlayer)
audioPlayer = new AudioLivePlayer();
audioPlayer.enqueueChunk(audioData.buffer);
} else {
// also consider merging LiveAudioPlayer into AudioPlayer - note this will throw on malformed base64 data
void AudioPlayer.playBuffer(audioData.buffer); // fire-and-forget for whole audio
}
}
// Callback
callbacks?.onChunk?.(audioData.buffer);
break;
case 'log':
// intended to be user visible
console.log(`[Speex] (${particle.level})`, particle.message);
break;
case 'done':
const { chars, audioBytes, durationMs } = particle;
if (SPEEX_DEBUG) console.log(`[Speex RPC] Synthesis done: ${chars} chars, ${audioBytes} bytes, ${durationMs} ms`);
// NOTE: calling this will end the sound abruptly if the final chunk is still playing, so we don't do it for now
audioPlayer?.endPlayback();
break;
case 'error':
// noinspection ExceptionCaughtLocallyJS
throw new Error(particle.e);
}
}
callbacks?.onComplete?.();
// build result
const result: SpeexSpeakResult = { success: true };
if (options.returnAudio && audioChunks.length > 0) {
// Concatenate all chunks and convert to base64
const totalLength = audioChunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
const combined = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of audioChunks) {
combined.set(new Uint8Array(chunk), offset);
offset += chunk.byteLength;
}
result.audioBase64 = convert_UInt8Array_To_Base64(combined, 'speex.rpc.client');
}
return result;
} catch (error: any) {
if (SPEEX_DEBUG) console.error('[Speex RPC] Synthesis error:', { error });
// cleanup
if (audioPlayer)
void audioPlayer.stop();
const errorMessage = error.message || 'Synthesis failed';
callbacks?.onError?.(new Error(errorMessage));
return { success: false, errorType: 'tts-exception', error: errorMessage };
}
}
/**
* List voices via speex.router
*/
export async function speexListVoices_RPC_orThrow(engine: _DSpeexEngineRPC): Promise<SpeexListVoiceOption[]> {
const access = _buildRPCWireAccess(engine);
if (!access)
return [];
return (await apiAsync.speex.listVoices.query({ access })).voices;
}
// -- private helpers --
function _buildRPCWireAccess({ credentials: c, vendorType }: _DSpeexEngineRPC): SpeexWire_Access | null {
switch (c.type) {
// resolve from inline API keys
case 'api-key':
switch (vendorType) {
case 'elevenlabs':
return {
dialect: 'elevenlabs',
apiKey: c.apiKey,
...(c.apiHost && { apiHost: c.apiHost }),
};
case 'localai':
case 'openai':
return {
dialect: vendorType,
...(c.apiKey && { apiKey: c.apiKey }),
...(c.apiHost && { apiHost: c.apiHost }),
// ...(c.apiOrgId && { apiOrgId: c.apiOrgId }),
};
default:
const _exhaustiveCheck: never = vendorType;
return null;
}
// resolve from LLMs services
case 'llms-service':
const service = findModelsServiceOrNull(c.serviceId);
if (!service) return null;
switch (vendorType) {
case 'elevenlabs':
// no linking for ElevenLabs - we shall NOT be here
return null;
case 'openai':
const oai = (service.setup || {}) as DOpenAIServiceSettings;
return {
dialect: vendorType,
...(oai.oaiKey && { apiKey: oai.oaiKey }),
...(oai.oaiHost && { apiHost: oai.oaiHost }),
...(oai.oaiOrg && { apiOrgId: oai.oaiOrg }),
};
case 'localai':
const lai = (service.setup || {}) as DLocalAIServiceSettings;
return {
dialect: vendorType,
...(lai.localAIKey && { apiKey: lai.localAIKey }),
...(lai.localAIHost && { apiHost: lai.localAIHost }),
};
default:
const _exhaustiveCheck: never = vendorType;
return null;
}
}
}
@@ -0,0 +1,76 @@
import { createTRPCRouter, edgeProcedure } from '~/server/trpc/trpc.server';
import { SpeexSpeechParticle, SpeexWire, SpeexWire_Access, SpeexWire_ListVoices_Output, SpeexWire_Voice } from './rpc.wiretypes';
import { listVoicesElevenLabs, synthesizeElevenLabs } from './synthesize-elevenlabs';
import { listVoicesLocalAIOrThrow, listVoicesOpenAI, synthesizeOpenAIProtocol } from './synthesize-openai';
interface SynthesizeBackendFnParams<TSpeexAccess extends SpeexWire_Access> {
access: TSpeexAccess;
text: string;
voice: SpeexWire_Voice;
streaming: boolean;
languageCode?: string;
priority?: 'fast' | 'balanced' | 'quality';
signal?: AbortSignal;
}
export type SynthesizeBackendFn<TSpeexAccess extends SpeexWire_Access> = (params: SynthesizeBackendFnParams<TSpeexAccess>) => AsyncGenerator<SpeexSpeechParticle>;
export const speexRouter = createTRPCRouter({
/**
* Speech synthesis - streaming AsyncGenerator
* Yields SpeexParticle chunks: start, audio, done, error
*/
synthesize: edgeProcedure
.input(SpeexWire.Synthesize_input_schema)
.mutation(async function* ({ input, ctx }): AsyncGenerator<SpeexSpeechParticle> {
const { access, text, voice, streaming, languageCode, priority } = input;
try {
yield { t: 'start' };
switch (access.dialect) {
case 'elevenlabs':
yield* synthesizeElevenLabs({ access, text, voice, streaming, languageCode, priority, signal: ctx.reqSignal });
break;
case 'localai':
case 'openai':
yield* synthesizeOpenAIProtocol({ access, text, voice, streaming, languageCode, priority, signal: ctx.reqSignal });
break;
default:
const _exhaustiveCheck: never = access;
}
} catch (error) {
yield { t: 'error', e: error instanceof Error ? error.message : 'Synthesis failed' };
}
}),
/**
* List available voices for a dialect
*/
listVoices: edgeProcedure
.input(SpeexWire.ListVoices_input_schema)
.query(async ({ input }): Promise<SpeexWire_ListVoices_Output> => {
const { access } = input;
switch (access.dialect) {
case 'elevenlabs':
return await listVoicesElevenLabs(access);
case 'openai':
return { voices: listVoicesOpenAI() };
case 'localai':
return await listVoicesLocalAIOrThrow(access);
default:
const _exhaustiveCheck: never = access;
return { voices: [] };
}
}),
});
@@ -0,0 +1,87 @@
/**
* Shared streaming utilities for Speex RPC synthesizers
*
* Provides common streaming chunk accumulation logic used by
* ElevenLabs, OpenAI, and LocalAI synthesizers.
*/
import type { SpeexSpeechParticle } from './rpc.wiretypes';
/**
* Streams audio chunks from a Response, accumulating to minimum size before yielding.
*
* @param response - Fetch Response with audio body
* @param minChunkSize - Minimum bytes to accumulate before yielding (default 4096)
* @param textLength - Original text length for 'done' particle
*/
export async function* streamAudioChunksOrThrow(
response: Response,
minChunkSize: number,
textLength: number,
): AsyncGenerator<SpeexSpeechParticle> {
const reader = response.body?.getReader();
if (!reader)
return yield { t: 'error', e: 'No stream reader available' };
try {
const accumulatedChunks: Uint8Array[] = [];
let accumulatedSize = 0;
let totalAudioBytes = 0;
while (true) {
const { value, done: readerDone } = await reader.read();
if (readerDone) break;
if (!value) continue;
// Accumulate chunks
accumulatedChunks.push(value);
accumulatedSize += value.length;
// Yield when accumulated size reaches threshold
if (accumulatedSize >= minChunkSize) {
yield { t: 'audio', base64: Buffer.concat(accumulatedChunks).toString('base64'), chunk: true };
totalAudioBytes += accumulatedSize;
accumulatedChunks.length = 0;
accumulatedSize = 0;
}
}
// Yield any remaining data as final chunk
if (accumulatedSize > 0) {
yield { t: 'audio', base64: Buffer.concat(accumulatedChunks).toString('base64'), chunk: true /*, final: true*/ };
totalAudioBytes += accumulatedSize;
}
yield { t: 'done', chars: textLength, audioBytes: totalAudioBytes };
} finally {
reader.releaseLock();
}
}
/**
* Returns entire audio response as a single chunk (non-streaming mode).
* Includes optional metadata from response headers.
*/
export async function* returnAudioWholeOrThrow(
response: Response,
textLength: number,
audioMeta?: Pick<Extract<SpeexSpeechParticle, { t: 'audio' }>, 'contentType' | 'characterCost' | 'ttsLatencyMs'>,
): AsyncGenerator<SpeexSpeechParticle> {
const audioArrayBuffer = await response.arrayBuffer();
yield {
t: 'audio',
base64: Buffer.from(audioArrayBuffer).toString('base64'),
chunk: false,
...(audioMeta?.contentType ? { contentType: audioMeta.contentType } : {}),
...(audioMeta?.characterCost ? { characterCost: audioMeta.characterCost } : {}),
...(audioMeta?.ttsLatencyMs ? { ttsLatencyMs: audioMeta.ttsLatencyMs } : {}),
};
yield { t: 'done', chars: textLength, audioBytes: audioArrayBuffer.byteLength };
}
@@ -0,0 +1,117 @@
import * as z from 'zod/v4';
/**
* Streaming Speech Synthesis Particle (TS-only) schema
*/
export type SpeexSpeechParticle =
| { t: 'start' }
| { t: 'audio'; base64: string; chunk: boolean; contentType?: string; characterCost?: number; ttsLatencyMs?: number }
| { t: 'done'; durationMs?: number; chars: number; audioBytes: number }
| { t: 'log'; level: 'info', message: string }
| { t: 'error'; e: string }
;
export type SpeexWire_Access = z.infer<typeof SpeexWire.Access_schema>;
export type SpeexWire_Access_ElevenLabs = z.infer<typeof SpeexWire.AccessElevenLabs_schema>;
export type SpeexWire_Access_OpenAI = z.infer<typeof SpeexWire.AccessOpenAI_schema>;
export type SpeexWire_Voice = z.infer<typeof SpeexWire.Voice_schema>;
export type SpeexWire_Synthesize_Input = z.infer<typeof SpeexWire.Synthesize_input_schema>;
export type SpeexWire_VoiceOption = z.infer<typeof SpeexWire.VoiceOption_schema>;
export type SpeexWire_ListVoices_Input = z.infer<typeof SpeexWire.ListVoices_input_schema>;
export type SpeexWire_ListVoices_Output = z.infer<typeof SpeexWire.ListVoices_output_schema>;
/**
* Wire Protocol Schemas for Speex module
*/
export namespace SpeexWire {
// Access schemas - discriminated union by dialect
export const AccessElevenLabs_schema = z.object({
dialect: z.literal('elevenlabs'),
apiKey: z.string(),
apiHost: z.string().optional(),
});
export const AccessOpenAI_schema = z.object({
dialect: z.enum(['localai', 'openai']),
apiKey: z.string().optional(), // openai: required, localai: optional
apiHost: z.string().optional(), // localai: required, openai: optional
apiOrgId: z.string().optional(), // openai only
});
export const Access_schema = z.discriminatedUnion('dialect',
[AccessElevenLabs_schema, AccessOpenAI_schema],
);
// Voice schemas - per dialect
export const VoiceElevenLabs_schema = z.object({
dialect: z.literal('elevenlabs'),
ttsModel: z.string().optional(),
ttsVoiceId: z.string().optional(),
});
export const VoiceLocalAI_schema = z.object({
dialect: z.literal('localai'),
ttsBackend: z.string().optional(), // e.g., 'coqui', 'bark', 'piper', 'vall-e-x'
ttsModel: z.string().optional(), // e.g., 'kokoro', 'tts_models/en/ljspeech/glow-tts'
ttsLanguage: z.string().optional(), // for multilingual models like xtts_v2
});
export const VoiceOpenAI_schema = z.object({
dialect: z.literal('openai'),
ttsModel: z.enum(['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts']).optional(),
ttsVoiceId: z.string().optional(),
ttsSpeed: z.number().min(0.25).max(4.0).optional(),
ttsInstruction: z.string().optional(),
});
export const Voice_schema = z.discriminatedUnion('dialect',
[VoiceElevenLabs_schema, VoiceLocalAI_schema, VoiceOpenAI_schema],
);
// .Synthesize input schema
export const Synthesize_input_schema = z.object({
access: SpeexWire.Access_schema,
text: z.string(),
voice: SpeexWire.Voice_schema,
streaming: z.boolean(),
languageCode: z.string().optional(), // ISO language code (e.g., 'en', 'fr') for model selection fallback
priority: z.enum(['fast', 'balanced', 'quality']).optional(), // Hint for speed vs quality tradeoff
});
// .ListVoices voice schema
export const VoiceOption_schema = z.object({
id: z.string(),
name: z.string(),
description: z.string().optional(),
previewUrl: z.string().optional(),
category: z.string().optional(), // e.g., 'premade', 'cloned', 'professional'
// Voice labels (flattened for simplicity)
// gender: z.string().optional(), // e.g., 'male', 'female', 'neutral'
// accent: z.string().optional(), // e.g., 'american', 'british', 'australian'
// age: z.string().optional(), // e.g., 'young', 'middle_aged', 'old'
// language: z.string().optional(), // e.g., 'en', 'es', 'multilingual'
});
export const ListVoices_input_schema = z.object({
access: SpeexWire.Access_schema,
});
export const ListVoices_output_schema = z.object({
voices: z.array(VoiceOption_schema),
});
}
@@ -0,0 +1,199 @@
import * as z from 'zod/v4';
import { fetchJsonOrTRPCThrow, fetchResponseOrTRPCThrow } from '~/server/trpc/trpc.router.fetchers';
import type { SpeexSpeechParticle, SpeexWire_Access_ElevenLabs, SpeexWire_ListVoices_Output } from './rpc.wiretypes';
import type { SynthesizeBackendFn } from './rpc.router';
import { SPEEX_DEBUG, SPEEX_DEFAULTS } from '../../speex.config';
import { returnAudioWholeOrThrow, streamAudioChunksOrThrow } from './rpc.streaming';
// configuration
const SAFETY_TEXT_LENGTH = 1000;
const MIN_CHUNK_SIZE = 4096;
const _selectModel = (priority: 'fast' | 'balanced' | 'quality' | undefined, languageCode: string | undefined): string => {
const fast = SPEEX_DEFAULTS.ELEVENLABS_MODEL_FAST;
const quality = SPEEX_DEFAULTS.ELEVENLABS_MODEL;
return priority === 'fast' ? fast // lowest latency, best for real-time use cases like calls
: priority === 'quality' ? quality // multilingual v2 (highest quality)
: languageCode?.toLowerCase() === 'en' ? fast : quality; // 'balanced'/undefined: English → turbo, non-English → multilingual
};
export const synthesizeElevenLabs: SynthesizeBackendFn<SpeexWire_Access_ElevenLabs> = async function* (params) {
// destructure and validate
const { access, text: inputText, voice, streaming, languageCode, priority, signal } = params;
if (access.dialect !== 'elevenlabs' || voice.dialect !== 'elevenlabs')
throw new Error('Mismatched dialect in ElevenLabs synthesize');
// safety check: trim text that's too long
let text = inputText;
if (text.length > SAFETY_TEXT_LENGTH) {
text = text.slice(0, SAFETY_TEXT_LENGTH);
// -> log.info
yield { t: 'log', level: 'info', message: `Text truncated to ${SAFETY_TEXT_LENGTH} characters` };
}
// build request - narrow to elevenlabs dialect for type safety
const voiceId = voice.ttsVoiceId /*|| env.ELEVENLABS_VOICE_ID*/ || SPEEX_DEFAULTS.ELEVENLABS_VOICE;
const model = voice.ttsModel || _selectModel(priority, languageCode);
const path = `/v1/text-to-speech/${voiceId}${streaming ? '/stream' : ''}`;
const { headers, url } = _elevenlabsAccess(access, path);
const body: ElevenLabsWire.TTS_Request = {
text,
model_id: model,
} as const;
// Fetch
let response: Response;
try {
if (SPEEX_DEBUG) console.log(`[Speex][ElevenLabs] POST (stream=${streaming})`, { url, headers, body });
response = await fetchResponseOrTRPCThrow({
url,
method: 'POST',
headers,
body,
signal,
name: 'ElevenLabs',
});
} catch (error: any) {
yield { t: 'error', e: `ElevenLabs fetch failed: ${error.message || 'Unknown error'}` };
return;
}
// Stream or return whole audio (with metadata for non-streaming)
try {
yield* streaming
? streamAudioChunksOrThrow(response, MIN_CHUNK_SIZE, text.length)
: returnAudioWholeOrThrow(response, text.length, _parseTTSResponseHeaders(response.headers));
} catch (error: any) {
yield { t: 'error', e: `ElevenLabs audio error: ${error.message || 'Unknown error'}` };
}
};
export async function listVoicesElevenLabs(access: SpeexWire_Access_ElevenLabs): Promise<SpeexWire_ListVoices_Output> {
const { headers, url } = _elevenlabsAccess(access, '/v1/voices');
// fetch voices
const voicesList = ElevenLabsWire.VoicesList_schema.parse(
await fetchJsonOrTRPCThrow({
url,
headers,
name: 'ElevenLabs',
}),
);
// map to output
const voices = voicesList.voices.map(voice => ({
id: voice.voice_id,
name: voice.name,
description: voice.description || undefined,
previewUrl: voice.preview_url || undefined,
category: voice.category,
// Flatten labels for UI display
// gender: voice.labels?.gender || undefined,
// accent: voice.labels?.accent || undefined,
// age: voice.labels?.age || undefined,
// language: voice.labels?.language || undefined,
}));
// inject Rachel (default voice) if not already in the list
const rachelId = SPEEX_DEFAULTS.ELEVENLABS_VOICE;
if (!voices.some(v => v.id === rachelId))
voices.unshift({
id: rachelId,
name: 'Rachel',
description: 'Matter-of-fact, personable woman. Great for conversational use cases.',
category: 'premade',
previewUrl: undefined,
});
// sort: custom voices first, then premade
voices.sort((a, b) => {
if (a.category === 'premade' && b.category !== 'premade') return 1;
if (a.category !== 'premade' && b.category === 'premade') return -1;
return 0;
});
return { voices };
}
// Helpers
function _parseTTSResponseHeaders(headers: Headers): Pick<Extract<SpeexSpeechParticle, { t: 'audio' }>, 'contentType' | 'characterCost' | 'ttsLatencyMs'> {
return {
contentType: headers.get('content-type') || 'audio/mpeg',
characterCost: parseInt(headers.get('character-cost') || '0') || undefined,
ttsLatencyMs: parseInt(headers.get('tts-latency-ms') || '0') || undefined,
};
}
function _elevenlabsAccess(access: SpeexWire_Access_ElevenLabs, apiPath: string): { headers: HeadersInit; url: string } {
const apiKey = (access.apiKey /*|| env.ELEVENLABS_API_KEY */ || '').trim();
if (!apiKey)
throw new Error('Missing ElevenLabs API key');
let host = (access.apiHost /*|| env.ELEVENLABS_API_HOST*/ || 'api.elevenlabs.io').trim();
if (!host.startsWith('http'))
host = `https://${host}`;
if (host.endsWith('/') && apiPath.startsWith('/'))
host = host.slice(0, -1);
return {
headers: {
'Accept': 'audio/mpeg',
'Content-Type': 'application/json',
'xi-api-key': apiKey,
},
url: host + apiPath,
};
}
// Wire types for the upstream ElevenLabs API
namespace ElevenLabsWire {
// export type VoicesList = z.infer<typeof VoicesList_schema>;
export const VoicesList_schema = z.object({
voices: z.array(z.object({
voice_id: z.string(),
name: z.string(),
category: z.enum(['premade', 'cloned', 'professional']).or(z.string()),
labels: z.looseObject({
gender: z.enum(['male', 'female', 'neutral']).or(z.string()).nullish(),
accent: z.string().nullish(),
age: z.string().nullish(),
language: z.string().nullish(),
}),
description: z.string().nullish(),
preview_url: z.string().nullish(),
settings: z.object({
stability: z.number(),
similarity_boost: z.number(),
}).nullish(),
// high_quality_base_model_ids: z.array(z.string()).nullish(),
is_owner: z.boolean().nullish(),
is_legacy: z.boolean().nullish(),
})),
});
export type TTS_Request = z.infer<typeof TTS_Request_schema>;
export const TTS_Request_schema = z.object({
text: z.string(),
model_id: z.string().optional(),
voice_settings: z.object({
stability: z.number(),
similarity_boost: z.number(),
}).optional(),
});
}

Some files were not shown because too many files have changed in this diff Show More