mirror of
https://github.com/enricoros/big-AGI.git
synced 2026-05-10 21:50:14 -07:00
Compare commits
81 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e5f674509c | |||
| 197a4ae5c0 | |||
| 64d2dcf39c | |||
| caf54c736b | |||
| 423c2cce28 | |||
| a1af51efcb | |||
| ffc1bf9c58 | |||
| a54bfdb342 | |||
| 03861d2dbd | |||
| 8c080da6bf | |||
| a8c98056b6 | |||
| 78e663f955 | |||
| 70546a5039 | |||
| 30f78b33cb | |||
| 712e8c1f16 | |||
| 933dfdfb53 | |||
| 9ce86b029f | |||
| 13580cc69d | |||
| a7dee0002d | |||
| c84b2df3fa | |||
| d9471a8684 | |||
| ef630c2272 | |||
| e188c71652 | |||
| 910260c2c8 | |||
| 22752abc38 | |||
| 92bc3a5d64 | |||
| 1383752cc1 | |||
| 66af16fb81 | |||
| fc019d7b46 | |||
| ac4f0fcb12 | |||
| a6c2bc663d | |||
| e62ffa02e9 | |||
| a003600839 | |||
| ea73feb06d | |||
| 3bdf69e1b7 | |||
| 590fe78bd1 | |||
| 76187ba0e7 | |||
| 5eba375f4d | |||
| 8fa6a8251f | |||
| 75fa046f30 | |||
| 08a8cd1430 | |||
| 3afbb78a39 | |||
| fca6ccd816 | |||
| 8d351822c1 | |||
| 7d274a31fe | |||
| e36dde0d25 | |||
| 51cc6e5ae5 | |||
| 28d911c617 | |||
| b1e9fe58fb | |||
| 16ba014ade | |||
| e9d5a20c1a | |||
| 6e0036f9c4 | |||
| d7e189aa1c | |||
| ea2b444fb2 | |||
| cd1efaf26e | |||
| e47f0e5d43 | |||
| 5284d37984 | |||
| 1bf6fa0e4d | |||
| fc294c82f1 | |||
| 7b1dc49dda | |||
| d15ddeea24 | |||
| eaac213859 | |||
| 02c1460351 | |||
| 2fff35b7d9 | |||
| c5b9072bde | |||
| 8a570e912a | |||
| 1dcc40afb8 | |||
| c2092f8035 | |||
| 886c4b411e | |||
| 8888fd40cd | |||
| 31cd01bccf | |||
| c59b221004 | |||
| cb3cc3e74c | |||
| 9e90015fcc | |||
| 95e0517056 | |||
| 2b2f47915f | |||
| 9acd178ce1 | |||
| f381f80184 | |||
| c83be61343 | |||
| f6e49d31ec | |||
| cc0429a362 |
@@ -42,7 +42,8 @@ It comes packed with **world-class features** like Beam, and is praised for its
|
||||
[](https://big-agi.com/inspector)
|
||||
|
||||
### What makes Big-AGI different:
|
||||
**Intelligence**: with [Beam & Merge](https://big-agi.com/beam) for multi-model de-hallucination, native search, and bleeding-edge AI models like Nano Banana, Kimi K2 Thinking or GPT 5.1 -
|
||||
|
||||
**Intelligence**: with [Beam & Merge](https://big-agi.com/beam) for multi-model de-hallucination, native search, and bleeding-edge AI models like Opus 4.5, Nano Banana, Kimi K2 or GPT 5.1 -
|
||||
**Control**: with personas, data ownership, requests inspection, unlimited usage with API keys, and *no vendor lock-in* -
|
||||
and **Speed**: with a local-first, over-powered, zero-latency, madly optimized web app.
|
||||
|
||||
@@ -138,9 +139,14 @@ so you **are not vendor locked-in**, and obsessed over a powerful UI that works,
|
||||
NOTE: this is a powerful tool - if you need a toy UI or clone, this ain't it.
|
||||
|
||||
|
||||
## What's New in 2.0 · Oct 31, 2025 · Open
|
||||
---
|
||||
|
||||
👉 **[See the full changelog](https://big-agi.com/changes)**
|
||||
## Release Notes
|
||||
|
||||
👉 **[See the Live Release Notes](https://big-agi.com/changes)**
|
||||
- Open 2.0.1: **Opus 4.5** full support, **Gemini 3 Pro** w/ code exec, **Nano Banana Pro**, **Grok 4.1**, **GPT-5.1**, **Kimi K2 Thinking** + 280 fixes
|
||||
|
||||
### What's New in 2.0 · Oct 31, 2025 · Open
|
||||
|
||||
- **Big-AGI Open** is ready and more productive and faster than ever, with:
|
||||
- **Beam 2**: multi-modal, program-based, follow-ups, save presets
|
||||
@@ -153,7 +159,7 @@ NOTE: this is a powerful tool - if you need a toy UI or clone, this ain't it.
|
||||
|
||||
<img width="830" height="385" alt="image" src="https://github.com/user-attachments/assets/ad52761d-7e3f-44d8-b41e-947ce8b4faa1" />
|
||||
|
||||
### Open links: 👉 [changelog](https://big-agi.com/changes) 👉 [installation](docs/installation.md) 👉 [roadmap](https://github.com/users/enricoros/projects/4/views/2) 👉 [documentation](docs/README.md)
|
||||
#### **Open** links: 👉 [changelog](https://big-agi.com/changes) 👉 [installation](docs/installation.md) 👉 [roadmap](https://github.com/users/enricoros/projects/4/views/2) 👉 [documentation](docs/README.md)
|
||||
|
||||
**For teams and institutions:** Need shared prompts, SSO, or managed deployments? Reach out at enrico@big-agi.com. We're actively collecting requirements from research groups and IT departments.
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ const handlerNodeRoutes = (req: Request) => fetchRequestHandler({
|
||||
|
||||
// NOTE: the following statement breaks the build on non-pro deployments, and conditionals don't work either
|
||||
// so we resorted to raising the timeout from 10s to 60s in the vercel.json file instead
|
||||
export const maxDuration = 60;
|
||||
// export const maxDuration = 60;
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
export { handlerNodeRoutes as GET, handlerNodeRoutes as POST };
|
||||
@@ -14,5 +14,7 @@ const handlerEdgeRoutes = (req: Request) => fetchRequestHandler({
|
||||
: undefined,
|
||||
});
|
||||
|
||||
// NOTE: we don't set maxDuration explicitly here - however we set it in the Vercel project settings, raising to the limit of 300s
|
||||
// export const maxDuration = 60;
|
||||
export const runtime = 'edge';
|
||||
export { handlerEdgeRoutes as GET, handlerEdgeRoutes as POST };
|
||||
@@ -14,6 +14,9 @@ Internal documentation for Big-AGI architecture and systems, for use by AI agent
|
||||
- **[AIX.md](modules/AIX.md)** - AIX streaming architecture documentation
|
||||
- **[AIX-callers-analysis.md](modules/AIX-callers-analysis.md)** - Analysis of AIX entry points, call chains, common and different rendering, error handling, etc.
|
||||
|
||||
#### CSF - Client-Side Fetch
|
||||
- **[CSF.md](systems/client-side-fetch.md)** - Direct browser-to-API communication for LLM requests
|
||||
|
||||
### Systems Documentation
|
||||
|
||||
#### Core Platform Systems
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# CSF - Client-Side Fetch
|
||||
|
||||
Client-Side Fetch (CSF) enables direct browser-to-API communication, bypassing the server for LLM requests. When enabled, the browser makes requests directly to vendor APIs (e.g., `api.openai.com`, `api.groq.com`) instead of routing through the Next.js server. This reduces latency, decreases server load, and is particularly useful for local models where the browser can communicate directly with Ollama or LM Studio.
|
||||
|
||||
## Implementation
|
||||
|
||||
CSF is implemented as an opt-in setting stored as `csf: boolean` in each vendor's service settings. The vendor interface exposes `csfAvailable?: (setup) => boolean` to determine if CSF can be enabled (typically checking if an API key or host is configured). The actual execution happens in `aix.client.direct-chatGenerate.ts` which dynamically imports when CSF is active, making direct fetch calls using the same wire protocols as the server.
|
||||
|
||||
All 16 supported vendors (OpenAI, Anthropic, Gemini, Ollama, LocalAI, Deepseek, Groq, Mistral, xAI, OpenRouter, Perplexity, Together AI, Alibaba, Moonshot, OpenPipe, LM Studio) support CSF. Cloud vendors require CORS support from the API provider (all tested vendors return `access-control-allow-origin: *`). Local vendors (Ollama, LocalAI, LM Studio) require CORS to be enabled on the local server.
|
||||
|
||||
## UI
|
||||
|
||||
The CSF toggle appears in each vendor's setup panel under "Advanced" settings, labeled "Direct Connection". It becomes visible when the prerequisites are met (API key present for cloud vendors, host configured for local vendors). The setting is managed through `useModelServiceClientSideFetch` hook which provides `csfAvailable`, `csfActive`, `csfToggle`, and `csfReset` for UI consumption.
|
||||
+1
-1
@@ -141,7 +141,7 @@ if (process.env.POSTHOG_API_KEY && process.env.POSTHOG_ENV_ID) {
|
||||
personalApiKey: process.env.POSTHOG_API_KEY,
|
||||
envId: process.env.POSTHOG_ENV_ID,
|
||||
host: 'https://us.i.posthog.com', // backtrace upload host
|
||||
logLevel: 'info',
|
||||
logLevel: 'error', // lowered, too noisy
|
||||
sourcemaps: {
|
||||
enabled: process.env.NODE_ENV === 'production',
|
||||
project: 'big-agi',
|
||||
|
||||
Generated
+102
-102
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "big-agi",
|
||||
"version": "2.0.1",
|
||||
"version": "2.0.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "big-agi",
|
||||
"version": "2.0.1",
|
||||
"version": "2.0.2",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
@@ -42,7 +42,7 @@
|
||||
"next": "~15.1.8",
|
||||
"nprogress": "^0.2.0",
|
||||
"pdfjs-dist": "5.4.54",
|
||||
"posthog-js": "^1.298.0",
|
||||
"posthog-js": "^1.298.1",
|
||||
"posthog-node": "^5.14.0",
|
||||
"prismjs": "^1.30.0",
|
||||
"puppeteer-core": "^24.31.0",
|
||||
@@ -58,7 +58,7 @@
|
||||
"remark-mark-highlight": "^0.1.1",
|
||||
"remark-math": "^6.0.0",
|
||||
"sharp": "^0.34.5",
|
||||
"superjson": "^2.2.5",
|
||||
"superjson": "^2.2.6",
|
||||
"tesseract.js": "^6.0.1",
|
||||
"tiktoken": "^1.0.22",
|
||||
"turndown": "^7.2.2",
|
||||
@@ -1556,25 +1556,25 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@mux/mux-player": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@mux/mux-player/-/mux-player-3.9.0.tgz",
|
||||
"integrity": "sha512-OjRXdJFPstCoTipqJCXyC3e3PVoLp8jOheCaWxe2a8qvHkSs/sg+UoYegr++hAoLXXIyy2M7F6vi+tWq0W5bYA==",
|
||||
"version": "3.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@mux/mux-player/-/mux-player-3.9.1.tgz",
|
||||
"integrity": "sha512-buvulWJD9qevaCp02oTMezn6QKmV7OwU1WDt7In8IgOJ8GKn1ZWLvfSbWEd1s3Ea1tC33NastKVG8sYlJ7HNkA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mux/mux-video": "0.28.0",
|
||||
"@mux/playback-core": "0.31.3",
|
||||
"@mux/mux-video": "0.28.1",
|
||||
"@mux/playback-core": "0.31.4",
|
||||
"media-chrome": "~4.16.0",
|
||||
"player.style": "^0.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@mux/mux-player-react": {
|
||||
"version": "3.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@mux/mux-player-react/-/mux-player-react-3.9.0.tgz",
|
||||
"integrity": "sha512-AYBX89T02qOJ6rF4X2sB8WmPoHBQIrASvp6rxCf9wWYdp5lYtAjjTwaAM2aTlVEXSzdDPaOwgC7VmR7LhBJn3g==",
|
||||
"version": "3.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@mux/mux-player-react/-/mux-player-react-3.9.1.tgz",
|
||||
"integrity": "sha512-4cumt+5ObUKIZioSajLLxIghCzOn4LBcFz2mNj1OGAZGckEk8jbFB/W4o2HRlOS+4cTCEKPnOQ1l7AgjGuzZ2A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mux/mux-player": "3.9.0",
|
||||
"@mux/playback-core": "0.31.3",
|
||||
"@mux/mux-player": "3.9.1",
|
||||
"@mux/playback-core": "0.31.4",
|
||||
"prop-types": "^15.8.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
@@ -1592,32 +1592,32 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@mux/mux-video": {
|
||||
"version": "0.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@mux/mux-video/-/mux-video-0.28.0.tgz",
|
||||
"integrity": "sha512-pqpoaoxHXsGX/l7jOZGZ7jOVBVdct8jq+Be1cQ9+n5N/XrkXIGNvO/liprQET3wRuWs2Xri5lWZFRe3ZkOkYHw==",
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@mux/mux-video/-/mux-video-0.28.1.tgz",
|
||||
"integrity": "sha512-lmxPmAPtUITZ+EButcZBoVs2FZbt7UgxYV/sCVOo0kGnOCrnmqjw1G3CW9tEcFST+NzdW/nIdzpRu9kg9FxBeg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mux/mux-data-google-ima": "0.2.8",
|
||||
"@mux/playback-core": "0.31.3",
|
||||
"@mux/playback-core": "0.31.4",
|
||||
"castable-video": "~1.1.11",
|
||||
"custom-media-element": "~1.4.5",
|
||||
"media-tracks": "~0.3.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@mux/playback-core": {
|
||||
"version": "0.31.3",
|
||||
"resolved": "https://registry.npmjs.org/@mux/playback-core/-/playback-core-0.31.3.tgz",
|
||||
"integrity": "sha512-IiNnF6LeE8xB5lXzwCUmHUi40q/88ook/YYKTwDsIMG/0zY8b9Ypyzw9ghsELZH0mzuFmv+rvm3ufOIoaIi9eA==",
|
||||
"version": "0.31.4",
|
||||
"resolved": "https://registry.npmjs.org/@mux/playback-core/-/playback-core-0.31.4.tgz",
|
||||
"integrity": "sha512-qQrNAAdJ7vjr1XEObE1hOUmuYngk/fjwmtYhpzkX4jJZwUC8I0rHjeFv7LXuCQD1D/mYJlWEpuyA0gPd6Y2eQw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"hls.js": "~1.6.13",
|
||||
"hls.js": "~1.6.15",
|
||||
"mux-embed": "^5.8.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.82.tgz",
|
||||
"integrity": "sha512-FGjyUBoF0sl1EenSiE4UV2WYu76q6F9GSYedq5EiOCOyGYoQ/Owulcv6rd7v/tWOpljDDtefXXIaOCJrVKem4w==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.83.tgz",
|
||||
"integrity": "sha512-f9GVB9VNc9vn/nroc9epXRNkVpvNPZh69+qzLJIm9DfruxFqX0/jsXG46OGWAJgkO4mN0HvFHjRROMXKVmPszg==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
@@ -1627,22 +1627,22 @@
|
||||
"node": ">= 10"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.82",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.82",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.82",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.82",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.82",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.82",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.82",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.82",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.82",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.82"
|
||||
"@napi-rs/canvas-android-arm64": "0.1.83",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.83",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.83",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.83",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.83",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.83",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.83",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.83",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.83",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.83"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.82.tgz",
|
||||
"integrity": "sha512-bvZhN0iI54ouaQOrgJV96H2q7J3ZoufnHf4E1fUaERwW29Rz4rgicohnAg4venwBJZYjGl5Yl3CGmlAl1LZowQ==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.83.tgz",
|
||||
"integrity": "sha512-TbKM2fh9zXjqFIU8bgMfzG7rkrIYdLKMafgPhFoPwKrpWk1glGbWP7LEu8Y/WrMDqTGFdRqUmuX89yQEzZbkiw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1656,9 +1656,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.82.tgz",
|
||||
"integrity": "sha512-InuBHKCyuFqhNwNr4gpqazo5Xp6ltKflqOLiROn4hqAS8u21xAHyYCJRgHwd+a5NKmutFTaRWeUIT/vxWbU/iw==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.83.tgz",
|
||||
"integrity": "sha512-gp8IDVUloPUmkepHly4xRUOfUJSFNvA4jR7ZRF5nk3YcGzegSFGeICiT4PnYyPgSKEhYAFe1Y2XNy0Mp6Tu8mQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1672,9 +1672,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.82.tgz",
|
||||
"integrity": "sha512-aQGV5Ynn96onSXcuvYb2y7TRXD/t4CL2EGmnGqvLyeJX1JLSNisKQlWN/1bPDDXymZYSdUqbXehj5qzBlOx+RQ==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.83.tgz",
|
||||
"integrity": "sha512-r4ZJxiP9OgUbdGZhPDEXD3hQ0aIPcVaywtcTXvamYxTU/SWKAbKVhFNTtpRe1J30oQ25gWyxTkUKSBgUkNzdnw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1688,9 +1688,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.82.tgz",
|
||||
"integrity": "sha512-YIUpmHWeHGGRhWitT1KJkgj/JPXPfc9ox8oUoyaGPxolLGPp5AxJkq8wIg8CdFGtutget968dtwmx71m8o3h5g==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.83.tgz",
|
||||
"integrity": "sha512-Uc6aSB05qH1r+9GUDxIE6F5ZF7L0nTFyyzq8ublWUZhw8fEGK8iy931ff1ByGFT04+xHJad1kBcL4R1ZEV8z7Q==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
@@ -1704,9 +1704,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.82.tgz",
|
||||
"integrity": "sha512-AwLzwLBgmvk7kWeUgItOUor/QyG31xqtD26w1tLpf4yE0hiXTGp23yc669aawjB6FzgIkjh1NKaNS52B7/qEBQ==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.83.tgz",
|
||||
"integrity": "sha512-eEeaJA7V5KOFq7W0GtoRVbd3ak8UZpK+XLkCgUiFGtlunNw+ZZW9Cr/92MXflGe7o3SqqMUg+f975LPxO/vsOQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1720,9 +1720,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.82.tgz",
|
||||
"integrity": "sha512-moZWuqepAwWBffdF4JDadt8TgBD02iMhG6I1FHZf8xO20AsIp9rB+p0B8Zma2h2vAF/YMjeFCDmW5un6+zZz9g==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.83.tgz",
|
||||
"integrity": "sha512-cAvonp5XpbatVGegF9lMQNchs3z5RH6EtamRVnQvtoRtwbzOMcdzwuLBqDBQxQF79MFbuZNkWj3YRJjZCjHVzw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1736,9 +1736,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.82.tgz",
|
||||
"integrity": "sha512-w9++2df2kG9eC9LWYIHIlMLuhIrKGQYfUxs97CwgxYjITeFakIRazI9LYWgVzEc98QZ9x9GQvlicFsrROV59MQ==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.83.tgz",
|
||||
"integrity": "sha512-WFUPQ9qZy31vmLxIJ3MfmHw+R2g/mLCgk8zmh7maJW8snV3vLPA7pZfIS65Dc61EVDp1vaBskwQ2RqPPzwkaew==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
@@ -1752,9 +1752,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.82.tgz",
|
||||
"integrity": "sha512-lZulOPwrRi6hEg/17CaqdwWEUfOlIJuhXxincx1aVzsVOCmyHf+xFq4i6liJl1P+x2v6Iz2Z/H5zHvXJCC7Bwg==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.83.tgz",
|
||||
"integrity": "sha512-X9YwIjsuy50WwOyYeNhEHjKHO8rrfH9M4U8vNqLuGmqsZdKua/GrUhdQGdjq7lTgdY3g4+Ta5jF8MzAa7UAs/g==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1768,9 +1768,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.82.tgz",
|
||||
"integrity": "sha512-Be9Wf5RTv1w6GXlTph55K3PH3vsAh1Ax4T1FQY1UYM0QfD0yrwGdnJ8/fhqw7dEgMjd59zIbjJQC8C3msbGn5g==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.83.tgz",
|
||||
"integrity": "sha512-Vv2pLWQS8EnlSM1bstJ7vVhKA+mL4+my4sKUIn/bgIxB5O90dqiDhQjUDLP+5xn9ZMestRWDt3tdQEkGAmzq/A==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1784,9 +1784,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.82",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.82.tgz",
|
||||
"integrity": "sha512-LN/i8VrvxTDmEEK1c10z2cdOTkWT76LlTGtyZe5Kr1sqoSomKeExAjbilnu1+oee5lZUgS5yfZ2LNlVhCeARuw==",
|
||||
"version": "0.1.83",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.83.tgz",
|
||||
"integrity": "sha512-K1TtjbScfRNYhq8dengLLufXGbtEtWdUXPV505uLFPovyGHzDUGXLFP/zUJzj6xWXwgUjHNLgEPIt7mye0zr6Q==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -4958,9 +4958,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/cloudflare-video-element": {
|
||||
"version": "1.3.4",
|
||||
"resolved": "https://registry.npmjs.org/cloudflare-video-element/-/cloudflare-video-element-1.3.4.tgz",
|
||||
"integrity": "sha512-F9g+tXzGEXI6v6L48qXxr8vnR8+L6yy7IhpJxK++lpzuVekMHTixxH7/dzLuq6OacVGziU4RB5pzZYJ7/LYtJg==",
|
||||
"version": "1.3.5",
|
||||
"resolved": "https://registry.npmjs.org/cloudflare-video-element/-/cloudflare-video-element-1.3.5.tgz",
|
||||
"integrity": "sha512-zj9gjJa6xW8MNrfc4oKuwgGS0njRLpOlQjdifbuNxvy8k4Y3pKCyKCMG2XIsjd2iQGhgjS57b1P5VWdJlxcXBw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/clsx": {
|
||||
@@ -5187,9 +5187,9 @@
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/dash-video-element": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/dash-video-element/-/dash-video-element-0.3.0.tgz",
|
||||
"integrity": "sha512-Pe+BxG153n+CH++3gmWMApVXEUs767YGxsRebdNZRSZdXjbv7OGbsitYbjNMC4QAjCWBvBjIclAYV4hoc7OWSQ==",
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/dash-video-element/-/dash-video-element-0.3.1.tgz",
|
||||
"integrity": "sha512-KSdCd6lqjum4LizHLtB2EGvaGr7YJU7SZekTTDHixRondaRNcm0t9W2V3I7/itNBzQwdDbC1cKkXryc8I8IViA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"custom-media-element": "^1.4.5",
|
||||
@@ -7034,9 +7034,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/hls-video-element": {
|
||||
"version": "1.5.9",
|
||||
"resolved": "https://registry.npmjs.org/hls-video-element/-/hls-video-element-1.5.9.tgz",
|
||||
"integrity": "sha512-hDXhSI3IpSSODJF8ecNzDHKP5cqsouOuKDMjoTexyFePKr9KpXVCPAnVrXFTTH8VbOim4xkLtPkVJFt7J1Rs6w==",
|
||||
"version": "1.5.10",
|
||||
"resolved": "https://registry.npmjs.org/hls-video-element/-/hls-video-element-1.5.10.tgz",
|
||||
"integrity": "sha512-FruzD03CaQlPlNKfXO1njPbo3jCSImAtFwX1OqgFbMllTQzdYqAHODiWan0q3mr1cYCONOWiAz2/nX+2qHHC+g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"custom-media-element": "^1.4.5",
|
||||
@@ -8446,12 +8446,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/media-chrome": {
|
||||
"version": "4.16.0",
|
||||
"resolved": "https://registry.npmjs.org/media-chrome/-/media-chrome-4.16.0.tgz",
|
||||
"integrity": "sha512-c5xpTYcYo9nYsC/G/C1PyOcPXEL6iIaSR9MH3GncVuj4S90aHqvGbsyUWFDPPBKx5sCwWLxDnbszE/24eMT54g==",
|
||||
"version": "4.16.1",
|
||||
"resolved": "https://registry.npmjs.org/media-chrome/-/media-chrome-4.16.1.tgz",
|
||||
"integrity": "sha512-qtFlsy0lNDVCyVo//ZCAfRPKwgehfOYp6rThZzDUuZ5ypv41yqUfAxK+P9TOs+XSVWXATPTT2WRV0fbW0BH4vQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ce-la-react": "^0.3.0"
|
||||
"ce-la-react": "^0.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/media-tracks": {
|
||||
@@ -10207,9 +10207,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/posthog-js": {
|
||||
"version": "1.298.0",
|
||||
"resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.298.0.tgz",
|
||||
"integrity": "sha512-Zwzsf7TO8qJ6DFLuUlQSsT/5OIOcxSBZlKOSk3satkEnwKdmnBXUuxgVXRHrvq1kj7OB2PVAPgZiQ8iHHj9DRA==",
|
||||
"version": "1.298.1",
|
||||
"resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.298.1.tgz",
|
||||
"integrity": "sha512-MynFhC2HO6sg5moUfpkd0s6RzAqcqFX75kjIi4Xrj2Gl0/YQWYvFUgvv8FCpWPKPs2mdvNWYhs+oqJg0BVVHPw==",
|
||||
"license": "SEE LICENSE IN LICENSE",
|
||||
"dependencies": {
|
||||
"@posthog/core": "1.6.0",
|
||||
@@ -11209,9 +11209,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/spotify-audio-element": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/spotify-audio-element/-/spotify-audio-element-1.0.3.tgz",
|
||||
"integrity": "sha512-I1/qD8cg/UnTlCIMiKSdZUJTyYfYhaqFK7LIVElc48eOqUUbVCaw1bqL8I6mJzdMJTh3eoNyF/ewvB7NoS/g9A==",
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/spotify-audio-element/-/spotify-audio-element-1.0.4.tgz",
|
||||
"integrity": "sha512-QdKrJPkYCzaNwwz2vN2eDGyoW0KmQFmnwVprB41mpMzj4qujbqr6pegEchQeTn0b5PceKiLoVu0pp2QDpTcWnw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/sprintf-js": {
|
||||
@@ -11502,9 +11502,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/superjson": {
|
||||
"version": "2.2.5",
|
||||
"resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.5.tgz",
|
||||
"integrity": "sha512-zWPTX96LVsA/eVYnqOM2+ofcdPqdS1dAF1LN4TS2/MWuUpfitd9ctTa87wt4xrYnZnkLtS69xpBdSxVBP5Rm6w==",
|
||||
"version": "2.2.6",
|
||||
"resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.6.tgz",
|
||||
"integrity": "sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"copy-anything": "^4"
|
||||
@@ -11641,9 +11641,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tiktok-video-element": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tiktok-video-element/-/tiktok-video-element-0.1.1.tgz",
|
||||
"integrity": "sha512-BaiVzvNz2UXDKTdSrXzrNf4q6Ecc+/utYUh7zdEu2jzYcJVDoqYbVfUl0bCfMoOeeAqg28vD/yN63Y3E9jOrlA==",
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/tiktok-video-element/-/tiktok-video-element-0.1.2.tgz",
|
||||
"integrity": "sha512-w6TboLm236XJKKiIXIhCbYCnUxbixBbaAoty0etaEAZ/2kHkVIdfZdv2oouMU/HGMsWCHI/VjQ3wU3MJ+s192Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tiktoken": {
|
||||
@@ -11790,9 +11790,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/twitch-video-element": {
|
||||
"version": "0.1.5",
|
||||
"resolved": "https://registry.npmjs.org/twitch-video-element/-/twitch-video-element-0.1.5.tgz",
|
||||
"integrity": "sha512-3UdWMa5ytWFdpgJAM6XEqqRK/1FvWdJVcKDOw4IHBPt4p52E+4fXT42fBdRZFfoxBPXQNZUDDNHFW8wIopD7Og==",
|
||||
"version": "0.1.6",
|
||||
"resolved": "https://registry.npmjs.org/twitch-video-element/-/twitch-video-element-0.1.6.tgz",
|
||||
"integrity": "sha512-X7l8gy+DEFKJ/EztUwaVnAYwQN9fUJxPkOVJj2sE62sGvGU4DNLyvmOsmVulM+8Plc5dMg6hYIMNRAPaH+39Uw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/type-check": {
|
||||
@@ -12181,9 +12181,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/vimeo-video-element": {
|
||||
"version": "1.6.1",
|
||||
"resolved": "https://registry.npmjs.org/vimeo-video-element/-/vimeo-video-element-1.6.1.tgz",
|
||||
"integrity": "sha512-UwDLzhgg98pct1xb6799I1vRDXIzaAX6rs1TG/QOf6y+VrXpTFrI7mYz2gnj9QCtBcGK68f4z64A+MRYRsLJaQ==",
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/vimeo-video-element/-/vimeo-video-element-1.6.2.tgz",
|
||||
"integrity": "sha512-/l+/ugpFdCMLUDfUanzP8aSxiaNNsWvecnoIi1ziWd1shS0q7UAG3dqV9mFwO6l2GMOKtpl9uPHNHpV6RloSQA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vimeo/player": "2.29.0"
|
||||
@@ -12432,9 +12432,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/wistia-video-element": {
|
||||
"version": "1.3.5",
|
||||
"resolved": "https://registry.npmjs.org/wistia-video-element/-/wistia-video-element-1.3.5.tgz",
|
||||
"integrity": "sha512-aIG0xEtclPb9xfklAkOwHFv/BMiH3Ql0yWWKQ1XyUCoSDaF3sOD+JNLmakOChvn2LLUX7FqH/mYb8bXT4ACnMw==",
|
||||
"version": "1.3.6",
|
||||
"resolved": "https://registry.npmjs.org/wistia-video-element/-/wistia-video-element-1.3.6.tgz",
|
||||
"integrity": "sha512-wPizIpXDaCs6fvDzhU3MBtEpxIqhgXlu00kSrKgmjPb5DRqZt927xZZjE1qm81Df40d445u4a/mRKX5I66zaYA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"super-media-element": "~1.4.2"
|
||||
@@ -12589,9 +12589,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/youtube-video-element": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/youtube-video-element/-/youtube-video-element-1.8.0.tgz",
|
||||
"integrity": "sha512-u3M0MgO+KUtVwIyKJXZXXJ0As0k6d5NflOrh1GjyG8NNOp+liW2nFU29hpXeUcxUWbVKhudIYd39hMVeEgCilQ==",
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmjs.org/youtube-video-element/-/youtube-video-element-1.8.1.tgz",
|
||||
"integrity": "sha512-+5UuAGaj+5AnBf39huLVpy/4dLtR0rmJP1TxOHVZ81bac4ZHFpTtQ4Dz2FAn2GPnfXISezvUEaQoAdFW4hH9Xg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/zlibjs": {
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "big-agi",
|
||||
"version": "2.0.1",
|
||||
"version": "2.0.2",
|
||||
"private": true,
|
||||
"author": "Enrico Ros <enrico.ros@gmail.com>",
|
||||
"repository": "https://github.com/enricoros/big-agi",
|
||||
@@ -54,7 +54,7 @@
|
||||
"next": "~15.1.8",
|
||||
"nprogress": "^0.2.0",
|
||||
"pdfjs-dist": "5.4.54",
|
||||
"posthog-js": "^1.298.0",
|
||||
"posthog-js": "^1.298.1",
|
||||
"posthog-node": "^5.14.0",
|
||||
"prismjs": "^1.30.0",
|
||||
"puppeteer-core": "^24.31.0",
|
||||
@@ -70,7 +70,7 @@
|
||||
"remark-mark-highlight": "^0.1.1",
|
||||
"remark-math": "^6.0.0",
|
||||
"sharp": "^0.34.5",
|
||||
"superjson": "^2.2.5",
|
||||
"superjson": "^2.2.6",
|
||||
"tesseract.js": "^6.0.1",
|
||||
"tiktoken": "^1.0.22",
|
||||
"turndown": "^7.2.2",
|
||||
|
||||
@@ -18,7 +18,7 @@ import { ROUTE_APP_CHAT, ROUTE_INDEX } from '~/common/app.routes';
|
||||
import { Release } from '~/common/app.release';
|
||||
|
||||
// capabilities access
|
||||
import { useCapabilityBrowserSpeechRecognition, useCapabilityElevenLabs, useCapabilityTextToImage } from '~/common/components/useCapabilities';
|
||||
import { useCapabilityBrowserSpeechRecognition, useCapabilityTextToImage } from '~/common/components/useCapabilities';
|
||||
|
||||
// stores access
|
||||
import { getLLMsDebugInfo } from '~/common/stores/llms/store-llms';
|
||||
@@ -95,7 +95,6 @@ function AppDebug() {
|
||||
const cProduct = {
|
||||
capabilities: {
|
||||
mic: useCapabilityBrowserSpeechRecognition(),
|
||||
elevenLabs: useCapabilityElevenLabs(),
|
||||
textToImage: useCapabilityTextToImage(),
|
||||
},
|
||||
models: getLLMsDebugInfo(),
|
||||
|
||||
@@ -6,13 +6,15 @@ import ChatIcon from '@mui/icons-material/Chat';
|
||||
import CheckRoundedIcon from '@mui/icons-material/CheckRounded';
|
||||
import CloseRoundedIcon from '@mui/icons-material/CloseRounded';
|
||||
import MicIcon from '@mui/icons-material/Mic';
|
||||
import RecordVoiceOverTwoToneIcon from '@mui/icons-material/RecordVoiceOverTwoTone';
|
||||
import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
|
||||
|
||||
import { useSpeexGlobalEngine } from '~/modules/speex/store-module-speex';
|
||||
|
||||
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
|
||||
import { animationColorRainbow } from '~/common/util/animUtils';
|
||||
import { navigateBack } from '~/common/app.routes';
|
||||
import { optimaOpenPreferences } from '~/common/layout/optima/useOptima';
|
||||
import { useCapabilityBrowserSpeechRecognition, useCapabilityElevenLabs } from '~/common/components/useCapabilities';
|
||||
import { useCapabilityBrowserSpeechRecognition } from '~/common/components/useCapabilities';
|
||||
import { useChatStore } from '~/common/stores/chat/store-chats';
|
||||
import { useUICounter } from '~/common/stores/store-ui';
|
||||
|
||||
@@ -45,7 +47,7 @@ export function CallWizard(props: { strict?: boolean, conversationId: string | n
|
||||
|
||||
// external state
|
||||
const recognition = useCapabilityBrowserSpeechRecognition();
|
||||
const synthesis = useCapabilityElevenLabs();
|
||||
const speexGlobalEngine = useSpeexGlobalEngine();
|
||||
const chatIsEmpty = useChatStore(state => {
|
||||
if (!props.conversationId)
|
||||
return false;
|
||||
@@ -58,15 +60,16 @@ export function CallWizard(props: { strict?: boolean, conversationId: string | n
|
||||
const outOfTheBlue = !props.conversationId;
|
||||
const overriddenEmptyChat = chatEmptyOverride || !chatIsEmpty;
|
||||
const overriddenRecognition = recognitionOverride || recognition.mayWork;
|
||||
const allGood = overriddenEmptyChat && overriddenRecognition && synthesis.mayWork;
|
||||
const fatalGood = overriddenRecognition && synthesis.mayWork;
|
||||
const synthesisShallWork = !!speexGlobalEngine;
|
||||
const allGood = overriddenEmptyChat && overriddenRecognition && synthesisShallWork;
|
||||
const fatalGood = overriddenRecognition && synthesisShallWork;
|
||||
|
||||
|
||||
const handleOverrideChatEmpty = React.useCallback(() => setChatEmptyOverride(true), []);
|
||||
|
||||
const handleOverrideRecognition = React.useCallback(() => setRecognitionOverride(true), []);
|
||||
|
||||
const handleConfigureElevenLabs = React.useCallback(() => optimaOpenPreferences('voice'), []);
|
||||
const handleConfigureVoice = React.useCallback(() => optimaOpenPreferences('voice'), []);
|
||||
|
||||
const handleFinishButton = React.useCallback(() => {
|
||||
if (!allGood)
|
||||
@@ -128,17 +131,17 @@ export function CallWizard(props: { strict?: boolean, conversationId: string | n
|
||||
|
||||
{/* Text to Speech status */}
|
||||
<StatusCard
|
||||
icon={<RecordVoiceOverTwoToneIcon />}
|
||||
icon={<PhVoice />}
|
||||
text={
|
||||
(synthesis.mayWork ? 'Voice synthesis should be ready.' : 'There might be an issue with ElevenLabs voice synthesis.')
|
||||
+ (synthesis.isConfiguredServerSide ? '' : (synthesis.isConfiguredClientSide ? '' : ' Please add your API key in the settings.'))
|
||||
(synthesisShallWork ? 'Voice synthesis should be ready.' : 'There might be an issue with voice synthesis.')
|
||||
// + (synthesis.isConfiguredServerSide ? '' : (synthesis.isConfiguredClientSide ? '' : ' Please add your API key in the settings.'))
|
||||
}
|
||||
button={synthesis.mayWork ? undefined : (
|
||||
<Button variant='outlined' onClick={handleConfigureElevenLabs} sx={{ mx: 1 }}>
|
||||
button={synthesisShallWork ? undefined : (
|
||||
<Button variant='outlined' onClick={handleConfigureVoice} sx={{ mx: 1 }}>
|
||||
Configure
|
||||
</Button>
|
||||
)}
|
||||
hasIssue={!synthesis.mayWork}
|
||||
hasIssue={!synthesisShallWork}
|
||||
/>
|
||||
|
||||
{/*<Typography>*/}
|
||||
|
||||
@@ -317,7 +317,7 @@ export function Contacts(props: { setCallIntent: (intent: AppCallIntent) => void
|
||||
issue={354}
|
||||
text='Call App: Support thread and compatibility matrix'
|
||||
note={<>
|
||||
Voice input uses the HTML Web Speech API, and speech output requires an ElevenLabs API Key.
|
||||
Voice input uses the HTML Web Speech API.
|
||||
</>}
|
||||
// note2='Please report any issues you encounter'
|
||||
sx={{
|
||||
|
||||
+17
-30
@@ -7,22 +7,22 @@ import CallEndIcon from '@mui/icons-material/CallEnd';
|
||||
import CallIcon from '@mui/icons-material/Call';
|
||||
import MicIcon from '@mui/icons-material/Mic';
|
||||
import MicNoneIcon from '@mui/icons-material/MicNone';
|
||||
import RecordVoiceOverTwoToneIcon from '@mui/icons-material/RecordVoiceOverTwoTone';
|
||||
|
||||
import { ScrollToBottom } from '~/common/scroll-to-bottom/ScrollToBottom';
|
||||
import { ScrollToBottomButton } from '~/common/scroll-to-bottom/ScrollToBottomButton';
|
||||
import { useChatLLMDropdown } from '../chat/components/layout-bar/useLLMDropdown';
|
||||
|
||||
import { SystemPurposeId, SystemPurposes } from '../../data';
|
||||
import { elevenLabsSpeakText } from '~/modules/elevenlabs/elevenlabs.client';
|
||||
import { AixChatGenerateContent_DMessageGuts, aixChatGenerateContent_DMessage_FromConversation } from '~/modules/aix/client/aix.client';
|
||||
import { useElevenLabsVoiceDropdown } from '~/modules/elevenlabs/useElevenLabsVoiceDropdown';
|
||||
|
||||
import { aixChatGenerateContent_DMessage_FromConversation, AixChatGenerateContent_DMessageGuts } from '~/modules/aix/client/aix.client';
|
||||
import { speakText } from '~/modules/speex/speex.client';
|
||||
|
||||
import type { OptimaBarControlMethods } from '~/common/layout/optima/bar/OptimaBarDropdown';
|
||||
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { OptimaPanelGroupedList } from '~/common/layout/optima/panel/OptimaPanelGroupedList';
|
||||
import { OptimaPanelIn, OptimaToolbarIn } from '~/common/layout/optima/portals/OptimaPortalsIn';
|
||||
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
|
||||
import { SpeechResult, useSpeechRecognition } from '~/common/components/speechrecognition/useSpeechRecognition';
|
||||
import { conversationTitle, remapMessagesSysToUsr } from '~/common/stores/chat/chat.conversation';
|
||||
import { createDMessageFromFragments, createDMessageTextContent, DMessage, messageFragmentsReduceText, messageWasInterruptedAtStart } from '~/common/stores/chat/chat.message';
|
||||
@@ -43,18 +43,13 @@ import { useAppCallStore } from './state/store-app-call';
|
||||
function CallMenu(props: {
|
||||
pushToTalk: boolean,
|
||||
setPushToTalk: (pushToTalk: boolean) => void,
|
||||
override: boolean,
|
||||
setOverride: (overridePersonaVoice: boolean) => void,
|
||||
}) {
|
||||
|
||||
// external state
|
||||
const { grayUI, toggleGrayUI } = useAppCallStore();
|
||||
const { voicesDropdown } = useElevenLabsVoiceDropdown(false, !props.override);
|
||||
|
||||
const handlePushToTalkToggle = () => props.setPushToTalk(!props.pushToTalk);
|
||||
|
||||
const handleChangeVoiceToggle = () => props.setOverride(!props.override);
|
||||
|
||||
return <OptimaPanelGroupedList title='Call'>
|
||||
|
||||
<MenuItem onClick={handlePushToTalkToggle}>
|
||||
@@ -63,17 +58,6 @@ function CallMenu(props: {
|
||||
<Switch checked={props.pushToTalk} onChange={handlePushToTalkToggle} sx={{ ml: 'auto' }} />
|
||||
</MenuItem>
|
||||
|
||||
<MenuItem onClick={handleChangeVoiceToggle}>
|
||||
<ListItemDecorator><RecordVoiceOverTwoToneIcon /></ListItemDecorator>
|
||||
Change Voice
|
||||
<Switch checked={props.override} onChange={handleChangeVoiceToggle} sx={{ ml: 'auto' }} />
|
||||
</MenuItem>
|
||||
|
||||
<MenuItem>
|
||||
<ListItemDecorator>{' '}</ListItemDecorator>
|
||||
{voicesDropdown}
|
||||
</MenuItem>
|
||||
|
||||
<ListDivider />
|
||||
|
||||
<MenuItem onClick={toggleGrayUI}>
|
||||
@@ -98,7 +82,6 @@ export function Telephone(props: {
|
||||
const [avatarClickCount, setAvatarClickCount] = React.useState<number>(0);// const [micMuted, setMicMuted] = React.useState(false);
|
||||
const [callElapsedTime, setCallElapsedTime] = React.useState<string>('00:00');
|
||||
const [callMessages, setCallMessages] = React.useState<DMessage[]>([]);
|
||||
const [overridePersonaVoice, setOverridePersonaVoice] = React.useState<boolean>(false);
|
||||
const [personaTextInterim, setPersonaTextInterim] = React.useState<string | null>(null);
|
||||
const [pushToTalk, setPushToTalk] = React.useState(true);
|
||||
const [stage, setStage] = React.useState<'ring' | 'declined' | 'connected' | 'ended'>('ring');
|
||||
@@ -118,7 +101,7 @@ export function Telephone(props: {
|
||||
}));
|
||||
const persona = SystemPurposes[props.callIntent.personaId as SystemPurposeId] ?? undefined;
|
||||
const personaCallStarters = persona?.call?.starters ?? undefined;
|
||||
const personaVoiceId = overridePersonaVoice ? undefined : (persona?.voices?.elevenLabs?.voiceId ?? undefined);
|
||||
// const personaVoiceSelector = React.useMemo(() => personaGetVoiceSelector(persona), [persona]);
|
||||
const personaSystemMessage = persona?.systemMessage ?? undefined;
|
||||
|
||||
// hooks and speech
|
||||
@@ -165,7 +148,6 @@ export function Telephone(props: {
|
||||
};
|
||||
|
||||
// [E] pickup -> seed message and call timer
|
||||
// FIXME: Overriding the voice will reset the call - not a desired behavior
|
||||
React.useEffect(() => {
|
||||
if (!isConnected) return;
|
||||
|
||||
@@ -185,11 +167,14 @@ export function Telephone(props: {
|
||||
|
||||
setCallMessages([createDMessageTextContent('assistant', firstMessage)]); // [state] set assistant:hello message
|
||||
|
||||
// fire/forget
|
||||
void elevenLabsSpeakText(firstMessage, personaVoiceId, true, true);
|
||||
// fire/forget - use 'fast' priority for real-time conversation
|
||||
void speakText(firstMessage,
|
||||
undefined,
|
||||
{ label: 'Call', priority: 'fast' },
|
||||
);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isConnected, personaCallStarters, personaVoiceId]);
|
||||
}, [isConnected, personaCallStarters]);
|
||||
|
||||
// [E] persona streaming response - upon new user message
|
||||
React.useEffect(() => {
|
||||
@@ -270,9 +255,12 @@ export function Telephone(props: {
|
||||
fullMessage.generator = status.lastDMessage.generator;
|
||||
setCallMessages(messages => [...messages, fullMessage]); // [state] append assistant:call_response
|
||||
|
||||
// fire/forget
|
||||
// fire/forget - use 'fast' priority for real-time conversation
|
||||
if (status.outcome === 'success' && finalText?.length >= 1)
|
||||
void elevenLabsSpeakText(finalText, personaVoiceId, true, true);
|
||||
void speakText(finalText,
|
||||
undefined,
|
||||
{ label: 'Call', priority: 'fast' },
|
||||
);
|
||||
|
||||
}).catch((err: DOMException) => {
|
||||
if (err?.name !== 'AbortError') {
|
||||
@@ -288,7 +276,7 @@ export function Telephone(props: {
|
||||
responseAbortController.current?.abort();
|
||||
responseAbortController.current = null;
|
||||
};
|
||||
}, [isConnected, callMessages, modelId, personaVoiceId, personaSystemMessage, reMessages]);
|
||||
}, [callMessages, isConnected, modelId, personaSystemMessage, reMessages]);
|
||||
|
||||
// [E] Message interrupter
|
||||
const abortTrigger = isConnected && recognitionState.hasSpeech;
|
||||
@@ -325,7 +313,6 @@ export function Telephone(props: {
|
||||
<OptimaPanelIn>
|
||||
<CallMenu
|
||||
pushToTalk={pushToTalk} setPushToTalk={setPushToTalk}
|
||||
override={overridePersonaVoice} setOverride={setOverridePersonaVoice}
|
||||
/>
|
||||
</OptimaPanelIn>
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import type { DiagramConfig } from '~/modules/aifn/digrams/DiagramsModal';
|
||||
import type { TradeConfig } from '~/modules/trade/TradeModal';
|
||||
import { downloadSingleChat, importConversationsFromFilesAtRest, openConversationsAtRestPicker } from '~/modules/trade/trade.client';
|
||||
import { imaginePromptFromTextOrThrow } from '~/modules/aifn/imagine/imaginePromptFromText';
|
||||
import { elevenLabsSpeakText } from '~/modules/elevenlabs/elevenlabs.client';
|
||||
import { useAreBeamsOpen } from '~/modules/beam/store-beam.hooks';
|
||||
import { useCapabilityTextToImage } from '~/modules/t2i/t2i.client';
|
||||
|
||||
@@ -346,11 +345,6 @@ export function AppChat() {
|
||||
});
|
||||
}, [handleExecuteAndOutcome]);
|
||||
|
||||
const handleTextSpeak = React.useCallback(async (text: string): Promise<void> => {
|
||||
await elevenLabsSpeakText(text, undefined, true, true);
|
||||
}, []);
|
||||
|
||||
|
||||
// Chat actions
|
||||
|
||||
const handleConversationNewInFocusedPane = React.useCallback((forceNoRecycle: boolean, isIncognito: boolean) => {
|
||||
@@ -725,7 +719,6 @@ export function AppChat() {
|
||||
onConversationNew={handleConversationNewInFocusedPane}
|
||||
onTextDiagram={handleTextDiagram}
|
||||
onTextImagine={handleImagineFromText}
|
||||
onTextSpeak={handleTextSpeak}
|
||||
sx={chatMessageListSx}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { Box, List } from '@mui/joy';
|
||||
import type { SystemPurposeExample } from '../../../data';
|
||||
|
||||
import type { DiagramConfig } from '~/modules/aifn/digrams/DiagramsModal';
|
||||
import { speakText } from '~/modules/speex/speex.client';
|
||||
|
||||
import type { ConversationHandler } from '~/common/chat-overlay/ConversationHandler';
|
||||
import type { DLLMContextTokens } from '~/common/stores/llms/llms.types';
|
||||
@@ -17,7 +18,6 @@ import { createDMessageFromFragments, createDMessageTextContent, DMessage, DMess
|
||||
import { createTextContentFragment, DMessageFragment, DMessageFragmentId } from '~/common/stores/chat/chat.fragments';
|
||||
import { openFileForAttaching } from '~/common/components/ButtonAttachFiles';
|
||||
import { optimaOpenPreferences } from '~/common/layout/optima/useOptima';
|
||||
import { useCapabilityElevenLabs } from '~/common/components/useCapabilities';
|
||||
import { useChatOverlayStore } from '~/common/chat-overlay/store-perchat_vanilla';
|
||||
import { useChatStore } from '~/common/stores/chat/store-chats';
|
||||
import { useScrollToBottom } from '~/common/scroll-to-bottom/useScrollToBottom';
|
||||
@@ -50,7 +50,6 @@ export function ChatMessageList(props: {
|
||||
onConversationNew: (forceNoRecycle: boolean, isIncognito: boolean) => void,
|
||||
onTextDiagram: (diagramConfig: DiagramConfig | null) => void,
|
||||
onTextImagine: (conversationId: DConversationId, selectedText: string) => Promise<void>,
|
||||
onTextSpeak: (selectedText: string) => Promise<void>,
|
||||
setIsMessageSelectionMode: (isMessageSelectionMode: boolean) => void,
|
||||
sx?: SxProps,
|
||||
}) {
|
||||
@@ -75,10 +74,9 @@ export function ChatMessageList(props: {
|
||||
_composerInReferenceToCount: state.inReferenceTo?.length ?? 0,
|
||||
ephemerals: state.ephemerals?.length ? state.ephemerals : null,
|
||||
})));
|
||||
const { mayWork: isSpeakable } = useCapabilityElevenLabs();
|
||||
|
||||
// derived state
|
||||
const { conversationHandler, conversationId, capabilityHasT2I, onConversationBranch, onConversationExecuteHistory, onTextDiagram, onTextImagine, onTextSpeak } = props;
|
||||
const { conversationHandler, conversationId, capabilityHasT2I, onConversationBranch, onConversationExecuteHistory, onTextDiagram, onTextImagine } = props;
|
||||
const composerCanAddInReferenceTo = _composerInReferenceToCount < 5;
|
||||
const composerHasInReferenceto = _composerInReferenceToCount > 0;
|
||||
|
||||
@@ -212,12 +210,15 @@ export function ChatMessageList(props: {
|
||||
}, [capabilityHasT2I, conversationId, onTextImagine]);
|
||||
|
||||
const handleTextSpeak = React.useCallback(async (text: string) => {
|
||||
if (!isSpeakable)
|
||||
return optimaOpenPreferences('voice');
|
||||
// sandwich the speaking with the indicator
|
||||
setIsSpeaking(true);
|
||||
await onTextSpeak(text);
|
||||
const result = await speakText(text, undefined, { label: 'Chat speak' });
|
||||
setIsSpeaking(false);
|
||||
}, [isSpeakable, onTextSpeak]);
|
||||
|
||||
// open voice preferences
|
||||
if (!result.success && (result.errorType === 'tts-no-engine' || result.errorType === 'tts-unconfigured'))
|
||||
optimaOpenPreferences('voice');
|
||||
}, []);
|
||||
|
||||
|
||||
// operate on the local selection set
|
||||
@@ -377,7 +378,7 @@ export function ChatMessageList(props: {
|
||||
onMessageTruncate={handleMessageTruncate}
|
||||
onTextDiagram={handleTextDiagram}
|
||||
onTextImagine={capabilityHasT2I ? handleTextImagine : undefined}
|
||||
onTextSpeak={isSpeakable ? handleTextSpeak : undefined}
|
||||
onTextSpeak={handleTextSpeak}
|
||||
/>
|
||||
|
||||
);
|
||||
|
||||
@@ -905,7 +905,7 @@ export function Composer(props: {
|
||||
)}
|
||||
|
||||
{!showChatInReferenceTo && !isDraw && tokenLimit > 0 && (
|
||||
<TokenBadgeMemo hideBelowDollars={0.005} chatPricing={tokenChatPricing} direct={tokensComposer} history={tokensHistory} responseMax={tokensResponseMax} limit={tokenLimit} showCost={labsShowCost} enableHover={!isMobile} showExcess absoluteBottomRight />
|
||||
<TokenBadgeMemo hideBelowDollars={0.01} chatPricing={tokenChatPricing} direct={tokensComposer} history={tokensHistory} responseMax={tokensResponseMax} limit={tokenLimit} showCost={labsShowCost} enableHover={!isMobile} showExcess absoluteBottomRight />
|
||||
)}
|
||||
|
||||
</Box>
|
||||
|
||||
@@ -47,9 +47,9 @@ function TokenBadge(props: {
|
||||
const showAltCosts = !!props.showCost && !!costMax && costMin !== undefined;
|
||||
if (showAltCosts) {
|
||||
// Note: switched to 'min cost (>= ...)' on mobile as well, to restore the former behavior, just uncomment the !props.enableHover (a proxy for isMobile)
|
||||
badgeValue = (/*!props.enableHover ||*/ isHovering)
|
||||
? '< ' + formatModelsCost(costMax)
|
||||
: '> ' + formatModelsCost(costMin);
|
||||
badgeValue =
|
||||
// (/*!props.enableHover ||*/ isHovering) ? '< ' + formatModelsCost(costMax) :
|
||||
'> ' + formatModelsCost(costMin);
|
||||
} else {
|
||||
|
||||
// show the direct tokens, unless we exceed the limit and 'showExcess' is enabled
|
||||
@@ -77,7 +77,7 @@ function TokenBadge(props: {
|
||||
slotProps={{
|
||||
root: {
|
||||
sx: {
|
||||
...((props.absoluteBottomRight) && { position: 'absolute', bottom: 8, right: 8 }),
|
||||
...((props.absoluteBottomRight) && { position: 'absolute', bottom: 8, right: '1rem' }),
|
||||
cursor: 'help',
|
||||
...(shallInvisible && {
|
||||
opacity: 0,
|
||||
@@ -92,6 +92,13 @@ function TokenBadge(props: {
|
||||
fontFamily: 'code',
|
||||
fontSize: 'xs',
|
||||
...((props.absoluteBottomRight || props.inline) && { position: 'static', transform: 'none' }),
|
||||
// make it transparent over text
|
||||
// backgroundColor: `rgb(var(--joy-palette-${color}-lightChannel) / 15%)`, // similar to success.50
|
||||
background: 'transparent',
|
||||
boxShadow: 'none', // outline
|
||||
'&:hover': {
|
||||
backgroundColor: `${color}.softHoverBg`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
@@ -21,7 +21,6 @@ import InsertLinkIcon from '@mui/icons-material/InsertLink';
|
||||
import MoreVertIcon from '@mui/icons-material/MoreVert';
|
||||
import NotificationsActiveIcon from '@mui/icons-material/NotificationsActive';
|
||||
import NotificationsOutlinedIcon from '@mui/icons-material/NotificationsOutlined';
|
||||
import RecordVoiceOverOutlinedIcon from '@mui/icons-material/RecordVoiceOverOutlined';
|
||||
import ReplayIcon from '@mui/icons-material/Replay';
|
||||
import ReplyAllRoundedIcon from '@mui/icons-material/ReplyAllRounded';
|
||||
import ReplyRoundedIcon from '@mui/icons-material/ReplyRounded';
|
||||
@@ -40,6 +39,7 @@ import { CloseablePopup } from '~/common/components/CloseablePopup';
|
||||
import { DMessage, DMessageId, DMessageUserFlag, DMetaReferenceItem, MESSAGE_FLAG_AIX_SKIP, MESSAGE_FLAG_NOTIFY_COMPLETE, MESSAGE_FLAG_STARRED, MESSAGE_FLAG_VND_ANT_CACHE_AUTO, MESSAGE_FLAG_VND_ANT_CACHE_USER, messageFragmentsReduceText, messageHasUserFlag } from '~/common/stores/chat/chat.message';
|
||||
import { KeyStroke } from '~/common/components/KeyStroke';
|
||||
import { MarkHighlightIcon } from '~/common/components/icons/MarkHighlightIcon';
|
||||
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
|
||||
import { Release } from '~/common/app.release';
|
||||
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
|
||||
import { adjustContentScaling, themeScalingMap, themeZIndexChatBubble } from '~/common/app.theme';
|
||||
@@ -1027,7 +1027,7 @@ export function ChatMessage(props: {
|
||||
)}
|
||||
{!!props.onTextSpeak && (
|
||||
<MenuItem onClick={handleOpsSpeak} disabled={!couldSpeak || props.isSpeaking}>
|
||||
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <RecordVoiceOverOutlinedIcon />}</ListItemDecorator>
|
||||
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <PhVoice />}</ListItemDecorator>
|
||||
Speak
|
||||
</MenuItem>
|
||||
)}
|
||||
@@ -1155,7 +1155,7 @@ export function ChatMessage(props: {
|
||||
</Tooltip>}
|
||||
{!!props.onTextSpeak && <Tooltip disableInteractive arrow placement='top' title='Speak'>
|
||||
<IconButton color='success' onClick={handleOpsSpeak} disabled={!couldSpeak || props.isSpeaking}>
|
||||
{!props.isSpeaking ? <RecordVoiceOverOutlinedIcon /> : <CircularProgress sx={{ '--CircularProgress-size': '16px' }} />}
|
||||
{!props.isSpeaking ? <PhVoice /> : <CircularProgress sx={{ '--CircularProgress-size': '16px' }} />}
|
||||
</IconButton>
|
||||
</Tooltip>}
|
||||
{(!!props.onTextDiagram || !!props.onTextImagine || !!props.onTextSpeak) && <Divider />}
|
||||
@@ -1195,7 +1195,7 @@ export function ChatMessage(props: {
|
||||
Auto-Draw
|
||||
</MenuItem>}
|
||||
{!!props.onTextSpeak && <MenuItem onClick={handleOpsSpeak} disabled={!couldSpeak || props.isSpeaking}>
|
||||
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <RecordVoiceOverOutlinedIcon />}</ListItemDecorator>
|
||||
<ListItemDecorator>{props.isSpeaking ? <CircularProgress size='sm' /> : <PhVoice />}</ListItemDecorator>
|
||||
Speak
|
||||
</MenuItem>}
|
||||
</CloseablePopup>
|
||||
|
||||
+2
-2
@@ -7,13 +7,13 @@ import CodeIcon from '@mui/icons-material/Code';
|
||||
import EditRoundedIcon from '@mui/icons-material/EditRounded';
|
||||
import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined';
|
||||
import PictureAsPdfIcon from '@mui/icons-material/PictureAsPdf';
|
||||
import RecordVoiceOverOutlinedIcon from '@mui/icons-material/RecordVoiceOverOutlined';
|
||||
import TextFieldsIcon from '@mui/icons-material/TextFields';
|
||||
import TextureIcon from '@mui/icons-material/Texture';
|
||||
|
||||
import { ContentScaling, themeScalingMap } from '~/common/app.theme';
|
||||
import { DMessageAttachmentFragment, DMessageFragmentId, DVMimeType, isDocPart } from '~/common/stores/chat/chat.fragments';
|
||||
import { LiveFileIcon } from '~/common/livefile/liveFile.icons';
|
||||
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
|
||||
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
|
||||
import { ellipsizeMiddle } from '~/common/util/textUtils';
|
||||
import { useLiveFileMetadata } from '~/common/livefile/useLiveFileMetadata';
|
||||
@@ -41,7 +41,7 @@ export function buttonIconForFragment(part: DMessageAttachmentFragment['part']):
|
||||
case 'image':
|
||||
return ImageOutlinedIcon;
|
||||
case 'audio':
|
||||
return RecordVoiceOverOutlinedIcon;
|
||||
return PhVoice;
|
||||
default:
|
||||
const _exhaustiveCheck: never = assetType;
|
||||
return TextureIcon; // missing zync asset type
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { ContentScaling } from '~/common/app.theme';
|
||||
import type { DMessageErrorPart } from '~/common/stores/chat/chat.fragments';
|
||||
import type { DMessageRole } from '~/common/stores/chat/chat.message';
|
||||
|
||||
import { BlockPartError_NetDisconnected } from './BlockPartError_NetDisconnected';
|
||||
import { BlockPartError_RequestExceeded } from './BlockPartError_RequestExceeded';
|
||||
|
||||
|
||||
@@ -19,6 +20,19 @@ export function BlockPartError(props: {
|
||||
|
||||
// special error presentation, based on hints
|
||||
switch (props.errorHint) {
|
||||
case 'aix-net-disconnected':
|
||||
// determine the 2 'kinds' of disconnection errors in aix.client.ts
|
||||
const kind =
|
||||
props.errorText.includes('**network error**') ? 'net-client-closed'
|
||||
: props.errorText.includes('**connection terminated**') ? 'net-server-closed'
|
||||
: 'net-unknown-closed';
|
||||
|
||||
// For client-side error, we don't show the _NetDisconnected component
|
||||
if (kind === 'net-client-closed')
|
||||
break;
|
||||
|
||||
return <BlockPartError_NetDisconnected disconnectionKind={kind} messageGeneratorLlmId={props.messageGeneratorLlmId} contentScaling={props.contentScaling} />;
|
||||
|
||||
case 'aix-request-exceeded':
|
||||
return <BlockPartError_RequestExceeded messageGeneratorLlmId={props.messageGeneratorLlmId} contentScaling={props.contentScaling} />;
|
||||
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Alert, Box, FormHelperText, Switch } from '@mui/joy';
|
||||
import WifiOffRoundedIcon from '@mui/icons-material/WifiOffRounded';
|
||||
|
||||
import type { ContentScaling } from '~/common/app.theme';
|
||||
import { useLLM } from '~/common/stores/llms/llms.hooks';
|
||||
import { useModelServiceClientSideFetch } from '~/common/stores/llms/hooks/useModelServiceClientSideFetch';
|
||||
|
||||
|
||||
/**
|
||||
* Error recovery component for "Connection terminated" errors.
|
||||
*/
|
||||
export function BlockPartError_NetDisconnected(props: {
|
||||
disconnectionKind: 'net-client-closed' | 'net-server-closed' | 'net-unknown-closed';
|
||||
messageGeneratorLlmId?: string | null;
|
||||
contentScaling: ContentScaling;
|
||||
}) {
|
||||
|
||||
// external state
|
||||
const model = useLLM(props.messageGeneratorLlmId) ?? null;
|
||||
const isServerSideClosed = props.disconnectionKind === 'net-server-closed'; // do not show CSF option for non-server-side
|
||||
const { csfAvailable, csfActive, csfToggle, vendorName } = useModelServiceClientSideFetch(isServerSideClosed, model);
|
||||
|
||||
return (
|
||||
<Alert
|
||||
size={props.contentScaling === 'xs' ? 'sm' : 'md'}
|
||||
color='danger'
|
||||
variant='plain'
|
||||
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1 }}
|
||||
>
|
||||
|
||||
|
||||
<Box sx={{ flex: 1, display: 'flex', flexDirection: 'column', gap: 0.5, alignItems: 'flex-start' }}>
|
||||
|
||||
{/* Header */}
|
||||
<Box sx={{ display: 'flex', gap: 2 }}>
|
||||
<WifiOffRoundedIcon sx={{ flexShrink: 0, mt: 0.5 }} />
|
||||
<div>
|
||||
<Box fontSize='larger'>
|
||||
Connection Terminated
|
||||
</Box>
|
||||
<div>
|
||||
The connection was unexpectedly closed before the response completed.
|
||||
</div>
|
||||
</div>
|
||||
</Box>
|
||||
|
||||
|
||||
{/* Recovery options */}
|
||||
{csfAvailable ? <>
|
||||
|
||||
{/* Explanation */}
|
||||
<Box color='text.tertiary' fontSize='sm' my={2}>
|
||||
<strong>Experimental:</strong> enable direct connection to {vendorName} to bypass server timeouts - then try again.
|
||||
</Box>
|
||||
|
||||
{/* Toggle */}
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
p: 2,
|
||||
borderRadius: 'sm',
|
||||
bgcolor: 'background.popup',
|
||||
boxShadow: 'md',
|
||||
// border: '1px solid',
|
||||
// borderColor: 'divider',
|
||||
}}
|
||||
>
|
||||
|
||||
<Box sx={{ flex: 1 }}>
|
||||
<Box color={!csfActive ? undefined : 'primary.solidBg'} fontWeight='lg' mb={0.5}>
|
||||
Direct Connection {csfActive && '- Now Try Again'}
|
||||
</Box>
|
||||
<FormHelperText>
|
||||
Connect directly from this client -> {vendorName || 'AI service'}
|
||||
</FormHelperText>
|
||||
</Box>
|
||||
|
||||
<Switch
|
||||
checked={csfActive}
|
||||
onChange={(e) => csfToggle(e.target.checked)}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
</> : (
|
||||
<div>
|
||||
<Box sx={{ color: 'text.secondary', my: 1 }}>
|
||||
Suggestions:
|
||||
</Box>
|
||||
<Box component='ul' sx={{ color: 'text.secondary' }}>
|
||||
<li>Check your internet connection and try again</li>
|
||||
<li>The AI service may be experiencing issues - wait a moment and retry</li>
|
||||
<li>If the issue persists, please let us know promptly on Discord or GitHib</li>
|
||||
</Box>
|
||||
</div>
|
||||
)}
|
||||
</Box>
|
||||
</Alert>
|
||||
);
|
||||
}
|
||||
+58
-59
@@ -1,6 +1,6 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Alert, Box, FormHelperText, Switch, Typography } from '@mui/joy';
|
||||
import { Alert, Box, FormHelperText, Switch } from '@mui/joy';
|
||||
import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
|
||||
|
||||
import type { ContentScaling } from '~/common/app.theme';
|
||||
@@ -19,13 +19,13 @@ export function BlockPartError_RequestExceeded(props: {
|
||||
|
||||
// external state
|
||||
const model = useLLM(props.messageGeneratorLlmId) ?? null;
|
||||
const { csfAvailable, csfActive, csfToggle } = useModelServiceClientSideFetch(true, model);
|
||||
const { csfAvailable, csfActive, csfToggle, vendorName } = useModelServiceClientSideFetch(true, model);
|
||||
|
||||
return (
|
||||
<Alert
|
||||
size={props.contentScaling === 'xs' ? 'sm' : 'md'}
|
||||
color='danger'
|
||||
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1 }}
|
||||
color='warning'
|
||||
sx={{ display: 'flex', alignItems: 'flex-start', gap: 1, border: '1px solid', borderColor: 'warning.outlinedBorder' }}
|
||||
>
|
||||
|
||||
<WarningRoundedIcon sx={{ flexShrink: 0, mt: 0.25 }} />
|
||||
@@ -36,70 +36,69 @@ export function BlockPartError_RequestExceeded(props: {
|
||||
Request Too Large
|
||||
</Box>
|
||||
<div>
|
||||
Your message or attachments exceed the limit of the Vercel edge network.
|
||||
Your message or attachments exceed the limit of the Vercel edge network
|
||||
</div>
|
||||
|
||||
{/* Recovery options */}
|
||||
{csfAvailable ? (
|
||||
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
||||
{csfAvailable ? <>
|
||||
|
||||
{/* Explanation */}
|
||||
<Box color='text.secondary' fontSize='sm'>
|
||||
<strong>Experimental:</strong> enable direct connection to the AI services, and try again.
|
||||
</Box>
|
||||
|
||||
{/* Toggle */}
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
p: 2,
|
||||
borderRadius: 'sm',
|
||||
bgcolor: 'background.popup',
|
||||
boxShadow: 'md',
|
||||
}}
|
||||
>
|
||||
|
||||
<Box sx={{ flex: 1 }}>
|
||||
<Box color={!csfActive ? undefined : 'primary.solidBg'} fontWeight='lg' mb={0.5}>
|
||||
Direct Connection {csfActive && '- Now Try Again'}
|
||||
</Box>
|
||||
<FormHelperText>
|
||||
Bypassing servers and connect directly from this client -> AI provider
|
||||
</FormHelperText>
|
||||
</Box>
|
||||
|
||||
<Switch
|
||||
checked={csfActive}
|
||||
onChange={(e) => csfToggle(e.target.checked)}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
{/* Regenerate button */}
|
||||
{/*{props.onRegenerate && (*/}
|
||||
{/* <Button*/}
|
||||
{/* size='sm'*/}
|
||||
{/* variant={csfActive ? 'solid' : 'outlined'}*/}
|
||||
{/* color={csfActive ? 'success' : 'neutral'}*/}
|
||||
{/* startDecorator={<RefreshIcon />}*/}
|
||||
{/* onClick={props.onRegenerate}*/}
|
||||
{/* sx={{ alignSelf: 'flex-start' }}*/}
|
||||
{/* >*/}
|
||||
{/* {csfActive ? 'Regenerate with Direct Connection' : 'Regenerate'}*/}
|
||||
{/* </Button>*/}
|
||||
{/*)}*/}
|
||||
{/* Explanation */}
|
||||
<Box color='text.secondary' fontSize='sm' my={2}>
|
||||
<strong>Experimental:</strong> enable Direct Connection to {vendorName} to work around size limitations.
|
||||
</Box>
|
||||
) : (
|
||||
|
||||
{/* Toggle */}
|
||||
<Box
|
||||
sx={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
p: 2,
|
||||
borderRadius: 'sm',
|
||||
bgcolor: 'background.popup',
|
||||
boxShadow: 'md',
|
||||
}}
|
||||
>
|
||||
|
||||
<Box sx={{ flex: 1 }}>
|
||||
<Box color={!csfActive ? undefined : 'primary.solidBg'} fontWeight='lg' mb={0.5}>
|
||||
Direct Connection {csfActive && '- Now Try Again'}
|
||||
</Box>
|
||||
<FormHelperText>
|
||||
Connect directly from this client -> {vendorName || 'AI service'}
|
||||
</FormHelperText>
|
||||
</Box>
|
||||
|
||||
<Switch
|
||||
checked={csfActive}
|
||||
onChange={(e) => csfToggle(e.target.checked)}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
{/* Regenerate button */}
|
||||
{/*{props.onRegenerate && (*/}
|
||||
{/* <Button*/}
|
||||
{/* size='sm'*/}
|
||||
{/* variant={csfActive ? 'solid' : 'outlined'}*/}
|
||||
{/* color={csfActive ? 'success' : 'neutral'}*/}
|
||||
{/* startDecorator={<RefreshIcon />}*/}
|
||||
{/* onClick={props.onRegenerate}*/}
|
||||
{/* sx={{ alignSelf: 'flex-start' }}*/}
|
||||
{/* >*/}
|
||||
{/* {csfActive ? 'Regenerate with Direct Connection' : 'Regenerate'}*/}
|
||||
{/* </Button>*/}
|
||||
{/*)}*/}
|
||||
|
||||
</> : (
|
||||
<Box>
|
||||
<Typography level='body-sm' sx={{ mb: 1 }}>
|
||||
<strong>Suggestions:</strong>
|
||||
</Typography>
|
||||
<Typography component='ul' level='body-sm' sx={{ pl: 2, m: 0 }}>
|
||||
<Box sx={{ color: 'text.secondary', my: 1 }}>
|
||||
Suggestions:
|
||||
</Box>
|
||||
<Box component='ul' sx={{ color: 'text.secondary' }}>
|
||||
<li>Use the cleanup button in the right pane to hide old messages</li>
|
||||
<li>Remove large attachments from the conversation</li>
|
||||
{/*<li>Reduce conversation length before sending</li>*/}
|
||||
</Typography>
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { elevenLabsSpeakText } from '~/modules/elevenlabs/elevenlabs.client';
|
||||
import type { AixChatGenerateContent_DMessageGuts } from '~/modules/aix/client/aix.client';
|
||||
import { speakText } from '~/modules/speex/speex.client';
|
||||
|
||||
import { isTextContentFragment } from '~/common/stores/chat/chat.fragments';
|
||||
|
||||
import type { AixChatGenerateContent_DMessageGuts } from '~/modules/aix/client/aix.client';
|
||||
|
||||
import type { PersonaProcessorInterface } from '../chat-persona';
|
||||
|
||||
|
||||
@@ -58,7 +57,7 @@ export class PersonaChatMessageSpeak implements PersonaProcessorInterface {
|
||||
#speak(text: string) {
|
||||
console.log('📢 TTS:', text);
|
||||
this.spokenLine = true;
|
||||
// fire/forget: we don't want to stall this loop
|
||||
void elevenLabsSpeakText(text, undefined, false, true);
|
||||
// fire/forget: we don't want to stall streaming
|
||||
void speakText(text, undefined, { label: 'Chat message' });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,11 +72,13 @@ export const DevNewsItem: NewsItem = {
|
||||
// news and feature surfaces
|
||||
export const NewsItems: NewsItem[] = [
|
||||
{
|
||||
versionCode: '2.0.1',
|
||||
versionCode: '2.0.2',
|
||||
versionName: 'Heavy Critters',
|
||||
versionDate: new Date('2025-11-24T23:30:00Z'),
|
||||
versionDate: new Date('2025-12-01T06:00:00Z'), // 2.0.2
|
||||
// versionDate: new Date('2025-11-24T23:30:00Z'), // 2.0.1
|
||||
items: [
|
||||
{ text: <>New: <B>Opus 4.5</B>, <B>Gemini 3 Pro</B>, <B>Nano Banana Pro</B>, <B>Grok 4.1</B>, <B>GPT-5.1</B>, <B>Kimi K2</B></> },
|
||||
{ text: <><B>New in 2.0.2</B> Speech synthesis with Web Speech, LocalAI, OpenAI and more</> },
|
||||
{ text: <><B>Opus 4.5</B>, <B>Gemini 3 Pro</B>, <B>Nano Banana Pro</B>, <B>Grok 4.1</B>, <B>GPT-5.1</B>, <B>Kimi K2</B></> },
|
||||
{ text: <><B>Image Generation</B> with Azure and LocalAI providers, in addition to OpenAI</> },
|
||||
{ text: <>Enhanced <B>OpenRouter</B> integration with auto-capabilities and reasoning</> },
|
||||
{ text: <>Call transcripts, generate persona images, search button in beams</> },
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
import * as React from 'react';
|
||||
import { Accordion, AccordionDetails, accordionDetailsClasses, AccordionGroup, AccordionSummary, accordionSummaryClasses, Avatar, Box, Button, ListItemContent, styled, Tab, TabList, TabPanel, Tabs } from '@mui/joy';
|
||||
|
||||
import { Accordion, AccordionDetails, AccordionGroup, AccordionSummary, accordionSummaryClasses, Avatar, Box, Button, ListItemContent, styled, Tab, TabList, TabPanel, Tabs, Typography } from '@mui/joy';
|
||||
import AddIcon from '@mui/icons-material/Add';
|
||||
import AutoAwesomeIcon from '@mui/icons-material/AutoAwesome';
|
||||
import KeyboardCommandKeyOutlinedIcon from '@mui/icons-material/KeyboardCommandKeyOutlined';
|
||||
import LanguageRoundedIcon from '@mui/icons-material/LanguageRounded';
|
||||
import MicIcon from '@mui/icons-material/Mic';
|
||||
import RecordVoiceOverIcon from '@mui/icons-material/RecordVoiceOver';
|
||||
import ScienceIcon from '@mui/icons-material/Science';
|
||||
import SearchIcon from '@mui/icons-material/Search';
|
||||
import TerminalOutlinedIcon from '@mui/icons-material/TerminalOutlined';
|
||||
|
||||
import { BrowseSettings } from '~/modules/browse/BrowseSettings';
|
||||
import { DallESettings } from '~/modules/t2i/dalle/DallESettings';
|
||||
import { ElevenlabsSettings } from '~/modules/elevenlabs/ElevenlabsSettings';
|
||||
import { GoogleSearchSettings } from '~/modules/google/GoogleSearchSettings';
|
||||
import { T2ISettings } from '~/modules/t2i/T2ISettings';
|
||||
|
||||
@@ -20,14 +19,15 @@ import type { PreferencesTabId } from '~/common/layout/optima/store-layout-optim
|
||||
import { AppBreadcrumbs } from '~/common/components/AppBreadcrumbs';
|
||||
import { DarkModeToggleButton, darkModeToggleButtonSx } from '~/common/components/DarkModeToggleButton';
|
||||
import { GoodModal } from '~/common/components/modals/GoodModal';
|
||||
import { Is } from '~/common/util/pwaUtils';
|
||||
import { PhVoice } from '~/common/components/icons/phosphor/PhVoice';
|
||||
import { optimaActions } from '~/common/layout/optima/useOptima';
|
||||
import { useIsMobile } from '~/common/components/useMatchMedia';
|
||||
|
||||
import { AppChatSettingsAI } from './AppChatSettingsAI';
|
||||
import { AppChatSettingsUI } from './settings-ui/AppChatSettingsUI';
|
||||
import { UxLabsSettings } from './UxLabsSettings';
|
||||
import { VoiceSettings } from './VoiceSettings';
|
||||
import { VoiceInSettings } from './VoiceInSettings';
|
||||
import { VoiceOutSettings } from './VoiceOutSettings';
|
||||
|
||||
|
||||
// configuration
|
||||
@@ -271,10 +271,10 @@ export function SettingsModal(props: {
|
||||
<TabPanel value='voice' color='primary' variant='outlined' sx={_styles.tabPanel}>
|
||||
<Topics>
|
||||
<Topic icon={/*'🎙️'*/ <MicIcon />} title='Microphone'>
|
||||
<VoiceSettings />
|
||||
<VoiceInSettings isMobile={isMobile} />
|
||||
</Topic>
|
||||
<Topic icon={/*'📢'*/ <RecordVoiceOverIcon />} title='ElevenLabs API'>
|
||||
<ElevenlabsSettings />
|
||||
<Topic icon={/*'📢'*/ <PhVoice />} title={'Speech'/*<>Voices <GoodBadge badge='New' /></>*/}>
|
||||
<VoiceOutSettings isMobile={isMobile} />
|
||||
</Topic>
|
||||
</Topics>
|
||||
</TabPanel>
|
||||
@@ -291,6 +291,44 @@ export function SettingsModal(props: {
|
||||
</TabPanel>
|
||||
|
||||
<TabPanel value='tools' color='primary' variant='outlined' sx={_styles.tabPanel}>
|
||||
|
||||
{/* Search Modifier Info */}
|
||||
<Box sx={{
|
||||
p: 2,
|
||||
borderRadius: 'calc(var(--joy-radius-md) - 1px)',
|
||||
// backgroundColor: 'background.level1',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 2,
|
||||
}}>
|
||||
<Button
|
||||
variant='soft'
|
||||
color='success'
|
||||
startDecorator={<SearchIcon />}
|
||||
sx={{
|
||||
// this is copied frmo ButtonSearchControl._styles.desktop
|
||||
minWidth: 100,
|
||||
justifyContent: 'flex-start',
|
||||
borderRadius: '18px',
|
||||
pointerEvents: 'none',
|
||||
'[data-joy-color-scheme="light"] &': {
|
||||
bgcolor: '#d5ec31',
|
||||
},
|
||||
boxShadow: 'inset 0 2px 4px -1px rgba(0,0,0,0.15)',
|
||||
textWrap: 'nowrap',
|
||||
}}
|
||||
>
|
||||
Search
|
||||
</Button>
|
||||
<Box sx={{ flex: 1 }}>
|
||||
<Typography level='body-sm' sx={{ fontWeight: 'md', mb: 0.5 }}>
|
||||
Use the Search button
|
||||
</Typography>
|
||||
<Typography level='body-xs' sx={{ color: 'text.secondary' }}>
|
||||
Modern AI models have native search built-in. Click the Search button when chatting to enable real-time web search.
|
||||
</Typography>
|
||||
</Box>
|
||||
</Box>
|
||||
<Topics>
|
||||
<Topic icon={<LanguageRoundedIcon />} title='Load Web Pages (with images)' startCollapsed>
|
||||
<BrowseSettings />
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { FormControl } from '@mui/joy';
|
||||
|
||||
import { useChatMicTimeoutMs } from '../chat/store-app-chat';
|
||||
|
||||
import type { FormRadioOption } from '~/common/components/forms/FormRadioControl';
|
||||
import { FormChipControl } from '~/common/components/forms/FormChipControl';
|
||||
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
|
||||
import { LanguageSelect } from '~/common/components/LanguageSelect';
|
||||
|
||||
|
||||
const _minTimeouts: ReadonlyArray<FormRadioOption<string>> = [
|
||||
{ value: '600', label: '0.6s', description: 'Best for quick calls' },
|
||||
{ value: '2000', label: '2s', description: 'Standard' },
|
||||
{ value: '5000', label: '5s', description: 'Breathe' },
|
||||
{ value: '15000', label: '15s', description: 'Best for thinking' },
|
||||
] as const;
|
||||
|
||||
|
||||
export function VoiceInSettings(props: { isMobile: boolean }) {
|
||||
|
||||
// external state
|
||||
const [chatTimeoutMs, setChatTimeoutMs] = useChatMicTimeoutMs();
|
||||
|
||||
// derived - converts from string keys to numbers and vice versa
|
||||
const chatTimeoutValue: string = '' + chatTimeoutMs;
|
||||
const setChatTimeoutValue = React.useCallback((value: string) => {
|
||||
value && setChatTimeoutMs(parseInt(value));
|
||||
}, [setChatTimeoutMs]);
|
||||
|
||||
return <>
|
||||
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
<FormLabelStart
|
||||
title='Language'
|
||||
description='Mic and voice'
|
||||
// tooltip='For Microphone input and Voice output. Microphone support varies by browser (iPhone/Safari lacks speech input).'
|
||||
/>
|
||||
<LanguageSelect />
|
||||
</FormControl>
|
||||
|
||||
{!props.isMobile && (
|
||||
<FormChipControl
|
||||
title='Timeout'
|
||||
// color='primary'
|
||||
options={_minTimeouts}
|
||||
value={chatTimeoutValue}
|
||||
onChange={setChatTimeoutValue}
|
||||
/>
|
||||
)}
|
||||
|
||||
</>;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { SpeexConfigureEngines } from '~/modules/speex/components/SpeexConfigureEngines';
|
||||
import { useSpeexEngines } from '~/modules/speex/store-module-speex';
|
||||
|
||||
import { ChatAutoSpeakType, useChatAutoAI } from '../chat/store-app-chat';
|
||||
|
||||
import { FormRadioOption } from '~/common/components/forms/FormRadioControl';
|
||||
import { FormChipControl } from '~/common/components/forms/FormChipControl';
|
||||
|
||||
|
||||
const _autoSpeakOptions: FormRadioOption<ChatAutoSpeakType>[] = [
|
||||
{ value: 'off', label: 'No', description: 'Off' },
|
||||
{ value: 'firstLine', label: 'Start', description: 'First paragraph' },
|
||||
{ value: 'all', label: 'Full', description: 'Complete response' },
|
||||
] as const;
|
||||
|
||||
|
||||
/**
|
||||
* Voice output settings - Auto-speak mode and TTS engine configuration
|
||||
*/
|
||||
export function VoiceOutSettings(props: { isMobile: boolean }) {
|
||||
|
||||
// external state
|
||||
const { autoSpeak, setAutoSpeak } = useChatAutoAI();
|
||||
|
||||
// external state - module
|
||||
const hasEngines = useSpeexEngines().length > 0;
|
||||
|
||||
return <>
|
||||
|
||||
{/* Auto-speak setting */}
|
||||
<FormChipControl
|
||||
title='Speak Chats'
|
||||
size='md'
|
||||
// color='primary'
|
||||
tooltip={!hasEngines ? 'No voice engines available. Configure a TTS service or use system voice.' : undefined}
|
||||
disabled={!hasEngines}
|
||||
options={_autoSpeakOptions}
|
||||
value={autoSpeak}
|
||||
onChange={setAutoSpeak}
|
||||
/>
|
||||
|
||||
{/* Engine configuration */}
|
||||
<SpeexConfigureEngines isMobile={props.isMobile} />
|
||||
|
||||
</>;
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { FormControl } from '@mui/joy';
|
||||
|
||||
import { useChatMicTimeoutMs } from '../chat/store-app-chat';
|
||||
|
||||
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
|
||||
import { FormRadioControl } from '~/common/components/forms/FormRadioControl';
|
||||
import { LanguageSelect } from '~/common/components/LanguageSelect';
|
||||
import { useIsMobile } from '~/common/components/useMatchMedia';
|
||||
|
||||
|
||||
export function VoiceSettings() {
|
||||
|
||||
// external state
|
||||
const isMobile = useIsMobile();
|
||||
const [chatTimeoutMs, setChatTimeoutMs] = useChatMicTimeoutMs();
|
||||
|
||||
|
||||
// this converts from string keys to numbers and vice versa
|
||||
const chatTimeoutValue: string = '' + chatTimeoutMs;
|
||||
const setChatTimeoutValue = (value: string) => value && setChatTimeoutMs(parseInt(value));
|
||||
|
||||
return <>
|
||||
|
||||
{/* LanguageSelect: moved from the UI settings (where it logically belongs), just to group things better from an UX perspective */}
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
<FormLabelStart title='Language'
|
||||
description='ASR and TTS'
|
||||
tooltip='Currently for Microphone input and Voice output. Microphone support varies by browser (iPhone/Safari lacks speech input). We will use the ElevenLabs MultiLanguage model if a language other than English is selected.' />
|
||||
<LanguageSelect />
|
||||
</FormControl>
|
||||
|
||||
{!isMobile && <FormRadioControl
|
||||
title='Mic Timeout'
|
||||
description={chatTimeoutMs < 1000 ? 'Best for quick calls' : chatTimeoutMs > 5000 ? 'Best for thinking' : 'Standard'}
|
||||
options={[
|
||||
{ value: '600', label: '.6s' },
|
||||
{ value: '2000', label: '2s' },
|
||||
{ value: '5000', label: '5s' },
|
||||
{ value: '15000', label: '15s' },
|
||||
]}
|
||||
value={chatTimeoutValue} onChange={setChatTimeoutValue}
|
||||
/>}
|
||||
|
||||
</>;
|
||||
}
|
||||
@@ -23,8 +23,8 @@ export const Release = {
|
||||
|
||||
// this is here to trigger revalidation of data, e.g. models refresh
|
||||
Monotonics: {
|
||||
Aix: 42,
|
||||
NewsVersion: 201,
|
||||
Aix: 43,
|
||||
NewsVersion: 202,
|
||||
},
|
||||
|
||||
// Frontend: pretty features
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Chip, ChipProps } from '@mui/joy';
|
||||
|
||||
|
||||
/**
|
||||
* Simple badge/label component for inline status indicators like "New", "Beta", etc.
|
||||
*/
|
||||
export function GoodBadge(props: {
|
||||
badge: React.ReactNode;
|
||||
color?: ChipProps['color'];
|
||||
variant?: ChipProps['variant'];
|
||||
sx?: ChipProps['sx'];
|
||||
}) {
|
||||
return (
|
||||
<Chip
|
||||
size='sm'
|
||||
color={props.color ?? 'success'}
|
||||
variant={props.variant ?? 'soft'}
|
||||
sx={{
|
||||
ml: 1.5,
|
||||
fontSize: 'xs',
|
||||
fontWeight: 'md',
|
||||
borderRadius: 'xs',
|
||||
px: 1,
|
||||
py: 0.25,
|
||||
// default "new" color - lime/yellow-green
|
||||
...(props.color === undefined && {
|
||||
bgcolor: '#d5ec31',
|
||||
color: 'primary.softColor',
|
||||
}),
|
||||
...props.sx,
|
||||
}}
|
||||
>
|
||||
{props.badge}
|
||||
</Chip>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Option, Select } from '@mui/joy';
|
||||
import { Option, optionClasses, Select, SelectSlotsAndSlotProps } from '@mui/joy';
|
||||
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
|
||||
|
||||
import { useUIPreferencesStore } from '~/common/stores/store-ui';
|
||||
@@ -10,6 +10,20 @@ import { useUIPreferencesStore } from '~/common/stores/store-ui';
|
||||
import languages from './Languages.json';
|
||||
|
||||
|
||||
// copied from useLLMSelect.tsx - inspired by optimaSelectSlotProps.listbox
|
||||
const _selectSlotProps: SelectSlotsAndSlotProps<false>['slotProps'] = {
|
||||
root: { sx: { minWidth: 200 } },
|
||||
listbox: {
|
||||
sx: {
|
||||
boxShadow: 'xl',
|
||||
[`& .${optionClasses.root}`]: {
|
||||
maxWidth: 'min(640px, calc(100dvw - 0.25rem))',
|
||||
},
|
||||
},
|
||||
} as const,
|
||||
} as const;
|
||||
|
||||
|
||||
export function LanguageSelect() {
|
||||
// external state
|
||||
|
||||
@@ -32,19 +46,19 @@ export function LanguageSelect() {
|
||||
</Option>
|
||||
) : (
|
||||
Object.entries(localesOrCode).map(([country, code]) => (
|
||||
<Option key={code} value={code}>
|
||||
<Option key={code} value={code} label={language}>
|
||||
{`${language} (${country})`}
|
||||
</Option>
|
||||
))
|
||||
)), []);
|
||||
|
||||
return (
|
||||
<Select value={preferredLanguage} onChange={handleLanguageChanged}
|
||||
indicator={<KeyboardArrowDownIcon />}
|
||||
slotProps={{
|
||||
root: { sx: { minWidth: 200 } },
|
||||
indicator: { sx: { opacity: 0.5 } },
|
||||
}}>
|
||||
<Select
|
||||
value={preferredLanguage}
|
||||
onChange={handleLanguageChanged}
|
||||
indicator={<KeyboardArrowDownIcon />}
|
||||
slotProps={_selectSlotProps}
|
||||
>
|
||||
{languageOptions}
|
||||
</Select>
|
||||
);
|
||||
|
||||
@@ -21,6 +21,13 @@ const _styles = {
|
||||
gap: 1,
|
||||
} as const,
|
||||
|
||||
chipGroupEnd: {
|
||||
display: 'flex',
|
||||
flexWrap: 'wrap',
|
||||
justifyContent: 'flex-end',
|
||||
gap: 1,
|
||||
} as const,
|
||||
|
||||
chip: {
|
||||
'--Chip-minHeight': '1.75rem', // this makes it prob better
|
||||
px: 1.5,
|
||||
@@ -36,6 +43,7 @@ export const FormChipControl = <TValue extends string>(props: {
|
||||
// specific
|
||||
size?: 'sm' | 'md' | 'lg',
|
||||
color?: ColorPaletteProp,
|
||||
alignEnd?: boolean,
|
||||
// =FormRadioControl
|
||||
title: string | React.JSX.Element;
|
||||
description?: string | React.JSX.Element;
|
||||
@@ -48,6 +56,9 @@ export const FormChipControl = <TValue extends string>(props: {
|
||||
|
||||
const { onChange } = props;
|
||||
|
||||
const selectedOption = props.options.find(option => option.value === props.value);
|
||||
const description = selectedOption?.description ?? props.description;
|
||||
|
||||
const handleChipClick = React.useCallback((value: Immutable<TValue>) => {
|
||||
if (!props.disabled)
|
||||
onChange(value);
|
||||
@@ -55,8 +66,8 @@ export const FormChipControl = <TValue extends string>(props: {
|
||||
|
||||
return (
|
||||
<FormControl orientation='horizontal' disabled={props.disabled} sx={_styles.control}>
|
||||
{(!!props.title || !!props.description) && <FormLabelStart title={props.title} description={props.description} tooltip={props.tooltip} />}
|
||||
<Box sx={_styles.chipGroup}>
|
||||
{(!!props.title || !!description) && <FormLabelStart title={props.title} description={description} tooltip={props.tooltip} />}
|
||||
<Box sx={props.alignEnd ? _styles.chipGroupEnd : _styles.chipGroup}>
|
||||
{props.options.map((option) => (
|
||||
<Chip
|
||||
key={'opt-' + option.value}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import type { SxProps } from '@mui/joy/styles/types';
|
||||
import { FormControl, IconButton, Input } from '@mui/joy';
|
||||
import KeyIcon from '@mui/icons-material/Key';
|
||||
import VisibilityIcon from '@mui/icons-material/Visibility';
|
||||
import VisibilityOffIcon from '@mui/icons-material/VisibilityOff';
|
||||
|
||||
import { FormLabelStart } from './FormLabelStart';
|
||||
|
||||
|
||||
const _styles = {
|
||||
formControl: {
|
||||
flexWrap: 'wrap',
|
||||
justifyContent: 'space-between',
|
||||
alignItems: 'center',
|
||||
},
|
||||
inputDefault: {
|
||||
flexGrow: 1,
|
||||
},
|
||||
} as const satisfies Record<string, SxProps>;
|
||||
|
||||
|
||||
/**
|
||||
* Secret/API key form field with visibility toggle.
|
||||
* Same inline layout as FormTextField but with secret-specific features:
|
||||
* - Password masking with visibility toggle
|
||||
* - Key icon (customizable)
|
||||
* - Password manager integration
|
||||
*/
|
||||
export function FormSecretField(props: {
|
||||
autoCompleteId: string;
|
||||
title: string | React.JSX.Element;
|
||||
description?: string | React.JSX.Element;
|
||||
tooltip?: string | React.JSX.Element;
|
||||
placeholder?: string;
|
||||
value: string;
|
||||
onChange: (text: string) => void;
|
||||
// Behavior
|
||||
required?: boolean;
|
||||
disabled?: boolean;
|
||||
isError?: boolean;
|
||||
// Appearance
|
||||
inputSx?: SxProps;
|
||||
/** Custom start decorator, or false to hide. Default: KeyIcon */
|
||||
startDecorator?: React.ReactNode | false;
|
||||
}) {
|
||||
|
||||
// state
|
||||
const [isVisible, setIsVisible] = React.useState(false);
|
||||
|
||||
// derived
|
||||
const acId = 'secret-' + props.autoCompleteId;
|
||||
// password manager username
|
||||
const ghost = props.autoCompleteId.replace(/-key$/, '').replace(/-/g, ' ');
|
||||
|
||||
const endDecorator = React.useMemo(() => !!props.value && (
|
||||
<IconButton size='sm' onClick={() => setIsVisible(on => !on)}>
|
||||
{isVisible ? <VisibilityIcon sx={{ fontSize: 'md' }} /> : <VisibilityOffIcon sx={{ fontSize: 'md' }} />}
|
||||
</IconButton>
|
||||
), [props.value, isVisible]);
|
||||
|
||||
return (
|
||||
<FormControl
|
||||
id={acId}
|
||||
orientation='horizontal'
|
||||
disabled={props.disabled}
|
||||
sx={_styles.formControl}
|
||||
>
|
||||
<FormLabelStart title={props.title} description={props.description} tooltip={props.tooltip} />
|
||||
{/* Hidden username field for password manager association */}
|
||||
<input
|
||||
type='text'
|
||||
autoComplete='username'
|
||||
value={ghost}
|
||||
readOnly
|
||||
tabIndex={-1}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
<Input
|
||||
name={acId}
|
||||
type={isVisible ? 'text' : 'password'}
|
||||
autoComplete='new-password'
|
||||
variant='outlined'
|
||||
placeholder={props.required && !props.placeholder ? 'required' : props.placeholder}
|
||||
error={props.isError}
|
||||
value={props.value}
|
||||
onChange={event => props.onChange(event.target.value)}
|
||||
startDecorator={props.startDecorator ?? <KeyIcon sx={{ fontSize: 'md' }} />}
|
||||
endDecorator={endDecorator}
|
||||
sx={props.inputSx ?? _styles.inputDefault}
|
||||
/>
|
||||
</FormControl>
|
||||
);
|
||||
}
|
||||
@@ -24,6 +24,7 @@ export function FormSliderControl(props: {
|
||||
startAdornment?: React.ReactNode,
|
||||
endAdornment?: React.ReactNode,
|
||||
styleNoTrack?: boolean,
|
||||
sliderSx?: SxProps,
|
||||
}) {
|
||||
|
||||
|
||||
@@ -66,8 +67,7 @@ export function FormSliderControl(props: {
|
||||
onChange={handleChange}
|
||||
onChangeCommitted={handleChangeCommitted}
|
||||
valueLabelDisplay={props.valueLabelDisplay}
|
||||
sx={props.styleNoTrack ? _styleNoTrack : undefined}
|
||||
// sx={{ py: 1, mt: 1.1 }}
|
||||
sx={props.styleNoTrack ? _styleNoTrack : props.sliderSx}
|
||||
/>
|
||||
{props.endAdornment}
|
||||
</FormControl>
|
||||
|
||||
@@ -6,11 +6,16 @@ import { FormControl, Input } from '@mui/joy';
|
||||
import { FormLabelStart } from './FormLabelStart';
|
||||
|
||||
|
||||
const formControlSx: SxProps = {
|
||||
flexWrap: 'wrap',
|
||||
justifyContent: 'space-between',
|
||||
alignItems: 'center',
|
||||
};
|
||||
const _styles = {
|
||||
formControl: {
|
||||
flexWrap: 'wrap',
|
||||
justifyContent: 'space-between',
|
||||
alignItems: 'center',
|
||||
},
|
||||
inputDefault: {
|
||||
flexGrow: 1,
|
||||
},
|
||||
} as const satisfies Record<string, SxProps>;
|
||||
|
||||
|
||||
/**
|
||||
@@ -23,6 +28,7 @@ export function FormTextField(props: {
|
||||
tooltip?: string | React.JSX.Element,
|
||||
placeholder?: string, isError?: boolean, disabled?: boolean,
|
||||
value: string | undefined, onChange: (text: string) => void,
|
||||
inputSx?: SxProps,
|
||||
}) {
|
||||
const acId = 'text-' + props.autoCompleteId;
|
||||
return (
|
||||
@@ -30,7 +36,7 @@ export function FormTextField(props: {
|
||||
id={acId}
|
||||
orientation='horizontal'
|
||||
disabled={props.disabled}
|
||||
sx={formControlSx}
|
||||
sx={_styles.formControl}
|
||||
>
|
||||
<FormLabelStart title={props.title} description={props.description} tooltip={props.tooltip} />
|
||||
<Input
|
||||
@@ -39,7 +45,7 @@ export function FormTextField(props: {
|
||||
autoComplete='off'
|
||||
variant='outlined' placeholder={props.placeholder} error={props.isError}
|
||||
value={props.value} onChange={event => props.onChange(event.target.value)}
|
||||
sx={{ flexGrow: 1 }}
|
||||
sx={props.inputSx ?? _styles.inputDefault}
|
||||
/>
|
||||
</FormControl>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { SvgIcon, SvgIconProps } from '@mui/joy';
|
||||
|
||||
/*
|
||||
* Source: 'https://phosphoricons.com/' - user-sound
|
||||
*/
|
||||
export function PhVoice(props: SvgIconProps) {
|
||||
return (
|
||||
<SvgIcon viewBox='0 0 256 256' stroke='none' fill='currentColor' width='24' height='24' {...props}>
|
||||
<path d='M144,165.68a68,68,0,1,0-71.9,0c-20.65,6.76-39.23,19.39-54.17,37.17a8,8,0,0,0,12.25,10.3C50.25,189.19,77.91,176,108,176s57.75,13.19,77.88,37.15a8,8,0,1,0,12.25-10.3C183.18,185.07,164.6,172.44,144,165.68ZM56,108a52,52,0,1,1,52,52A52.06,52.06,0,0,1,56,108ZM207.36,65.6a108.36,108.36,0,0,1,0,84.8,8,8,0,0,1-7.36,4.86,8,8,0,0,1-7.36-11.15,92.26,92.26,0,0,0,0-72.22,8,8,0,0,1,14.72-6.29ZM248,108a139,139,0,0,1-11.29,55.15,8,8,0,0,1-14.7-6.3,124.43,124.43,0,0,0,0-97.7,8,8,0,1,1,14.7-6.3A139,139,0,0,1,248,108Z' />
|
||||
</SvgIcon>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { SvgIcon, SvgIconProps } from '@mui/joy';
|
||||
|
||||
export function ElevenLabsIcon(props: SvgIconProps) {
|
||||
return <SvgIcon viewBox='0 0 24 24' width='24' height='24' fill='currentColor' {...props}>
|
||||
<path d='M7 4h3v16H7V4zm7 0h3v16h-3V4z' />
|
||||
</SvgIcon>;
|
||||
}
|
||||
@@ -25,17 +25,6 @@ export interface CapabilityBrowserSpeechRecognition {
|
||||
export { browserSpeechRecognitionCapability as useCapabilityBrowserSpeechRecognition } from './speechrecognition/useSpeechRecognition';
|
||||
|
||||
|
||||
/// Speech Synthesis: ElevenLabs
|
||||
|
||||
export interface CapabilityElevenLabsSpeechSynthesis {
|
||||
mayWork: boolean;
|
||||
isConfiguredServerSide: boolean;
|
||||
isConfiguredClientSide: boolean;
|
||||
}
|
||||
|
||||
export { useCapability as useCapabilityElevenLabs } from '~/modules/elevenlabs/elevenlabs.client';
|
||||
|
||||
|
||||
/// Image Generation
|
||||
|
||||
export interface TextToImageProvider {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { Alert, IconButton } from '@mui/joy';
|
||||
import CloseRoundedIcon from '@mui/icons-material/CloseRounded';
|
||||
import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
|
||||
|
||||
import { Is, isBrowser, isPwa } from '~/common/util/pwaUtils';
|
||||
import { isBrowser, isPwa } from '~/common/util/pwaUtils';
|
||||
import { useUICounter } from '~/common/stores/store-ui';
|
||||
|
||||
|
||||
@@ -31,15 +31,14 @@ export function usePWADesktopModeWarning() {
|
||||
|
||||
// if PWA
|
||||
const isInPwaMode = isPwa();
|
||||
if (!isInPwaMode) return false;
|
||||
// if Mobile device (detected using touch points), while desktop have 0 touch points
|
||||
const isTouchDevice = navigator?.maxTouchPoints > 0;
|
||||
// if Physical Screen is small - typical mobile screen size - e.g. 412 for SGS24 Ultra
|
||||
const isSmallScreen = window.screen?.width < 600;
|
||||
// if Desktop mode - e.g. "Desktop Site" reports a large viewport width, typically 9xx+
|
||||
const isDesktopWidth = window.matchMedia('(min-width: 900px)').matches;
|
||||
|
||||
// if OS is mobile
|
||||
const isMobileOS = Is.OS.iOS || Is.OS.Android;
|
||||
if (!isMobileOS) return false;
|
||||
|
||||
// Check if viewport width suggests desktop mode (>= 900px)
|
||||
// This matches the mobile breakpoint used in useMatchMedia.ts
|
||||
return window.matchMedia('(min-width: 900px)').matches;
|
||||
return isInPwaMode && isTouchDevice && isSmallScreen && isDesktopWidth;
|
||||
}, []);
|
||||
|
||||
const showWarning = isInDesktopMode && !hideWarning && lessThanFive;
|
||||
|
||||
@@ -7,36 +7,38 @@ import type { DModelsService } from '../llms.service.types';
|
||||
import { llmsStoreActions, useModelsStore } from '../store-llms';
|
||||
|
||||
|
||||
const CSF_KEY = 'csf';
|
||||
|
||||
|
||||
/**
|
||||
* Hook to manage client-side fetch setting for a model's service
|
||||
* The CSF setting is stored as 'csf' in service settings for all vendors
|
||||
*/
|
||||
export function useModelServiceClientSideFetch(enabled: boolean, model: DLLM | null) {
|
||||
|
||||
// memo vendor
|
||||
const { vendor, csfKey } = React.useMemo(() => {
|
||||
if (!enabled) return { vendor: null, csfKey: '' };
|
||||
const vendor = findModelVendor(model?.vId);
|
||||
const csfKey = vendor?.csfKey || '';
|
||||
return { vendor, csfKey };
|
||||
const vendor = React.useMemo(() => {
|
||||
if (!enabled) return null;
|
||||
return findModelVendor(model?.vId);
|
||||
}, [enabled, model?.vId]);
|
||||
|
||||
// external state
|
||||
const service: null | DModelsService = useModelsStore(state => !model?.sId ? null : state.sources.find(s => s.id === model.sId) ?? null);
|
||||
|
||||
// actual state
|
||||
const csfAvailable: boolean | undefined = !!csfKey && vendor?.csfAvailable?.(service?.setup);
|
||||
const csfActive: boolean | undefined = csfAvailable && (service?.setup as any)?.[csfKey];
|
||||
const csfAvailable: boolean | undefined = !!vendor?.csfAvailable && vendor?.csfAvailable?.(service?.setup);
|
||||
const csfActive: boolean | undefined = csfAvailable && (service?.setup as any)?.[CSF_KEY];
|
||||
|
||||
const serviceId = service?.id || '';
|
||||
const csfToggle = React.useCallback((value: boolean) => {
|
||||
if (csfKey && serviceId)
|
||||
llmsStoreActions().updateServiceSettings(serviceId, { [csfKey]: value });
|
||||
}, [csfKey, serviceId]);
|
||||
if (serviceId)
|
||||
llmsStoreActions().updateServiceSettings(serviceId, { [CSF_KEY]: value });
|
||||
}, [serviceId]);
|
||||
|
||||
const csfReset = React.useCallback(() => {
|
||||
if (csfKey && serviceId)
|
||||
llmsStoreActions().updateServiceSettings(serviceId, { [csfKey]: false });
|
||||
}, [csfKey, serviceId]);
|
||||
if (serviceId)
|
||||
llmsStoreActions().updateServiceSettings(serviceId, { [CSF_KEY]: false });
|
||||
}, [serviceId]);
|
||||
|
||||
return { csfAvailable, csfActive, csfToggle, csfReset };
|
||||
return { csfAvailable, csfActive, csfToggle, csfReset, vendorName: vendor?.name || vendor?.id || 'AI Service' };
|
||||
}
|
||||
|
||||
@@ -19,6 +19,10 @@ export class AudioLivePlayer {
|
||||
this.audioElement.src = URL.createObjectURL(this.mediaSource);
|
||||
this.audioElement.autoplay = true;
|
||||
|
||||
// Suppress Android media notification by clearing media session metadata
|
||||
if ('mediaSession' in navigator)
|
||||
navigator.mediaSession.metadata = null;
|
||||
|
||||
// Connect the audio element to the audio context
|
||||
const sourceNode = this.audioContext.createMediaElementSource(this.audioElement);
|
||||
sourceNode.connect(this.audioContext.destination);
|
||||
@@ -60,14 +64,12 @@ export class AudioLivePlayer {
|
||||
private onSourceBufferUpdateEnd = () => {
|
||||
this.isSourceBufferUpdating = false;
|
||||
|
||||
// Continue appending if there's more data
|
||||
if (!this.isMediaSourceEnded) {
|
||||
// Always continue appending if there's more data in the queue
|
||||
if (this.chunkQueue.length > 0) {
|
||||
this.appendNextChunk();
|
||||
} else {
|
||||
// End the stream if all data has been appended
|
||||
if (this.sourceBuffer && !this.sourceBuffer.updating && this.chunkQueue.length === 0) {
|
||||
this.mediaSource.endOfStream();
|
||||
}
|
||||
} else if (this.isMediaSourceEnded) {
|
||||
// Only end the stream when queue is fully drained
|
||||
this._safeEndOfStream();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -86,9 +88,17 @@ export class AudioLivePlayer {
|
||||
}
|
||||
}
|
||||
} else if (this.isMediaSourceEnded) {
|
||||
if (this.sourceBuffer && !this.sourceBuffer.updating) {
|
||||
this.mediaSource.endOfStream();
|
||||
}
|
||||
if (this.sourceBuffer && !this.sourceBuffer.updating)
|
||||
this._safeEndOfStream();
|
||||
}
|
||||
}
|
||||
|
||||
private _safeEndOfStream() {
|
||||
if (this.mediaSource.readyState !== 'open') return;
|
||||
try {
|
||||
this.mediaSource.endOfStream();
|
||||
} catch (e) {
|
||||
// Ignore - MediaSource may have already ended
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,9 +116,8 @@ export class AudioLivePlayer {
|
||||
public endPlayback() {
|
||||
this.isMediaSourceEnded = true;
|
||||
// If the sourceBuffer is not updating, we can end the stream
|
||||
if (this.sourceBuffer && !this.sourceBuffer.updating && this.chunkQueue.length === 0) {
|
||||
this.mediaSource.endOfStream();
|
||||
}
|
||||
if (this.sourceBuffer && !this.sourceBuffer.updating && this.chunkQueue.length === 0)
|
||||
this._safeEndOfStream();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -119,14 +128,13 @@ export class AudioLivePlayer {
|
||||
this.chunkQueue = [];
|
||||
this.isMediaSourceEnded = true;
|
||||
|
||||
if (this.sourceBuffer) {
|
||||
// only abort SourceBuffer when MediaSource is 'open'
|
||||
if (this.sourceBuffer && this.mediaSource.readyState === 'open') {
|
||||
try {
|
||||
if (this.mediaSource.readyState === 'open') {
|
||||
this.mediaSource.endOfStream();
|
||||
}
|
||||
this.sourceBuffer.abort();
|
||||
this.mediaSource.endOfStream();
|
||||
} catch (e) {
|
||||
console.warn('Error stopping playback:', e);
|
||||
// Ignore - may race with natural stream end
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ export function agiCustomId(digits: number) {
|
||||
type UuidV4Scope =
|
||||
| 'conversation-2'
|
||||
| 'persona-2'
|
||||
| 'speex.engine.instance'
|
||||
;
|
||||
|
||||
|
||||
|
||||
@@ -28,3 +28,182 @@ export function countKeys(obj: object | null | undefined): number {
|
||||
for (const _ in obj) count++;
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip undefined fields from an object.
|
||||
*
|
||||
* Useful to prevent undefined becoming null over the wire (JSON serialization).
|
||||
*/
|
||||
export function stripUndefined<T extends object>(obj: T): T;
|
||||
export function stripUndefined<T extends object>(obj: T | null): T | null;
|
||||
export function stripUndefined<T extends object>(obj: T | null): T | null {
|
||||
if (!obj) return null;
|
||||
return Object.fromEntries(Object.entries(obj).filter(([_, v]) => v !== undefined)) as T;
|
||||
}
|
||||
|
||||
|
||||
// === Size Estimation ===
|
||||
|
||||
/**
|
||||
* Estimates JSON serialized size without actually stringifying.
|
||||
*
|
||||
* Avoids memory allocation spike on large objects. Useful for progress tracking,
|
||||
* batching decisions, or debug output sizing.
|
||||
*
|
||||
* Note: This is an ESTIMATE. It doesn't account for:
|
||||
* - UTF-8 multi-byte characters (assumes 1 byte per char)
|
||||
* - JSON escape sequences (\n, \t, unicode escapes)
|
||||
* - Floating point precision differences
|
||||
*
|
||||
* Returns 0 for the cyclic portion if circular references are detected.
|
||||
*/
|
||||
export function objectEstimateJsonSize(value: unknown, debugCaller: string): number {
|
||||
const seen = new WeakSet<object>();
|
||||
|
||||
function estimate(val: unknown): number {
|
||||
if (val === null) return 4; // "null"
|
||||
if (val === undefined) return 0; // omitted in JSON
|
||||
|
||||
switch (typeof val) {
|
||||
case 'string':
|
||||
return val.length + 2; // quotes
|
||||
case 'number':
|
||||
return String(val).length;
|
||||
case 'boolean':
|
||||
return val ? 4 : 5; // "true" or "false"
|
||||
case 'object': {
|
||||
// cycle detection
|
||||
if (seen.has(val as object)) {
|
||||
console.warn(`[estimateJsonSize (${debugCaller})] Circular reference detected, returning 0 for this branch`);
|
||||
return 0;
|
||||
}
|
||||
seen.add(val as object);
|
||||
|
||||
if (Array.isArray(val)) {
|
||||
let size = 2; // []
|
||||
for (let i = 0; i < val.length; i++) {
|
||||
size += estimate(val[i]);
|
||||
if (i < val.length - 1) size += 1; // comma
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
// plain object
|
||||
let size = 2; // {}
|
||||
const keys = Object.keys(val);
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i];
|
||||
size += key.length + 3; // "key":
|
||||
size += estimate((val as Record<string, unknown>)[key]);
|
||||
if (i < keys.length - 1) size += 1; // comma
|
||||
}
|
||||
return size;
|
||||
}
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return estimate(value);
|
||||
}
|
||||
|
||||
|
||||
// === Object Traversal ===
|
||||
|
||||
/**
|
||||
* Deep clones an object while truncating strings that exceed maxBytes.
|
||||
*
|
||||
* Useful for debug logging of large objects (e.g., requests with base64 images).
|
||||
* Truncates strings in the middle, preserving start and end with a byte count.
|
||||
*
|
||||
* @returns Deep clone with truncated strings, or "[Circular]" for cyclic refs
|
||||
*/
|
||||
export function objectDeepCloneWithStringLimit(value: unknown, debugCaller: string, maxBytes: number = 2048): unknown {
|
||||
const seen = new WeakSet<object>();
|
||||
|
||||
function clone(val: unknown): unknown {
|
||||
// handle primitives first
|
||||
if (val === null || val === undefined) return val;
|
||||
|
||||
// handle strings - truncate if too long
|
||||
if (typeof val === 'string') {
|
||||
if (val.length <= maxBytes) return val;
|
||||
const ellipsis = `...[${(val.length - maxBytes).toLocaleString()} bytes]...`;
|
||||
const half = Math.floor((maxBytes - ellipsis.length) / 2);
|
||||
return val.slice(0, half) + ellipsis + val.slice(-half);
|
||||
}
|
||||
|
||||
// handle other primitives
|
||||
if (typeof val !== 'object') return val;
|
||||
|
||||
// cycle detection
|
||||
if (seen.has(val)) return '[Circular]';
|
||||
seen.add(val);
|
||||
|
||||
// handle arrays - recurse
|
||||
if (Array.isArray(val))
|
||||
return val.map(item => clone(item));
|
||||
|
||||
// handle objects - recurse
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const key in val)
|
||||
if (Object.prototype.hasOwnProperty.call(val, key))
|
||||
result[key] = clone((val as Record<string, unknown>)[key]);
|
||||
return result;
|
||||
}
|
||||
|
||||
return clone(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the largest string values in an object tree
|
||||
*
|
||||
* Recursively traverses an object to find the top N largest string values,
|
||||
* returning their paths, lengths, and preview snippets.
|
||||
*
|
||||
* @returns Array of {path, length, preview} sorted by length (descending)
|
||||
*/
|
||||
export function objectFindLargestStringPaths(obj: unknown, debugCaller: string, topN: number = 5, maxDepth: number = 20): Array<{ path: string; length: number; preview: string }> {
|
||||
const results: Array<{ path: string; length: number; preview: string }> = [];
|
||||
const seen = new WeakSet<object>();
|
||||
|
||||
function traverse(current: unknown, path: string, depth: number) {
|
||||
// prevent infinite recursion
|
||||
if (depth > maxDepth) return;
|
||||
|
||||
// handle strings
|
||||
if (typeof current === 'string') {
|
||||
results.push({
|
||||
path,
|
||||
length: current.length,
|
||||
preview: current.substring(0, 100) + (current.length > 100 ? '...' : ''),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// handle non-objects
|
||||
if (current === null || typeof current !== 'object') return;
|
||||
|
||||
// cycle detection
|
||||
if (seen.has(current)) {
|
||||
console.warn(`[findLargestStringPaths (${debugCaller})] Circular reference at path: ${path}`);
|
||||
return;
|
||||
}
|
||||
seen.add(current);
|
||||
|
||||
// handle arrays
|
||||
if (Array.isArray(current))
|
||||
return current.forEach((item, index) => traverse(item, `${path}[${index}]`, depth + 1));
|
||||
|
||||
// handle objects
|
||||
for (const [key, value] of Object.entries(current))
|
||||
traverse(value, path ? `${path}.${key}` : key, depth + 1);
|
||||
}
|
||||
|
||||
traverse(obj, '', 0);
|
||||
|
||||
// sort by length descending and return top N
|
||||
return results
|
||||
.sort((a, b) => b.length - a.length)
|
||||
.slice(0, topN);
|
||||
}
|
||||
|
||||
@@ -604,6 +604,8 @@ export class ContentReassembler {
|
||||
this.accumulator.fragments.unshift(placeholderFragment); // Add to beginning
|
||||
|
||||
// Placeholders don't affect text fragment indexing
|
||||
// NOTE: we could have placeholders breaking text accumulation into new fragments with `this.currentTextFragmentIndex = null;`, however
|
||||
// since placeholders are used a lot with hosted tool calls, this could lead to way too many fragments being created
|
||||
}
|
||||
|
||||
private onSetVendorState(vs: Extract<AixWire_Particles.PartParticleOp, { p: 'svs' }>): void {
|
||||
@@ -619,7 +621,7 @@ export class ContentReassembler {
|
||||
lastFragment.vendorState = {
|
||||
...lastFragment.vendorState,
|
||||
[vendor]: state,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Helper to remove placeholder when real content arrives
|
||||
|
||||
@@ -36,13 +36,14 @@ export function aixClassifyStreamingError(error: any, isUserAbort: boolean, hasF
|
||||
|
||||
// Browser-level network connection drops (TypeError, happens below tRPC error wrapping layer)
|
||||
// Network errors - when the client is disconnected (Vercel 5min timeout, Mobile timeout / disconnect, etc) - they show up as TypeErrors
|
||||
// IMPORTANT: we will differentiate between the 2 'net-disconnected' cases in the UI, checking for the errorMessage '**network error**' vs '**connection terminated**'
|
||||
if (error instanceof TypeError && error.message === 'network error')
|
||||
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **network error**.' };
|
||||
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **network error**.' /* DO NOT CHANGE '**network error**' - usually client-side broken */ };
|
||||
|
||||
// tRPC <= 11.5.1 - Vercel Edge network disconnects are thrown form tRPC as 'Stream closed'
|
||||
// NOTE The behavior changed in 11.6+ for which we have an open upstream ticket: #6989
|
||||
if (error instanceof Error && error.message === 'Stream closed')
|
||||
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **connection terminated**.' };
|
||||
return { errorType: 'net-disconnected', errorMessage: 'An unexpected issue occurred: **connection terminated**.' /* DO NOT CHANGE '**connection terminated**' - usually server (Vercel) side broken */ };
|
||||
|
||||
|
||||
// tRPC-level protocol errors (wrapped by tRPC client)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { SERVER_DEBUG_WIRE } from '~/server/wire';
|
||||
import { serverSideId } from '~/server/trpc/trpc.nanoid';
|
||||
|
||||
import { objectDeepCloneWithStringLimit, objectEstimateJsonSize } from '~/common/util/objectUtils';
|
||||
|
||||
import type { AixWire_Particles } from '../../api/aix.wiretypes';
|
||||
|
||||
import type { IParticleTransmitter, ParticleServerLogLevel } from './parsers/IParticleTransmitter';
|
||||
@@ -25,70 +27,6 @@ export const IssueSymbols = {
|
||||
};
|
||||
|
||||
|
||||
/** Estimates JSON size without stringifying (avoids memory spike on large objects). */
|
||||
function _fastEstimateJsonSize(value: any): number {
|
||||
if (value === null) return 4; // "null"
|
||||
if (value === undefined) return 0; // omitted in JSON
|
||||
if (typeof value === 'string')
|
||||
return value.length + 2; // quotes
|
||||
if (typeof value === 'number')
|
||||
return String(value).length;
|
||||
if (typeof value === 'boolean')
|
||||
return value ? 4 : 5; // "true" or "false"
|
||||
if (Array.isArray(value)) {
|
||||
let size = 2; // []
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
size += _fastEstimateJsonSize(value[i]);
|
||||
if (i < value.length - 1) size += 1; // comma
|
||||
}
|
||||
return size;
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
let size = 2; // {}
|
||||
const keys = Object.keys(value);
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i];
|
||||
size += key.length + 3; // "key":
|
||||
size += _fastEstimateJsonSize(value[key]);
|
||||
if (i < keys.length - 1) size += 1; // comma
|
||||
}
|
||||
return size;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Deep-clones an object while ellipsizing any string exceeding maxBytes in the middle. */
|
||||
function _fastEllipsizeStringsInObject(value: any, maxBytes: number = DEBUG_REQUEST_MAX_STRING_BYTES): any {
|
||||
// handle primitives first
|
||||
if (value === null || value === undefined)
|
||||
return value;
|
||||
|
||||
// handle strings - ellipsize if too long
|
||||
if (typeof value === 'string') {
|
||||
if (value.length <= maxBytes)
|
||||
return value;
|
||||
const ellipsis = `...[${(value.length - maxBytes).toLocaleString()} bytes]...`;
|
||||
const half = Math.floor((maxBytes - ellipsis.length) / 2);
|
||||
return value.slice(0, half) + ellipsis + value.slice(-half);
|
||||
}
|
||||
|
||||
// handle other primitives (number, boolean)
|
||||
if (typeof value !== 'object')
|
||||
return value;
|
||||
|
||||
// handle arrays - recurse
|
||||
if (Array.isArray(value))
|
||||
return value.map(item => _fastEllipsizeStringsInObject(item, maxBytes));
|
||||
|
||||
// handle objects - recurse
|
||||
const result: any = {};
|
||||
for (const key in value)
|
||||
if (value.hasOwnProperty(key))
|
||||
result[key] = _fastEllipsizeStringsInObject(value[key], maxBytes);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Queues up and emits small messages (particles) to the client, for the purpose of a stateful
|
||||
* full reconstruction of the AixWire_Parts[] objects.
|
||||
@@ -194,7 +132,7 @@ export class ChatGenerateTransmitter implements IParticleTransmitter {
|
||||
|
||||
addDebugRequest(hideSensitiveData: boolean, url: string, headers: HeadersInit, body?: object) {
|
||||
// Ellipsize individual strings in the body object (e.g., base64 images) to reduce debug packet size
|
||||
const ellipsizedBody = body ? _fastEllipsizeStringsInObject(body) : undefined;
|
||||
const ellipsizedBody = body ? objectDeepCloneWithStringLimit(body, 'aix.addDebugRequest', DEBUG_REQUEST_MAX_STRING_BYTES) : undefined;
|
||||
const processedBody = ellipsizedBody ? JSON.stringify(ellipsizedBody, null, 2) : '';
|
||||
|
||||
this.transmissionQueue.push({
|
||||
@@ -204,7 +142,7 @@ export class ChatGenerateTransmitter implements IParticleTransmitter {
|
||||
url: url,
|
||||
headers: hideSensitiveData ? '(hidden sensitive data)' : JSON.stringify(headers, null, 2),
|
||||
body: processedBody,
|
||||
bodySize: body ? _fastEstimateJsonSize(body) : 0,
|
||||
bodySize: body ? objectEstimateJsonSize(body, 'aix.addDebugRequest') : 0,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
|
||||
let timeToFirstEvent: number;
|
||||
let messageStartTime: number | undefined = undefined;
|
||||
let chatInTokens: number | undefined = undefined;
|
||||
let needsTextSeparator = false; // insert text separator when text follows server tool
|
||||
|
||||
return function(pt: IParticleTransmitter, eventData: string, eventName?: string, context?: { retriesAvailable: boolean }): void {
|
||||
|
||||
@@ -155,7 +156,9 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
|
||||
|
||||
switch (content_block.type) { // .content_block_start.type
|
||||
case 'text':
|
||||
pt.appendText(content_block.text);
|
||||
// add separator when text follows server tool execution
|
||||
pt.appendText(!needsTextSeparator ? content_block.text : '\n\n' + content_block.text);
|
||||
needsTextSeparator = false;
|
||||
// Note: In streaming mode, citations arrive via citations_delta events, not on content_block_start
|
||||
break;
|
||||
|
||||
@@ -379,6 +382,11 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
|
||||
const _exhaustiveCheck: never = content_block;
|
||||
throw new Error(`Unexpected content block type: ${(content_block as any).type}`);
|
||||
}
|
||||
|
||||
// set separator flag when server tools complete (text after tools needs visual separation)
|
||||
if (content_block.type.includes('tool_use') || content_block.type.includes('tool_result'))
|
||||
needsTextSeparator = true;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -576,6 +584,7 @@ export function createAnthropicMessageParser(): ChatGenerateParseFunction {
|
||||
|
||||
export function createAnthropicMessageParserNS(): ChatGenerateParseFunction {
|
||||
const parserCreationTimestamp = Date.now();
|
||||
let needsTextSeparator = false; // insert text separator when text follows server tool
|
||||
|
||||
return function(pt: IParticleTransmitter, fullData: string /*, eventName?: string, context?: { retriesAvailable: boolean } */): void {
|
||||
|
||||
@@ -597,7 +606,9 @@ export function createAnthropicMessageParserNS(): ChatGenerateParseFunction {
|
||||
const isLastBlock = i === content.length - 1;
|
||||
switch (contentBlock.type) { // .content_block (non-streaming)
|
||||
case 'text':
|
||||
pt.appendText(contentBlock.text);
|
||||
// add separator when text follows server tool execution
|
||||
pt.appendText(!needsTextSeparator ? contentBlock.text : '\n\n' + contentBlock.text);
|
||||
needsTextSeparator = false;
|
||||
// Handle citations if present (non-streaming mode has all citations attached)
|
||||
if (contentBlock.citations && Array.isArray(contentBlock.citations)) {
|
||||
for (const citation of contentBlock.citations) {
|
||||
@@ -824,6 +835,10 @@ export function createAnthropicMessageParserNS(): ChatGenerateParseFunction {
|
||||
const _exhaustiveCheck: never = contentBlock;
|
||||
throw new Error(`Unexpected content block type: ${(contentBlock as any).type}`);
|
||||
}
|
||||
|
||||
// set separator flag when server tools complete (text after tools needs visual separation)
|
||||
if (contentBlock.type.includes('tool_use') || contentBlock.type.includes('tool_result'))
|
||||
needsTextSeparator = true;
|
||||
}
|
||||
|
||||
// -> Token Stop Reason
|
||||
|
||||
@@ -586,6 +586,11 @@ export namespace AnthropicWire_Tools {
|
||||
required: z.array(z.string()).optional(), // 2025-02-24: seems to be removed; we may still have this, but it may also be within the 'properties' object
|
||||
}),
|
||||
|
||||
/**
|
||||
* [Anthropic, 2025-11-13] Structured Outputs - guarantees tool inputs to match `input_schema` exactly.
|
||||
*/
|
||||
strict: z.boolean().optional(),
|
||||
|
||||
/**
|
||||
* [Anthropic, 2025-11-24] Tool Search Tool - when true, this tool is not loaded into context initially and can be discovered via the tool search tool when needed.
|
||||
*/
|
||||
|
||||
@@ -143,13 +143,31 @@ async function workerPuppeteer(
|
||||
};
|
||||
|
||||
// [puppeteer] start the remote session
|
||||
const browser: Browser = await puppeteer.connect({
|
||||
browserWSEndpoint,
|
||||
// Add default options for better stability
|
||||
// defaultViewport: { width: 1024, height: 768 },
|
||||
// acceptInsecureCerts: true,
|
||||
protocolTimeout: WORKER_TIMEOUT + 2000, // 2s extra for taking the screenshot?
|
||||
});
|
||||
let browser: Browser;
|
||||
try {
|
||||
browser = await puppeteer.connect({
|
||||
browserWSEndpoint,
|
||||
// Add default options for better stability
|
||||
// defaultViewport: { width: 1024, height: 768 },
|
||||
// acceptInsecureCerts: true,
|
||||
protocolTimeout: WORKER_TIMEOUT + 2000, // 2s extra for taking the screenshot?
|
||||
});
|
||||
} catch (connectError: any) {
|
||||
// Transform connection errors into user-friendly messages
|
||||
const errorMessage = connectError?.message || '';
|
||||
if (errorMessage.includes('403'))
|
||||
throw new Error('Browse service authentication failed (403). Please check your browser endpoint credentials.');
|
||||
if (errorMessage.includes('401'))
|
||||
throw new Error('Browse service unauthorized (401). Invalid credentials for the browser endpoint.');
|
||||
if (errorMessage.includes('429'))
|
||||
throw new Error('Browse service rate limited (429). Too many requests, please try again later.');
|
||||
if (errorMessage.includes('502') || errorMessage.includes('503') || errorMessage.includes('504'))
|
||||
throw new Error('Browse service temporarily unavailable. Please try again later.');
|
||||
if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('ENOTFOUND'))
|
||||
throw new Error('Browse service unreachable. The browser endpoint is not accessible.');
|
||||
// Re-throw with a cleaner message for other connection errors
|
||||
throw new Error(`Browse service connection failed: ${errorMessage || 'Unknown error'}`);
|
||||
}
|
||||
|
||||
// for local testing, open an incognito context, to separate cookies
|
||||
let incognitoContext: BrowserContext | null = null;
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { FormControl } from '@mui/joy';
|
||||
|
||||
import { useChatAutoAI } from '../../apps/chat/store-app-chat';
|
||||
|
||||
import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
|
||||
import { FormRadioControl } from '~/common/components/forms/FormRadioControl';
|
||||
import { useCapabilityElevenLabs } from '~/common/components/useCapabilities';
|
||||
|
||||
import { isElevenLabsEnabled } from './elevenlabs.client';
|
||||
import { useElevenLabsVoiceDropdown, useElevenLabsVoices } from './useElevenLabsVoiceDropdown';
|
||||
import { useElevenLabsApiKey } from './store-module-elevenlabs';
|
||||
|
||||
|
||||
export function ElevenlabsSettings() {
|
||||
|
||||
// state
|
||||
const [apiKey, setApiKey] = useElevenLabsApiKey();
|
||||
|
||||
// external state
|
||||
const { autoSpeak, setAutoSpeak } = useChatAutoAI();
|
||||
const { hasVoices } = useElevenLabsVoices();
|
||||
const { isConfiguredServerSide } = useCapabilityElevenLabs();
|
||||
const { voicesDropdown } = useElevenLabsVoiceDropdown(true);
|
||||
|
||||
|
||||
// derived state
|
||||
const isValidKey = isElevenLabsEnabled(apiKey);
|
||||
|
||||
|
||||
return <>
|
||||
|
||||
{/*<FormHelperText>*/}
|
||||
{/* 📢 Hear AI responses, even in your own voice*/}
|
||||
{/*</FormHelperText>*/}
|
||||
|
||||
<FormRadioControl
|
||||
title='Speak Responses'
|
||||
description={autoSpeak === 'off' ? 'Off' : 'First paragraph'}
|
||||
tooltip={!hasVoices ? 'No voices available, please configure a voice synthesis service' : undefined}
|
||||
disabled={!hasVoices}
|
||||
options={[
|
||||
{ value: 'off', label: 'Off' },
|
||||
{ value: 'firstLine', label: 'Start' },
|
||||
{ value: 'all', label: 'Full' },
|
||||
]}
|
||||
value={autoSpeak} onChange={setAutoSpeak}
|
||||
/>
|
||||
|
||||
|
||||
{!isConfiguredServerSide && <FormInputKey
|
||||
autoCompleteId='elevenlabs-key' label='ElevenLabs API Key'
|
||||
rightLabel={<AlreadySet required={!isConfiguredServerSide} />}
|
||||
value={apiKey} onChange={setApiKey}
|
||||
required={!isConfiguredServerSide} isError={!isValidKey}
|
||||
/>}
|
||||
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
<FormLabelStart title='Assistant Voice' />
|
||||
{voicesDropdown}
|
||||
</FormControl>
|
||||
|
||||
</>;
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
import { getBackendCapabilities } from '~/modules/backend/store-backend-capabilities';
|
||||
|
||||
import { AudioLivePlayer } from '~/common/util/audio/AudioLivePlayer';
|
||||
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
|
||||
import { CapabilityElevenLabsSpeechSynthesis } from '~/common/components/useCapabilities';
|
||||
import { apiStream } from '~/common/util/trpc.client';
|
||||
import { convert_Base64_To_UInt8Array } from '~/common/util/blobUtils';
|
||||
import { useUIPreferencesStore } from '~/common/stores/store-ui';
|
||||
|
||||
import { getElevenLabsData, useElevenLabsData } from './store-module-elevenlabs';
|
||||
|
||||
|
||||
export const isValidElevenLabsApiKey = (apiKey?: string) => !!apiKey && apiKey.trim()?.length >= 32;
|
||||
|
||||
export const isElevenLabsEnabled = (apiKey?: string) =>
|
||||
apiKey ? isValidElevenLabsApiKey(apiKey)
|
||||
: getBackendCapabilities().hasVoiceElevenLabs;
|
||||
|
||||
|
||||
export function useCapability(): CapabilityElevenLabsSpeechSynthesis {
|
||||
const [clientApiKey, voiceId] = useElevenLabsData();
|
||||
const isConfiguredServerSide = getBackendCapabilities().hasVoiceElevenLabs;
|
||||
const isConfiguredClientSide = clientApiKey ? isValidElevenLabsApiKey(clientApiKey) : false;
|
||||
const mayWork = isConfiguredServerSide || isConfiguredClientSide || !!voiceId;
|
||||
return { mayWork, isConfiguredServerSide, isConfiguredClientSide };
|
||||
}
|
||||
|
||||
|
||||
interface ElevenLabsSpeakResult {
|
||||
success: boolean;
|
||||
audioBase64?: string; // Available when not streaming
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Speaks text using ElevenLabs TTS
|
||||
* @returns Object with success status and optionally the audio base64 (when not streaming)
|
||||
*/
|
||||
export async function elevenLabsSpeakText(text: string, voiceId: string | undefined, audioStreaming: boolean, audioTurbo: boolean): Promise<ElevenLabsSpeakResult> {
|
||||
// Early validation
|
||||
if (!(text?.trim())) {
|
||||
// console.log('ElevenLabs: No text to speak');
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
const { elevenLabsApiKey, elevenLabsVoiceId } = getElevenLabsData();
|
||||
if (!isElevenLabsEnabled(elevenLabsApiKey)) {
|
||||
// console.warn('ElevenLabs: Service not enabled or configured');
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
const { preferredLanguage } = useUIPreferencesStore.getState();
|
||||
const nonEnglish = !(preferredLanguage?.toLowerCase()?.startsWith('en'));
|
||||
|
||||
// audio live player instance, if needed
|
||||
let liveAudioPlayer: AudioLivePlayer | undefined;
|
||||
let playbackStarted = false;
|
||||
let audioBase64: string | undefined;
|
||||
|
||||
try {
|
||||
|
||||
const stream = await apiStream.elevenlabs.speech.mutate({
|
||||
xiKey: elevenLabsApiKey,
|
||||
voiceId: voiceId || elevenLabsVoiceId,
|
||||
text: text,
|
||||
nonEnglish,
|
||||
audioStreaming,
|
||||
audioTurbo,
|
||||
});
|
||||
|
||||
for await (const piece of stream) {
|
||||
|
||||
// ElevenLabs stream buffer
|
||||
if (piece.audioChunk) {
|
||||
try {
|
||||
// create the live audio player as needed
|
||||
// NOTE: in the future we can have a centralized audio playing system
|
||||
if (!liveAudioPlayer)
|
||||
liveAudioPlayer = new AudioLivePlayer();
|
||||
|
||||
// enqueue a decoded audio chunk - this will throw on malformed base64 data
|
||||
const chunkArray = convert_Base64_To_UInt8Array(piece.audioChunk.base64, 'elevenLabsSpeakText (chunk)');
|
||||
liveAudioPlayer.enqueueChunk(chunkArray.buffer);
|
||||
playbackStarted = true;
|
||||
} catch (audioError) {
|
||||
console.error('ElevenLabs audio chunk error:', audioError);
|
||||
return { success: false };
|
||||
}
|
||||
}
|
||||
|
||||
// ElevenLabs full audio buffer
|
||||
else if (piece.audio) {
|
||||
try {
|
||||
// return base64 for potential reuse
|
||||
if (!audioStreaming)
|
||||
audioBase64 = piece.audio.base64;
|
||||
|
||||
// also consider merging LiveAudioPlayer into AudioPlayer - note this will throw on malformed base64 data
|
||||
const audioArray = convert_Base64_To_UInt8Array(piece.audio.base64, 'elevenLabsSpeakText');
|
||||
void AudioPlayer.playBuffer(audioArray.buffer); // fire/forget - it's a single piece of audio (could be long tho)
|
||||
playbackStarted = true;
|
||||
} catch (audioError) {
|
||||
console.error('ElevenLabs audio buffer error:', audioError);
|
||||
return { success: false };
|
||||
}
|
||||
}
|
||||
|
||||
// Errors
|
||||
else if (piece.errorMessage) {
|
||||
console.error('ElevenLabs error:', piece.errorMessage);
|
||||
return { success: false };
|
||||
} else if (piece.warningMessage) {
|
||||
console.warn('ElevenLabs warning:', piece.warningMessage);
|
||||
// Continue processing warnings
|
||||
} else if (piece.control === 'start' || piece.control === 'end') {
|
||||
// Control messages - continue processing
|
||||
} else {
|
||||
console.log('ElevenLabs unknown piece:', piece);
|
||||
}
|
||||
}
|
||||
return { success: playbackStarted, audioBase64 };
|
||||
} catch (error) {
|
||||
console.error('ElevenLabs playback error:', error);
|
||||
return { success: false };
|
||||
}
|
||||
}
|
||||
@@ -1,277 +0,0 @@
|
||||
import * as z from 'zod/v4';
|
||||
|
||||
import { createTRPCRouter, publicProcedure } from '~/server/trpc/trpc.server';
|
||||
import { env } from '~/server/env.server';
|
||||
import { fetchJsonOrTRPCThrow, fetchResponseOrTRPCThrow } from '~/server/trpc/trpc.router.fetchers';
|
||||
|
||||
|
||||
// configuration
|
||||
const SAFETY_TEXT_LENGTH = 1000;
|
||||
const MIN_CHUNK_SIZE = 4096; // Minimum chunk size in bytes
|
||||
|
||||
|
||||
// Schema definitions
|
||||
export type SpeechInputSchema = z.infer<typeof speechInputSchema>;
|
||||
export const speechInputSchema = z.object({
|
||||
xiKey: z.string().optional(),
|
||||
voiceId: z.string().optional(),
|
||||
text: z.string(),
|
||||
nonEnglish: z.boolean(),
|
||||
audioStreaming: z.boolean(),
|
||||
audioTurbo: z.boolean(),
|
||||
});
|
||||
|
||||
export type VoiceSchema = z.infer<typeof voiceSchema>;
|
||||
const voiceSchema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
description: z.string().nullable(),
|
||||
previewUrl: z.string().nullable(),
|
||||
category: z.string(),
|
||||
default: z.boolean(),
|
||||
});
|
||||
|
||||
|
||||
export const elevenlabsRouter = createTRPCRouter({
|
||||
|
||||
/**
|
||||
* List Voices available to this API key
|
||||
*/
|
||||
listVoices: publicProcedure
|
||||
.input(z.object({
|
||||
elevenKey: z.string().optional(),
|
||||
}))
|
||||
.output(z.object({
|
||||
voices: z.array(voiceSchema),
|
||||
}))
|
||||
.query(async ({ input }) => {
|
||||
|
||||
const { elevenKey } = input;
|
||||
const { headers, url } = elevenlabsAccess(elevenKey, '/v1/voices');
|
||||
|
||||
const voicesList = await fetchJsonOrTRPCThrow<ElevenlabsWire.VoicesList>({
|
||||
url,
|
||||
headers,
|
||||
name: 'ElevenLabs',
|
||||
});
|
||||
|
||||
// bring category != 'premade' to the top
|
||||
voicesList.voices.sort((a, b) => {
|
||||
if (a.category === 'premade' && b.category !== 'premade') return 1;
|
||||
if (a.category !== 'premade' && b.category === 'premade') return -1;
|
||||
return 0;
|
||||
});
|
||||
|
||||
return {
|
||||
voices: voicesList.voices.map((voice, idx) => ({
|
||||
id: voice.voice_id,
|
||||
name: voice.name,
|
||||
description: voice.description,
|
||||
previewUrl: voice.preview_url,
|
||||
category: voice.category,
|
||||
default: idx === 0,
|
||||
})),
|
||||
};
|
||||
|
||||
}),
|
||||
|
||||
/**
|
||||
* Speech synthesis procedure using tRPC streaming
|
||||
*/
|
||||
speech: publicProcedure
|
||||
.input(speechInputSchema)
|
||||
.mutation(async function* ({ input: { xiKey, text, voiceId, nonEnglish, audioStreaming, audioTurbo }, ctx }) {
|
||||
|
||||
// start streaming back
|
||||
yield { control: 'start' };
|
||||
|
||||
// Safety check: trim text that's too long
|
||||
if (text.length > SAFETY_TEXT_LENGTH) {
|
||||
text = text.slice(0, SAFETY_TEXT_LENGTH);
|
||||
yield { warningMessage: 'text was truncated to maximum length' };
|
||||
}
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
|
||||
// Prepare the upstream request
|
||||
const path = `/v1/text-to-speech/${elevenlabsVoiceId(voiceId)}${audioStreaming ? '/stream' : ''}`;
|
||||
const { headers, url } = elevenlabsAccess(xiKey, path);
|
||||
const body: ElevenlabsWire.TTSRequest = {
|
||||
text: text,
|
||||
model_id:
|
||||
audioTurbo ? 'eleven_turbo_v2_5'
|
||||
: nonEnglish ? 'eleven_multilingual_v2'
|
||||
: 'eleven_multilingual_v2', // even for english, use the latest multilingual model
|
||||
};
|
||||
|
||||
// Blocking fetch
|
||||
response = await fetchResponseOrTRPCThrow({ url, method: 'POST', headers, body, signal: ctx.reqSignal, name: 'ElevenLabs' });
|
||||
|
||||
} catch (error: any) {
|
||||
yield { errorMessage: `fetch issue: ${error.message || 'Unknown error'}` };
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse headers
|
||||
const responseHeaders = _safeParseTTSResponseHeaders(response.headers);
|
||||
|
||||
// If not streaming, return the entire audio
|
||||
if (!audioStreaming) {
|
||||
const audioArrayBuffer = await response.arrayBuffer();
|
||||
yield {
|
||||
audio: {
|
||||
base64: Buffer.from(audioArrayBuffer).toString('base64'),
|
||||
contentType: responseHeaders.contentType,
|
||||
characterCost: responseHeaders.characterCost,
|
||||
ttsLatencyMs: responseHeaders.ttsLatencyMs,
|
||||
},
|
||||
};
|
||||
yield { control: 'end' };
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
yield { errorMessage: 'stream issue: No reader' };
|
||||
return;
|
||||
}
|
||||
|
||||
// STREAM the audio chunks back to the client
|
||||
try {
|
||||
|
||||
// Initialize a buffer to accumulate chunks
|
||||
const accumulatedChunks: Uint8Array[] = [];
|
||||
let accumulatedSize = 0;
|
||||
|
||||
// Read loop
|
||||
while (true) {
|
||||
const { value, done: readerDone } = await reader.read();
|
||||
if (readerDone) break;
|
||||
if (!value) continue;
|
||||
|
||||
// Accumulate chunks
|
||||
accumulatedChunks.push(value);
|
||||
accumulatedSize += value.length;
|
||||
|
||||
// When accumulated size reaches or exceeds MIN_CHUNK_SIZE, yield the chunk
|
||||
if (accumulatedSize >= MIN_CHUNK_SIZE) {
|
||||
yield {
|
||||
audioChunk: {
|
||||
base64: Buffer.concat(accumulatedChunks).toString('base64'),
|
||||
},
|
||||
};
|
||||
// Reset the accumulation
|
||||
accumulatedChunks.length = 0;
|
||||
accumulatedSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// If there's any remaining data, yield it as well
|
||||
if (accumulatedSize) {
|
||||
yield {
|
||||
audioChunk: {
|
||||
base64: Buffer.concat(accumulatedChunks).toString('base64'),
|
||||
},
|
||||
};
|
||||
}
|
||||
} catch (error: any) {
|
||||
yield { errorMessage: `stream issue: ${error.message || 'Unknown error'}` };
|
||||
return;
|
||||
}
|
||||
|
||||
// end streaming (if a control error wasn't thrown)
|
||||
yield { control: 'end' };
|
||||
}),
|
||||
|
||||
});
|
||||
|
||||
/**
|
||||
* Helper function to construct ElevenLabs API access details
|
||||
*/
|
||||
export function elevenlabsAccess(elevenKey: string | undefined, apiPath: string): { headers: HeadersInit; url: string } {
|
||||
// API key
|
||||
elevenKey = (elevenKey || env.ELEVENLABS_API_KEY || '').trim();
|
||||
if (!elevenKey)
|
||||
throw new Error('Missing ElevenLabs API key.');
|
||||
|
||||
// API host
|
||||
let host = (env.ELEVENLABS_API_HOST || 'api.elevenlabs.io').trim();
|
||||
if (!host.startsWith('http'))
|
||||
host = `https://${host}`;
|
||||
if (host.endsWith('/') && apiPath.startsWith('/'))
|
||||
host = host.slice(0, -1);
|
||||
|
||||
return {
|
||||
headers: {
|
||||
'Accept': 'audio/mpeg',
|
||||
'Content-Type': 'application/json',
|
||||
'xi-api-key': elevenKey,
|
||||
},
|
||||
url: host + apiPath,
|
||||
};
|
||||
}
|
||||
|
||||
export function elevenlabsVoiceId(voiceId?: string): string {
|
||||
return voiceId?.trim() || env.ELEVENLABS_VOICE_ID || '21m00Tcm4TlvDq8ikWAM';
|
||||
}
|
||||
|
||||
|
||||
function _safeParseTTSResponseHeaders(headers: Headers): ElevenlabsWire.TTSResponseHeaders {
|
||||
return {
|
||||
contentType: headers.get('content-type') || 'audio/mpeg',
|
||||
characterCost: parseInt(headers.get('character-cost') || '0'),
|
||||
currentConcurrentRequests: parseInt(headers.get('current-concurrent-requests') || '0'),
|
||||
maximumConcurrentRequests: parseInt(headers.get('maximum-concurrent-requests') || '0'),
|
||||
ttsLatencyMs: parseInt(headers.get('tts-latency-ms') || '0'),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/// This is the upstream API [rev-eng on 2023-04-12]
|
||||
export namespace ElevenlabsWire {
|
||||
export interface TTSRequest {
|
||||
text: string;
|
||||
model_id?:
|
||||
| 'eleven_monolingual_v1'
|
||||
| 'eleven_multilingual_v1'
|
||||
| 'eleven_multilingual_v2'
|
||||
| 'eleven_turbo_v2'
|
||||
| 'eleven_turbo_v2_5';
|
||||
voice_settings?: {
|
||||
stability: number;
|
||||
similarity_boost: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TTSResponseHeaders {
|
||||
// Response metadata
|
||||
contentType: string; // Should be 'audio/mpeg'
|
||||
|
||||
// Cost and usage metrics
|
||||
characterCost: number; // Cost in characters for this generation
|
||||
currentConcurrentRequests: number; // Current number of concurrent requests
|
||||
maximumConcurrentRequests: number; // Maximum allowed concurrent requests
|
||||
ttsLatencyMs?: number; // Time taken to generate speech (not in streaming mode)
|
||||
}
|
||||
|
||||
export interface VoicesList {
|
||||
voices: Voice[];
|
||||
}
|
||||
|
||||
interface Voice {
|
||||
voice_id: string;
|
||||
name: string;
|
||||
//samples: Sample[];
|
||||
category: string;
|
||||
// fine_tuning: FineTuning;
|
||||
labels: Record<string, string>;
|
||||
description: string;
|
||||
preview_url: string;
|
||||
// available_for_tiers: string[];
|
||||
settings: {
|
||||
stability: number;
|
||||
similarity_boost: number;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
import { create } from 'zustand';
|
||||
import { persist } from 'zustand/middleware';
|
||||
import { useShallow } from 'zustand/react/shallow';
|
||||
|
||||
|
||||
interface ModuleElevenlabsStore {
|
||||
|
||||
// ElevenLabs Text to Speech settings
|
||||
|
||||
elevenLabsApiKey: string;
|
||||
setElevenLabsApiKey: (apiKey: string) => void;
|
||||
|
||||
elevenLabsVoiceId: string;
|
||||
setElevenLabsVoiceId: (voiceId: string) => void;
|
||||
|
||||
}
|
||||
|
||||
const useElevenlabsStore = create<ModuleElevenlabsStore>()(
|
||||
persist(
|
||||
(set) => ({
|
||||
|
||||
// ElevenLabs Text to Speech settings
|
||||
|
||||
elevenLabsApiKey: '',
|
||||
setElevenLabsApiKey: (elevenLabsApiKey: string) => set({ elevenLabsApiKey }),
|
||||
|
||||
elevenLabsVoiceId: '',
|
||||
setElevenLabsVoiceId: (elevenLabsVoiceId: string) => set({ elevenLabsVoiceId }),
|
||||
|
||||
}),
|
||||
{
|
||||
name: 'app-module-elevenlabs',
|
||||
}),
|
||||
);
|
||||
|
||||
export const useElevenLabsApiKey = (): [string, (apiKey: string) => void] => {
|
||||
const apiKey = useElevenlabsStore(state => state.elevenLabsApiKey);
|
||||
return [apiKey, useElevenlabsStore.getState().setElevenLabsApiKey];
|
||||
};
|
||||
|
||||
export const useElevenLabsVoiceId = (): [string, (voiceId: string) => void] => {
|
||||
const voiceId = useElevenlabsStore(state => state.elevenLabsVoiceId);
|
||||
return [voiceId, useElevenlabsStore.getState().setElevenLabsVoiceId];
|
||||
};
|
||||
|
||||
export const useElevenLabsData = (): [string, string] =>
|
||||
useElevenlabsStore(useShallow(state => [state.elevenLabsApiKey, state.elevenLabsVoiceId]));
|
||||
|
||||
export const getElevenLabsData = (): { elevenLabsApiKey: string, elevenLabsVoiceId: string } =>
|
||||
useElevenlabsStore.getState();
|
||||
@@ -1,102 +0,0 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { CircularProgress, Option, Select } from '@mui/joy';
|
||||
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
|
||||
import RecordVoiceOverTwoToneIcon from '@mui/icons-material/RecordVoiceOverTwoTone';
|
||||
|
||||
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
|
||||
import { apiQuery } from '~/common/util/trpc.client';
|
||||
|
||||
import { VoiceSchema } from './elevenlabs.router';
|
||||
import { isElevenLabsEnabled } from './elevenlabs.client';
|
||||
import { useElevenLabsApiKey, useElevenLabsVoiceId } from './store-module-elevenlabs';
|
||||
|
||||
|
||||
function VoicesDropdown(props: {
|
||||
isValidKey: boolean,
|
||||
isFetchingVoices: boolean,
|
||||
isErrorVoices: boolean,
|
||||
disabled?: boolean,
|
||||
voices: VoiceSchema[],
|
||||
voiceId: string | null,
|
||||
setVoiceId: (voiceId: string) => void,
|
||||
}) {
|
||||
|
||||
const handleVoiceChange = (_event: any, value: string | null) => props.setVoiceId(value || '');
|
||||
|
||||
return (
|
||||
<Select
|
||||
value={props.voiceId} onChange={handleVoiceChange}
|
||||
variant='outlined' disabled={props.disabled || !props.voices.length}
|
||||
// color={props.isErrorVoices ? 'danger' : undefined}
|
||||
placeholder={props.isErrorVoices ? 'Issue loading voices' : props.isValidKey ? 'Select a voice' : 'Missing API Key'}
|
||||
startDecorator={<RecordVoiceOverTwoToneIcon />}
|
||||
endDecorator={props.isValidKey && props.isFetchingVoices && <CircularProgress size='sm' />}
|
||||
indicator={<KeyboardArrowDownIcon />}
|
||||
slotProps={{
|
||||
root: { sx: { width: '100%' } },
|
||||
indicator: { sx: { opacity: 0.5 } },
|
||||
}}
|
||||
>
|
||||
{props.voices.map(voice => (
|
||||
<Option key={voice.id} value={voice.id}>
|
||||
{voice.name}
|
||||
</Option>
|
||||
))}
|
||||
</Select>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
export function useElevenLabsVoices() {
|
||||
const [apiKey] = useElevenLabsApiKey();
|
||||
|
||||
const isConfigured = isElevenLabsEnabled(apiKey);
|
||||
|
||||
const { data, isError, isFetching, isPending } = apiQuery.elevenlabs.listVoices.useQuery({ elevenKey: apiKey }, {
|
||||
enabled: isConfigured,
|
||||
staleTime: 1000 * 60 * 5, // 5 minutes
|
||||
});
|
||||
|
||||
return {
|
||||
isConfigured,
|
||||
isError,
|
||||
isFetching,
|
||||
hasVoices: !isPending && !!data?.voices.length,
|
||||
voices: data?.voices || [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
export function useElevenLabsVoiceDropdown(autoSpeak: boolean, disabled?: boolean) {
|
||||
|
||||
// external state
|
||||
const { isConfigured, isError, isFetching, hasVoices, voices } = useElevenLabsVoices();
|
||||
const [voiceId, setVoiceId] = useElevenLabsVoiceId();
|
||||
|
||||
// derived state
|
||||
const voice: VoiceSchema | undefined = voices.find(voice => voice.id === voiceId);
|
||||
|
||||
// [E] autoSpeak
|
||||
const previewUrl = (autoSpeak && voice?.previewUrl) || null;
|
||||
React.useEffect(() => {
|
||||
if (previewUrl)
|
||||
void AudioPlayer.playUrl(previewUrl);
|
||||
}, [previewUrl]);
|
||||
|
||||
const voicesDropdown = React.useMemo(() =>
|
||||
<VoicesDropdown
|
||||
isValidKey={isConfigured} isFetchingVoices={isFetching} isErrorVoices={isError} disabled={disabled}
|
||||
voices={voices}
|
||||
voiceId={voiceId} setVoiceId={setVoiceId}
|
||||
/>,
|
||||
[disabled, isConfigured, isError, isFetching, setVoiceId, voiceId, voices],
|
||||
);
|
||||
|
||||
return {
|
||||
hasVoices,
|
||||
voiceId,
|
||||
voiceName: voice?.name,
|
||||
voicesDropdown,
|
||||
};
|
||||
}
|
||||
+1
-2
@@ -20,8 +20,7 @@ export interface IModelVendor<TServiceSettings extends Record<string, any> = {},
|
||||
readonly hasServerConfigKey?: keyof BackendCapabilities;
|
||||
|
||||
/// client-side-fetch ///
|
||||
readonly csfKey?: string; // was keyof TServiceSettings, but caused TS troubles
|
||||
readonly csfAvailable?: (setup?: Partial<TServiceSettings>) => boolean; // undefined: not even, false: conditions not met
|
||||
readonly csfAvailable?: (setup?: Partial<TServiceSettings>) => boolean; // undefined: not supported, false: conditions not met
|
||||
|
||||
/// abstraction interface ///
|
||||
|
||||
|
||||
+11
-2
@@ -9,6 +9,7 @@ import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { FormTextField } from '~/common/components/forms/FormTextField';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
@@ -36,10 +37,11 @@ export function AlibabaServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorAlibaba);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: alibabaOaiKey, oaiHost: alibabaOaiHost } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: alibabaOaiKey, oaiHost: alibabaOaiHost } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const shallFetchSucceed = !needsUserKey || (!!alibabaOaiKey && serviceSetupValid);
|
||||
const showKeyError = !!alibabaOaiKey && !serviceSetupValid;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// fetch models
|
||||
const { isFetching, refetch, isError, error } =
|
||||
@@ -73,7 +75,7 @@ export function AlibabaServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{/* See the <ExternalLink href={ALIBABA_REG_LINK}>Alibaba Cloud Model Studio</ExternalLink> for more information.*/}
|
||||
{/*</Typography>*/}
|
||||
|
||||
{advanced.on && <FormTextField
|
||||
{showAdvanced && <FormTextField
|
||||
autoCompleteId='alibaba-host'
|
||||
title='API Endpoint'
|
||||
tooltip={`The API endpoint for the Alibaba Cloud OpenAI service, to be used instead of the default endpoint.`}
|
||||
@@ -82,6 +84,13 @@ export function AlibabaServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
onChange={text => updateSettings({ alibabaOaiHost: text })}
|
||||
/>}
|
||||
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!alibabaOaiKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Alibaba Cloud API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
interface DAlibabaServiceSettings {
|
||||
alibabaOaiKey: string;
|
||||
alibabaOaiHost: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -17,6 +18,9 @@ export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAcc
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmAlibaba',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfAlibabaAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
alibabaOaiKey: '',
|
||||
@@ -27,6 +31,7 @@ export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAcc
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'alibaba',
|
||||
clientSideFetch: _csfAlibabaAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.alibabaOaiKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.alibabaOaiHost || '',
|
||||
@@ -37,3 +42,7 @@ export const ModelVendorAlibaba: IModelVendor<DAlibabaServiceSettings, OpenAIAcc
|
||||
// OpenAI transport ('alibaba' dialect in 'access')
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
};
|
||||
|
||||
function _csfAlibabaAvailable(s?: Partial<DAlibabaServiceSettings>) {
|
||||
return !!s?.alibabaOaiKey;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import { useChatAutoAI } from '../../../../apps/chat/store-app-chat';
|
||||
|
||||
import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types';
|
||||
import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { ExternalLink } from '~/common/components/ExternalLink';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
|
||||
import { FormSwitchControl } from '~/common/components/forms/FormSwitchControl';
|
||||
@@ -110,7 +109,7 @@ export function AnthropicServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!anthropicKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ anthropicCSF: on })}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText="Fetch models and make requests directly to Anthropic's API using your browser instead of through the server. Useful for bypassing server limitations or ensuring requests use your API key directly."
|
||||
/>}
|
||||
|
||||
|
||||
+2
-3
@@ -10,7 +10,7 @@ export const isValidAnthropicApiKey = (apiKey?: string) => !!apiKey && (apiKey.s
|
||||
interface DAnthropicServiceSettings {
|
||||
anthropicKey: string;
|
||||
anthropicHost: string;
|
||||
anthropicCSF?: boolean;
|
||||
csf?: boolean;
|
||||
heliconeKey: string;
|
||||
}
|
||||
|
||||
@@ -25,13 +25,12 @@ export const ModelVendorAnthropic: IModelVendor<DAnthropicServiceSettings, Anthr
|
||||
hasServerConfigKey: 'hasLlmAnthropic',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfKey: 'anthropicCSF',
|
||||
csfAvailable: _csfAnthropicAvailable,
|
||||
|
||||
// functions
|
||||
getTransportAccess: (partialSetup): AnthropicAccessSchema => ({
|
||||
dialect: 'anthropic',
|
||||
clientSideFetch: _csfAnthropicAvailable(partialSetup) && !!partialSetup?.anthropicCSF,
|
||||
clientSideFetch: _csfAnthropicAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
anthropicKey: partialSetup?.anthropicKey || '',
|
||||
anthropicHost: partialSetup?.anthropicHost || null,
|
||||
heliconeKey: partialSetup?.heliconeKey || null,
|
||||
|
||||
+13
-2
@@ -9,8 +9,10 @@ import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { FormTextField } from '~/common/components/forms/FormTextField';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { asValidURL } from '~/common/util/urlUtils';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { useLlmUpdateModels } from '../../llm.client.hooks';
|
||||
@@ -22,6 +24,7 @@ import { isValidAzureApiKey, ModelVendorAzure } from './azure.vendor';
|
||||
export function AzureServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
const [checkboxExpanded, setCheckboxExpanded] = React.useState(false);
|
||||
|
||||
// external state
|
||||
@@ -29,8 +32,9 @@ export function AzureServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
useServiceSetup(props.serviceId, ModelVendorAzure);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: azureKey, oaiHost: azureEndpoint } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: azureKey, oaiHost: azureEndpoint } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
const keyValid = isValidAzureApiKey(azureKey);
|
||||
const keyError = (/*needsUserKey ||*/ !!azureKey) && !keyValid;
|
||||
@@ -81,7 +85,14 @@ export function AzureServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
placeholder='...'
|
||||
/>
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!(azureKey && azureEndpoint)}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Azure OpenAI API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
+10
-1
@@ -10,6 +10,7 @@ export const isValidAzureApiKey = (apiKey?: string) => !!apiKey && apiKey.length
|
||||
interface DAzureServiceSettings {
|
||||
azureEndpoint: string;
|
||||
azureKey: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
/** Implementation Notes for the Azure Vendor
|
||||
@@ -37,9 +38,13 @@ export const ModelVendorAzure: IModelVendor<DAzureServiceSettings, OpenAIAccessS
|
||||
instanceLimit: 2,
|
||||
hasServerConfigKey: 'hasLlmAzureOpenAI',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfAzureAvailable,
|
||||
|
||||
// functions
|
||||
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
|
||||
dialect: 'azure',
|
||||
clientSideFetch: _csfAzureAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.azureKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.azureEndpoint || '',
|
||||
@@ -50,4 +55,8 @@ export const ModelVendorAzure: IModelVendor<DAzureServiceSettings, OpenAIAccessS
|
||||
// OpenAI transport ('azure' dialect in 'access')
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
function _csfAzureAvailable(s?: Partial<DAzureServiceSettings>) {
|
||||
return !!(s?.azureKey && s?.azureEndpoint);
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
@@ -30,8 +31,9 @@ export function DeepseekAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorDeepseek);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: deepseekKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: deepseekKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// validate if url is a well formed proper url with zod
|
||||
const shallFetchSucceed = !needsUserKey || (!!deepseekKey && serviceSetupValid);
|
||||
@@ -57,6 +59,13 @@ export function DeepseekAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
placeholder='...'
|
||||
/>
|
||||
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!deepseekKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Deepseek API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
export interface DDeepseekServiceSettings {
|
||||
deepseekKey: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -17,6 +18,9 @@ export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIA
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmDeepseek',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfDeepseekAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
deepseekKey: '',
|
||||
@@ -26,6 +30,7 @@ export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIA
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'deepseek',
|
||||
clientSideFetch: _csfDeepseekAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.deepseekKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: '',
|
||||
@@ -37,3 +42,7 @@ export const ModelVendorDeepseek: IModelVendor<DDeepseekServiceSettings, OpenAIA
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfDeepseekAvailable(s?: Partial<DDeepseekServiceSettings>) {
|
||||
return !!s?.deepseekKey;
|
||||
}
|
||||
|
||||
+1
-1
@@ -109,7 +109,7 @@ export function GeminiServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!geminiKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ geminiCSF: on })}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText="Fetch models and make requests directly to Google's Gemini API using your browser instead of through the server."
|
||||
/>}
|
||||
|
||||
|
||||
+2
-3
@@ -9,7 +9,7 @@ import type { IModelVendor } from '../IModelVendor';
|
||||
interface DGeminiServiceSettings {
|
||||
geminiKey: string;
|
||||
geminiHost: string;
|
||||
geminiCSF?: boolean;
|
||||
csf?: boolean;
|
||||
minSafetyLevel: GeminiWire_Safety.HarmBlockThreshold;
|
||||
}
|
||||
|
||||
@@ -34,7 +34,6 @@ export const ModelVendorGemini: IModelVendor<DGeminiServiceSettings, GeminiAcces
|
||||
hasServerConfigKey: 'hasLlmGemini',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfKey: 'geminiCSF',
|
||||
csfAvailable: _csfGeminiAvailable,
|
||||
|
||||
// functions
|
||||
@@ -48,7 +47,7 @@ export const ModelVendorGemini: IModelVendor<DGeminiServiceSettings, GeminiAcces
|
||||
},
|
||||
getTransportAccess: (partialSetup): GeminiAccessSchema => ({
|
||||
dialect: 'gemini',
|
||||
clientSideFetch: _csfGeminiAvailable(partialSetup) && !!partialSetup?.geminiCSF,
|
||||
clientSideFetch: _csfGeminiAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
geminiKey: partialSetup?.geminiKey || '',
|
||||
geminiHost: partialSetup?.geminiHost || '',
|
||||
minSafetyLevel: partialSetup?.minSafetyLevel || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
|
||||
|
||||
+15
-4
@@ -1,13 +1,13 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Typography } from '@mui/joy';
|
||||
|
||||
import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types';
|
||||
import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { ModelVendorGroq } from './groq.vendor';
|
||||
@@ -20,6 +20,9 @@ const GROQ_REG_LINK = 'https://console.groq.com/keys';
|
||||
|
||||
export function GroqServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const {
|
||||
service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs,
|
||||
@@ -27,8 +30,9 @@ export function GroqServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorGroq);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: groqKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: groqKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// key validation
|
||||
const shallFetchSucceed = !needsUserKey || (!!groqKey && serviceSetupValid);
|
||||
@@ -54,7 +58,14 @@ export function GroqServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
placeholder='...'
|
||||
/>
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!groqKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Groq API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
interface DGroqServiceSettings {
|
||||
groqKey: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -17,6 +18,9 @@ export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSch
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmGroq',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfGroqAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
groqKey: '',
|
||||
@@ -26,6 +30,7 @@ export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSch
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'groq',
|
||||
clientSideFetch: _csfGroqAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.groqKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: '',
|
||||
@@ -37,3 +42,7 @@ export const ModelVendorGroq: IModelVendor<DGroqServiceSettings, OpenAIAccessSch
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfGroqAvailable(s?: Partial<DGroqServiceSettings>) {
|
||||
return !!s?.groqKey;
|
||||
}
|
||||
|
||||
+15
-2
@@ -9,8 +9,10 @@ import { ExpanderAccordion } from '~/common/components/ExpanderAccordion';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { VideoPlayerYouTube } from '~/common/components/VideoPlayerYouTube';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { useLlmUpdateModels } from '../../llm.client.hooks';
|
||||
import { useServiceSetup } from '../useServiceSetup';
|
||||
@@ -20,12 +22,16 @@ import { ModelVendorLMStudio } from './lmstudio.vendor';
|
||||
|
||||
export function LMStudioServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const { service, serviceAccess, updateSettings } =
|
||||
useServiceSetup(props.serviceId, ModelVendorLMStudio);
|
||||
|
||||
// derived state
|
||||
const { oaiHost } = serviceAccess;
|
||||
const { clientSideFetch, oaiHost } = serviceAccess;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// validate if url is a well formed proper url with zod
|
||||
const urlSchema = z.url().startsWith('http');
|
||||
@@ -62,7 +68,14 @@ export function LMStudioServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
value={oaiHost} onChange={value => updateSettings({ oaiHost: value })}
|
||||
/>
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!oaiHost}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to LM Studio from your browser. Requires CORS to be enabled in LM Studio.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
interface DLMStudioServiceSettings {
|
||||
oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path)
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorLMStudio: IModelVendor<DLMStudioServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -16,12 +17,16 @@ export const ModelVendorLMStudio: IModelVendor<DLMStudioServiceSettings, OpenAIA
|
||||
location: 'local',
|
||||
instanceLimit: 1,
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfLMStudioAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
oaiHost: 'http://localhost:1234',
|
||||
}),
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'lmstudio',
|
||||
clientSideFetch: _csfLMStudioAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.oaiHost || '',
|
||||
@@ -33,3 +38,7 @@ export const ModelVendorLMStudio: IModelVendor<DLMStudioServiceSettings, OpenAIA
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfLMStudioAvailable(s?: Partial<DLMStudioServiceSettings>) {
|
||||
return !!s?.oaiHost;
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ export function LocalAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
<SetupFormClientSideToggle
|
||||
visible={true}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ localAICSF: on })}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText="Fetch models and make requests directly from your LocalAI instance using the browser. Recommended for local setups."
|
||||
/>
|
||||
|
||||
|
||||
+4
-5
@@ -4,10 +4,10 @@ import type { OpenAIAccessSchema } from '../../server/openai/openai.access';
|
||||
import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
|
||||
interface DLocalAIServiceSettings {
|
||||
export interface DLocalAIServiceSettings {
|
||||
localAIHost: string; // use OpenAI-compatible non-default hosts (full origin path)
|
||||
localAIKey: string; // use OpenAI-compatible API keys
|
||||
localAICSF?: boolean;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorLocalAI: IModelVendor<DLocalAIServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -24,18 +24,17 @@ export const ModelVendorLocalAI: IModelVendor<DLocalAIServiceSettings, OpenAIAcc
|
||||
},
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfKey: 'localAICSF',
|
||||
csfAvailable: _csfLocalAIAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
localAIHost: '',
|
||||
localAIKey: '',
|
||||
// localAICSF: true, // eventually, but requires CORS support on the server: -e CORS=true -e CORS_ALLOW_ORIGINS="*"
|
||||
// csf: true, // eventually, but requires CORS support on the server: -e CORS=true -e CORS_ALLOW_ORIGINS="*"
|
||||
}),
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'localai',
|
||||
clientSideFetch: _csfLocalAIAvailable(partialSetup) && !!partialSetup?.localAICSF,
|
||||
clientSideFetch: _csfLocalAIAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.localAIKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.localAIHost || '',
|
||||
|
||||
+15
-2
@@ -7,7 +7,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { useLlmUpdateModels } from '../../llm.client.hooks';
|
||||
@@ -21,13 +23,17 @@ const MISTRAL_REG_LINK = 'https://console.mistral.ai/';
|
||||
|
||||
export function MistralServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, serviceSetupValid, updateSettings } =
|
||||
useServiceSetup(props.serviceId, ModelVendorMistral);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: mistralKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: mistralKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
const shallFetchSucceed = !needsUserKey || (!!mistralKey && serviceSetupValid);
|
||||
const showKeyError = !!mistralKey && !serviceSetupValid;
|
||||
@@ -56,7 +62,14 @@ export function MistralServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{/* Note the elegance of the numbers, representing the Year and Month or release (YYMM).*/}
|
||||
{/*</Typography>*/}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!mistralKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Mistral API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
+10
-2
@@ -6,7 +6,7 @@ import { DOpenAIServiceSettings, ModelVendorOpenAI } from '../openai/openai.vend
|
||||
|
||||
// special symbols
|
||||
|
||||
type DMistralServiceSettings = Pick<DOpenAIServiceSettings, 'oaiKey' | 'oaiHost'>;
|
||||
type DMistralServiceSettings = Pick<DOpenAIServiceSettings, 'oaiKey' | 'oaiHost' | 'csf'>;
|
||||
|
||||
|
||||
/** Implementation Notes for the Mistral vendor
|
||||
@@ -20,6 +20,9 @@ export const ModelVendorMistral: IModelVendor<DMistralServiceSettings, OpenAIAcc
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmMistral',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfMistralAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
oaiHost: 'https://api.mistral.ai/',
|
||||
@@ -30,6 +33,7 @@ export const ModelVendorMistral: IModelVendor<DMistralServiceSettings, OpenAIAcc
|
||||
},
|
||||
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
|
||||
dialect: 'mistral',
|
||||
clientSideFetch: _csfMistralAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.oaiKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.oaiHost || '',
|
||||
@@ -40,4 +44,8 @@ export const ModelVendorMistral: IModelVendor<DMistralServiceSettings, OpenAIAcc
|
||||
// OpenAI transport ('mistral' dialect in 'access')
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
function _csfMistralAvailable(s?: Partial<DMistralServiceSettings>) {
|
||||
return !!s?.oaiKey;
|
||||
}
|
||||
+15
-2
@@ -5,7 +5,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { ModelVendorMoonshot } from './moonshot.vendor';
|
||||
@@ -18,6 +20,9 @@ const MOONSHOT_API_LINK = 'https://platform.moonshot.ai/console/api-keys';
|
||||
|
||||
export function MoonshotServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const {
|
||||
service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs,
|
||||
@@ -25,8 +30,9 @@ export function MoonshotServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorMoonshot);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: moonshotKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: moonshotKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// key validation
|
||||
const shallFetchSucceed = !needsUserKey || (!!moonshotKey && serviceSetupValid);
|
||||
@@ -52,7 +58,14 @@ export function MoonshotServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
placeholder='...'
|
||||
/>
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!moonshotKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Moonshot API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
interface DMoonshotServiceSettings {
|
||||
moonshotKey: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -17,6 +18,9 @@ export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIA
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmMoonshot',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfMoonshotAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
moonshotKey: '',
|
||||
@@ -26,6 +30,7 @@ export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIA
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'moonshot',
|
||||
clientSideFetch: _csfMoonshotAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.moonshotKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: '',
|
||||
@@ -37,3 +42,7 @@ export const ModelVendorMoonshot: IModelVendor<DMoonshotServiceSettings, OpenAIA
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfMoonshotAvailable(s?: Partial<DMoonshotServiceSettings>) {
|
||||
return !!s?.moonshotKey;
|
||||
}
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ export function OllamaServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
<SetupFormClientSideToggle
|
||||
visible={true}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ ollamaCSF: on })}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText="Fetch models and make requests directly from your local Ollama instance using the browser. Recommended for local setups."
|
||||
/>
|
||||
|
||||
|
||||
+3
-4
@@ -6,7 +6,7 @@ import type { OllamaAccessSchema } from '../../server/ollama/ollama.access';
|
||||
|
||||
interface DOllamaServiceSettings {
|
||||
ollamaHost: string;
|
||||
ollamaCSF?: boolean;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,17 +20,16 @@ export const ModelVendorOllama: IModelVendor<DOllamaServiceSettings, OllamaAcces
|
||||
hasServerConfigKey: 'hasLlmOllama',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfKey: 'ollamaCSF',
|
||||
csfAvailable: _csfOllamaAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
ollamaHost: '',
|
||||
// ollamaCSF: true, // eventually
|
||||
// csf: true, // eventually
|
||||
}),
|
||||
getTransportAccess: (partialSetup): OllamaAccessSchema => ({
|
||||
dialect: 'ollama',
|
||||
clientSideFetch: _csfOllamaAvailable(partialSetup) && !!partialSetup?.ollamaCSF,
|
||||
clientSideFetch: _csfOllamaAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
ollamaHost: partialSetup?.ollamaHost || '',
|
||||
}),
|
||||
|
||||
|
||||
+1
-1
@@ -110,7 +110,7 @@ export function OpenAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!oaiHost || !!oaiKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ oaiCSF: on })}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText="Fetch models and make requests directly to OpenAI's Responses / Completions and List Models API using your browser instead of through the server."
|
||||
/>}
|
||||
|
||||
|
||||
+2
-3
@@ -11,7 +11,7 @@ export interface DOpenAIServiceSettings {
|
||||
oaiKey: string;
|
||||
oaiOrg: string;
|
||||
oaiHost: string; // use OpenAI-compatible non-default hosts (full origin path)
|
||||
oaiCSF?: boolean;
|
||||
csf?: boolean;
|
||||
heliKey: string; // helicone key (works in conjunction with oaiHost)
|
||||
moderationCheck: boolean;
|
||||
}
|
||||
@@ -26,13 +26,12 @@ export const ModelVendorOpenAI: IModelVendor<DOpenAIServiceSettings, OpenAIAcces
|
||||
hasServerConfigKey: 'hasLlmOpenAI',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfKey: 'oaiCSF',
|
||||
csfAvailable: _csfOpenAIAvailable,
|
||||
|
||||
// functions
|
||||
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
|
||||
dialect: 'openai',
|
||||
clientSideFetch: _csfOpenAIAvailable(partialSetup) && !partialSetup?.oaiCSF,
|
||||
clientSideFetch: _csfOpenAIAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: '',
|
||||
oaiOrg: '',
|
||||
oaiHost: '',
|
||||
|
||||
+13
-3
@@ -8,7 +8,9 @@ import { ExternalLink } from '~/common/components/ExternalLink';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { useLlmUpdateModels } from '../../llm.client.hooks';
|
||||
@@ -23,7 +25,7 @@ const OPENPIPE_API_KEY_LINK = 'https://app.openpipe.ai/settings';
|
||||
export function OpenPipeServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
// const advanced = useToggleableBoolean();
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const {
|
||||
@@ -32,8 +34,9 @@ export function OpenPipeServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorOpenPipe);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: openPipeKey, oaiOrg: openPipeTags } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: openPipeKey, oaiOrg: openPipeTags } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// validate if url is a well formed proper url with zod
|
||||
const shallFetchSucceed = !needsUserKey || (!!openPipeKey && serviceSetupValid);
|
||||
@@ -80,7 +83,14 @@ export function OpenPipeServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
and <strong>fine-tune</strong> and deploy custom models cost-effectively for specific tasks.
|
||||
</Typography>
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!openPipeKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to OpenPipe API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
export interface DOpenPipeServiceSettings {
|
||||
openPipeKey: string;
|
||||
openPipeTags: string; // hack: this will travel as 'oaiOrg' in the access schema - then interpreted in the openAIAccess() function
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -18,6 +19,9 @@ export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIA
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmOpenPipe',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfOpenPipeAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
openPipeKey: '',
|
||||
@@ -28,6 +32,7 @@ export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIA
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'openpipe',
|
||||
clientSideFetch: _csfOpenPipeAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.openPipeKey || '',
|
||||
oaiOrg: partialSetup?.openPipeTags || '', // HACK: use tags for org - should use type discrimination
|
||||
oaiHost: '',
|
||||
@@ -39,3 +44,7 @@ export const ModelVendorOpenPipe: IModelVendor<DOpenPipeServiceSettings, OpenAIA
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfOpenPipeAvailable(s?: Partial<DOpenPipeServiceSettings>) {
|
||||
return !!s?.openPipeKey;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Box, Button, Chip, Typography } from '@mui/joy';
|
||||
import { Box, Button, Typography } from '@mui/joy';
|
||||
import LaunchIcon from '@mui/icons-material/Launch';
|
||||
import VisibilityOffOutlinedIcon from '@mui/icons-material/VisibilityOffOutlined';
|
||||
import VisibilityOutlinedIcon from '@mui/icons-material/VisibilityOutlined';
|
||||
@@ -12,9 +12,11 @@ import { getLLMPricing } from '~/common/stores/llms/llms.types';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { PhGift } from '~/common/components/icons/phosphor/PhGift';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { getCallbackUrl } from '~/common/app.routes';
|
||||
import { llmsStoreActions, llmsStoreState } from '~/common/stores/llms/store-llms';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { useLlmUpdateModels } from '../../llm.client.hooks';
|
||||
@@ -25,13 +27,17 @@ import { isValidOpenRouterKey, ModelVendorOpenRouter } from './openrouter.vendor
|
||||
|
||||
export function OpenRouterServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, serviceHasVisibleLLMs, updateSettings } =
|
||||
useServiceSetup(props.serviceId, ModelVendorOpenRouter);
|
||||
|
||||
// derived state
|
||||
const { oaiKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
const keyValid = isValidOpenRouterKey(oaiKey);
|
||||
const keyError = (/*needsUserKey ||*/ !!oaiKey) && !keyValid;
|
||||
@@ -124,8 +130,15 @@ export function OpenRouterServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{/* These are usually moderated by the upstream provider (e.g. OpenAI).*/}
|
||||
{/*</Typography>*/}
|
||||
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!oaiKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to OpenRouter API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton
|
||||
refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError}
|
||||
refetch={refetch} disabled={!shallFetchSucceed || isFetching} loading={isFetching} error={isError} advanced={advanced}
|
||||
leftButton={
|
||||
<Box sx={{ display: 'flex', flexWrap: 'wrap', gap: 1 }}>
|
||||
<Button
|
||||
|
||||
@@ -13,6 +13,7 @@ export const isValidOpenRouterKey = (apiKey?: string) => !!apiKey && apiKey.star
|
||||
export interface DOpenRouterServiceSettings {
|
||||
oaiKey: string;
|
||||
oaiHost: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -36,6 +37,9 @@ export const ModelVendorOpenRouter: IModelVendor<DOpenRouterServiceSettings, Ope
|
||||
hasFreeModels: true,
|
||||
hasServerConfigKey: 'hasLlmOpenRouter',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfOpenRouterAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: (): DOpenRouterServiceSettings => ({
|
||||
oaiHost: 'https://openrouter.ai/api',
|
||||
@@ -43,6 +47,7 @@ export const ModelVendorOpenRouter: IModelVendor<DOpenRouterServiceSettings, Ope
|
||||
}),
|
||||
getTransportAccess: (partialSetup): OpenAIAccessSchema => ({
|
||||
dialect: 'openrouter',
|
||||
clientSideFetch: _csfOpenRouterAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.oaiKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.oaiHost || '',
|
||||
@@ -75,3 +80,7 @@ export const ModelVendorOpenRouter: IModelVendor<DOpenRouterServiceSettings, Ope
|
||||
|
||||
// rate limit timestamp
|
||||
let nextGenerationTs = 0;
|
||||
|
||||
function _csfOpenRouterAvailable(s?: Partial<DOpenRouterServiceSettings>) {
|
||||
return !!s?.oaiKey;
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { ModelVendorPerplexity } from './perplexity.vendor';
|
||||
@@ -20,6 +22,9 @@ const PERPLEXITY_REG_LINK = 'https://www.perplexity.ai/settings/api';
|
||||
|
||||
export function PerplexityServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const {
|
||||
service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs,
|
||||
@@ -27,8 +32,9 @@ export function PerplexityServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorPerplexity);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: perplexityKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: perplexityKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// key validation
|
||||
const shallFetchSucceed = !needsUserKey || (!!perplexityKey && serviceSetupValid);
|
||||
@@ -59,7 +65,14 @@ export function PerplexityServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
as a service for a variety of models. See the <Link href='https://www.perplexity.ai/' target='_blank'>Perplexity AI</Link> website for more information.
|
||||
</Typography>
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!perplexityKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Perplexity API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
interface DPerpexityServiceSettings {
|
||||
perplexityKey: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -17,6 +18,9 @@ export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, Open
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmPerplexity',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfPerplexityAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
perplexityKey: '',
|
||||
@@ -26,6 +30,7 @@ export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, Open
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'perplexity',
|
||||
clientSideFetch: _csfPerplexityAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.perplexityKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: '',
|
||||
@@ -37,3 +42,7 @@ export const ModelVendorPerplexity: IModelVendor<DPerpexityServiceSettings, Open
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfPerplexityAvailable(s?: Partial<DPerpexityServiceSettings>) {
|
||||
return !!s?.perplexityKey;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Alert, Typography } from '@mui/joy';
|
||||
import { Alert } from '@mui/joy';
|
||||
|
||||
import type { DModelsServiceId } from '~/common/stores/llms/llms.service.types';
|
||||
import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
@@ -8,6 +8,7 @@ import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { FormSwitchControl } from '~/common/components/forms/FormSwitchControl';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { Link } from '~/common/components/Link';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
@@ -33,8 +34,9 @@ export function TogetherAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
} = useServiceSetup(props.serviceId, ModelVendorTogetherAI);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: togetherKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: togetherKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// validate if url is a well formed proper url with zod
|
||||
const shallFetchSucceed = !needsUserKey || (!!togetherKey && serviceSetupValid);
|
||||
@@ -60,17 +62,24 @@ export function TogetherAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
placeholder='...'
|
||||
/>
|
||||
|
||||
{advanced.on && <FormSwitchControl
|
||||
{showAdvanced && <FormSwitchControl
|
||||
title='Rate Limiter' on='Enabled' off='Disabled'
|
||||
description={partialSettings?.togetherFreeTrial ? 'Free trial: 2 requests/2s' : 'Disabled'}
|
||||
checked={partialSettings?.togetherFreeTrial ?? false}
|
||||
onChange={on => updateSettings({ togetherFreeTrial: on })}
|
||||
/>}
|
||||
|
||||
{advanced.on && !!partialSettings?.togetherFreeTrial && <Alert variant='soft'>
|
||||
{showAdvanced && !!partialSettings?.togetherFreeTrial && <Alert variant='soft'>
|
||||
Note: Please refresh the models list if you toggle the rate limiter.
|
||||
</Alert>}
|
||||
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!togetherKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to Together AI API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={/*!shallFetchSucceed ||*/ isFetching} loading={isFetching} error={isError} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
@@ -8,6 +8,7 @@ interface DTogetherAIServiceSettings {
|
||||
togetherKey: string;
|
||||
togetherHost: string;
|
||||
togetherFreeTrial: boolean;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -19,6 +20,9 @@ export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, Ope
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmTogetherAI',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfTogetherAIAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({
|
||||
togetherKey: '',
|
||||
@@ -30,6 +34,7 @@ export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, Ope
|
||||
},
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'togetherai',
|
||||
clientSideFetch: _csfTogetherAIAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.togetherKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: partialSetup?.togetherHost || '',
|
||||
@@ -62,3 +67,7 @@ export const ModelVendorTogetherAI: IModelVendor<DTogetherAIServiceSettings, Ope
|
||||
|
||||
// rate limit timestamp
|
||||
let nextGenerationTs = 0;
|
||||
|
||||
function _csfTogetherAIAvailable(s?: Partial<DTogetherAIServiceSettings>) {
|
||||
return !!s?.togetherKey;
|
||||
}
|
||||
|
||||
+15
-2
@@ -5,7 +5,9 @@ import { AlreadySet } from '~/common/components/AlreadySet';
|
||||
import { ExternalLink } from '~/common/components/ExternalLink';
|
||||
import { FormInputKey } from '~/common/components/forms/FormInputKey';
|
||||
import { InlineError } from '~/common/components/InlineError';
|
||||
import { SetupFormClientSideToggle } from '~/common/components/forms/SetupFormClientSideToggle';
|
||||
import { SetupFormRefetchButton } from '~/common/components/forms/SetupFormRefetchButton';
|
||||
import { useToggleableBoolean } from '~/common/util/hooks/useToggleableBoolean';
|
||||
|
||||
import { ApproximateCosts } from '../ApproximateCosts';
|
||||
import { useLlmUpdateModels } from '../../llm.client.hooks';
|
||||
@@ -20,13 +22,17 @@ const EXTERNAL_LINK_XAI_API_KEYS = 'https://console.x.ai/';
|
||||
|
||||
export function XAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
|
||||
// state
|
||||
const advanced = useToggleableBoolean();
|
||||
|
||||
// external state
|
||||
const { service, serviceAccess, serviceHasCloudTenantConfig, serviceHasLLMs, serviceSetupValid, updateSettings } =
|
||||
useServiceSetup(props.serviceId, ModelVendorXAI);
|
||||
|
||||
// derived state
|
||||
const { oaiKey: xaiKey } = serviceAccess;
|
||||
const { clientSideFetch, oaiKey: xaiKey } = serviceAccess;
|
||||
const needsUserKey = !serviceHasCloudTenantConfig;
|
||||
const showAdvanced = advanced.on || !!clientSideFetch;
|
||||
|
||||
// key validation
|
||||
const shallFetchSucceed = !needsUserKey || (!!xaiKey && serviceSetupValid);
|
||||
@@ -62,7 +68,14 @@ export function XAIServiceSetup(props: { serviceId: DModelsServiceId }) {
|
||||
{/* onChange={(text) => updateSettings({ xaiHost: text })}*/}
|
||||
{/*/>*/}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={isFetching} error={isError} loading={isFetching} />
|
||||
{showAdvanced && <SetupFormClientSideToggle
|
||||
visible={!!xaiKey}
|
||||
checked={!!clientSideFetch}
|
||||
onChange={on => updateSettings({ csf: on })}
|
||||
helpText='Connect directly to xAI API from your browser instead of through the server.'
|
||||
/>}
|
||||
|
||||
<SetupFormRefetchButton refetch={refetch} disabled={isFetching} error={isError} loading={isFetching} advanced={advanced} />
|
||||
|
||||
{isError && <InlineError error={error} />}
|
||||
|
||||
|
||||
+9
@@ -6,6 +6,7 @@ import { ModelVendorOpenAI } from '../openai/openai.vendor';
|
||||
|
||||
export interface DXAIServiceSettings {
|
||||
xaiKey: string;
|
||||
csf?: boolean;
|
||||
}
|
||||
|
||||
export const ModelVendorXAI: IModelVendor<DXAIServiceSettings, OpenAIAccessSchema> = {
|
||||
@@ -17,11 +18,15 @@ export const ModelVendorXAI: IModelVendor<DXAIServiceSettings, OpenAIAccessSchem
|
||||
instanceLimit: 1,
|
||||
hasServerConfigKey: 'hasLlmXAI',
|
||||
|
||||
/// client-side-fetch ///
|
||||
csfAvailable: _csfXAIAvailable,
|
||||
|
||||
// functions
|
||||
initializeSetup: () => ({ xaiKey: '' }),
|
||||
validateSetup: setup => setup.xaiKey?.length >= 80, // we assume all API keys are 80 chars+ - we won't have a strict validation
|
||||
getTransportAccess: (partialSetup) => ({
|
||||
dialect: 'xai',
|
||||
clientSideFetch: _csfXAIAvailable(partialSetup) && !!partialSetup?.csf,
|
||||
oaiKey: partialSetup?.xaiKey || '',
|
||||
oaiOrg: '',
|
||||
oaiHost: '',
|
||||
@@ -33,3 +38,7 @@ export const ModelVendorXAI: IModelVendor<DXAIServiceSettings, OpenAIAccessSchem
|
||||
rpcUpdateModelsOrThrow: ModelVendorOpenAI.rpcUpdateModelsOrThrow,
|
||||
|
||||
};
|
||||
|
||||
function _csfXAIAvailable(s?: Partial<DXAIServiceSettings>) {
|
||||
return !!s?.xaiKey;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { ModelVendorId } from '~/modules/llms/vendors/vendors.registry';
|
||||
|
||||
import type { DSpeexCredentials, DSpeexVoice, DSpeexVendorType } from './speex.types';
|
||||
|
||||
|
||||
/**
|
||||
* Descriptions for each Speex TTS Engine Vendor
|
||||
* - used for DSpeexEngine instances creation, mainly
|
||||
*
|
||||
* Configuration including credentials and default voices are in DSpeexEngine instances
|
||||
* in the speex store.
|
||||
*/
|
||||
export interface ISpeexVendor<TVt extends DSpeexVendorType> {
|
||||
readonly vendorType: TVt;
|
||||
readonly name: string;
|
||||
readonly protocol: 'rpc' | 'webspeech';
|
||||
readonly location: 'browser' | 'local' | 'cloud';
|
||||
readonly priority: number; // display priority (lower = higher priority): elevenlabs=10, localai=20, openai=30, webspeech=100
|
||||
|
||||
// auto-detection info
|
||||
readonly autoFromLlmVendorIds?: ModelVendorId[];
|
||||
|
||||
// capabilities
|
||||
readonly capabilities: {
|
||||
streaming: boolean;
|
||||
voiceListing: boolean; // can list voices via API (vs hardcoded)
|
||||
speedControl: boolean;
|
||||
pitchControl: boolean;
|
||||
};
|
||||
|
||||
// defaults for creating new engines
|
||||
|
||||
getDefaultCredentials(): DSpeexCredentials<TVt>;
|
||||
|
||||
getDefaultVoice(): DSpeexVoice<TVt>;
|
||||
}
|
||||
|
||||
export type ISpeexVendorAny = { [TVt in DSpeexVendorType]: ISpeexVendor<TVt> }[DSpeexVendorType];
|
||||
@@ -0,0 +1,426 @@
|
||||
import * as React from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
|
||||
import { Box, Button, Divider, FormControl, Typography } from '@mui/joy';
|
||||
import PlayArrowRoundedIcon from '@mui/icons-material/PlayArrowRounded';
|
||||
import StopRoundedIcon from '@mui/icons-material/StopRounded';
|
||||
|
||||
import { FormChipControl } from '~/common/components/forms/FormChipControl';
|
||||
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
|
||||
import { FormSecretField } from '~/common/components/forms/FormSecretField';
|
||||
import { FormSliderControl } from '~/common/components/forms/FormSliderControl';
|
||||
import { FormTextField } from '~/common/components/forms/FormTextField';
|
||||
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
|
||||
|
||||
import type { DCredentialsApiKey, DSpeexEngine, DSpeexEngineAny, DSpeexVendorType, DVoiceElevenLabs, DVoiceLocalAI, DVoiceOpenAI, DVoiceWebSpeech } from '../speex.types';
|
||||
import { SPEEX_DEFAULTS, SPEEX_PREVIEW_STREAM, SPEEX_PREVIEW_TEXT } from '../speex.config';
|
||||
import { SpeexVoiceAutocomplete } from './SpeexVoiceAutocomplete';
|
||||
import { SpeexVoiceSelect } from './SpeexVoiceSelect';
|
||||
import { speakText } from '../speex.client';
|
||||
import { speexVendorTypeLabel } from './SpeexEngineSelect';
|
||||
|
||||
|
||||
function CredentialsApiKeyInputs({ credentials, onUpdate, vendorType, showHost, hostRequired, hostPlaceholder }: {
|
||||
credentials: DCredentialsApiKey;
|
||||
onUpdate: (credentials: DCredentialsApiKey) => void;
|
||||
vendorType: DSpeexVendorType;
|
||||
showHost?: boolean;
|
||||
hostRequired?: boolean;
|
||||
hostPlaceholder?: string;
|
||||
}) {
|
||||
return <>
|
||||
|
||||
<FormSecretField
|
||||
autoCompleteId={`speex-${vendorType}-key`}
|
||||
title='API Key'
|
||||
description={hostRequired ? 'Optional' : speexVendorTypeLabel(vendorType)}
|
||||
value={credentials.apiKey}
|
||||
onChange={value => onUpdate({ ...credentials, apiKey: value })}
|
||||
required={!hostRequired}
|
||||
startDecorator={credentials.apiKey ? false : undefined}
|
||||
// placeholder='Required'
|
||||
inputSx={{ maxWidth: 220 }}
|
||||
/>
|
||||
|
||||
{showHost && (
|
||||
<FormTextField
|
||||
autoCompleteId={`speex-${vendorType}-host`}
|
||||
title='API Host'
|
||||
description={hostRequired ? 'Required' : 'Optional'}
|
||||
value={credentials.apiHost ?? ''}
|
||||
onChange={text => onUpdate({ ...credentials, apiHost: text || undefined })}
|
||||
placeholder={hostPlaceholder ?? 'https://api.example.com'}
|
||||
inputSx={{ maxWidth: 220 }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showHost && <Divider inset='context' />}
|
||||
|
||||
</>;
|
||||
}
|
||||
|
||||
|
||||
function PreviewButton({ engineId }: { engineId: DSpeexEngineAny['engineId'] }) {
|
||||
|
||||
// async + cache
|
||||
const { isFetching, isError, error, refetch: previewVoice } = useQuery({
|
||||
enabled: false, // manual trigger only
|
||||
queryKey: ['speex-preview', engineId],
|
||||
queryFn: async () => {
|
||||
const result = await speakText(
|
||||
SPEEX_PREVIEW_TEXT,
|
||||
{ engineId: engineId },
|
||||
{ streaming: SPEEX_PREVIEW_STREAM },
|
||||
);
|
||||
if (!result.success) throw new Error(result.error || 'Preview failed');
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<TooltipOutlined color='danger' title={error?.message ? <pre>{error.message}</pre> : false}>
|
||||
<Button
|
||||
variant='outlined'
|
||||
color={isError ? 'danger' : 'neutral'}
|
||||
size='sm'
|
||||
onClick={() => previewVoice()}
|
||||
disabled={isFetching}
|
||||
startDecorator={isFetching ? <StopRoundedIcon /> : <PlayArrowRoundedIcon />}
|
||||
sx={{ ml: 'auto', minWidth: 130 }}
|
||||
>
|
||||
{isFetching ? 'Speaking...' : isError ? 'Retry' : 'Preview'}
|
||||
</Button>
|
||||
</TooltipOutlined>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
export function SpeexConfigureEngineFull(props: {
|
||||
engine: DSpeexEngineAny;
|
||||
isMobile: boolean;
|
||||
mode?: 'full' | 'voice-only';
|
||||
bottomStart?: React.ReactNode;
|
||||
onUpdate: (updates: Partial<DSpeexEngineAny>) => void;
|
||||
}) {
|
||||
const { engine, isMobile, mode = 'full', bottomStart, onUpdate } = props;
|
||||
return <>
|
||||
|
||||
{/*<Box mt={2} />*/}
|
||||
{/*<Divider sx={{ my: 1 }} inset='context' />*/}
|
||||
<Divider sx={{ my: 1 }} inset='context'>{isMobile ? 'Configuration' : 'App Voice Configuration'}</Divider>
|
||||
|
||||
{/* Engine-Specific pane */}
|
||||
{engine.vendorType === 'elevenlabs' ? (
|
||||
<ElevenLabsConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
|
||||
) : engine.vendorType === 'localai' ? (
|
||||
<LocalAIConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
|
||||
) : engine.vendorType === 'openai' ? (
|
||||
<OpenAIConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
|
||||
) : engine.vendorType === 'webspeech' ? (
|
||||
<WebSpeechConfig engine={engine} onUpdate={onUpdate} isMobile={isMobile} mode={mode} />
|
||||
) : (
|
||||
<Typography level='body-sm' color='warning'>Unknown engine type {(engine as any)?.vendorType}</Typography>
|
||||
)}
|
||||
|
||||
{/* (Delete | Chip) -- Preview */}
|
||||
<Box sx={{ display: 'flex', alignItems: 'center' }}>
|
||||
{bottomStart}
|
||||
<PreviewButton engineId={engine.engineId} />
|
||||
</Box>
|
||||
|
||||
</>;
|
||||
}
|
||||
|
||||
|
||||
// Vendor-specific configs
|
||||
|
||||
function ElevenLabsConfig({ engine, onUpdate, mode, isMobile }: {
|
||||
engine: DSpeexEngine<'elevenlabs'>,
|
||||
onUpdate: (updates: Partial<DSpeexEngine<'elevenlabs'>>) => void;
|
||||
isMobile: boolean;
|
||||
mode: 'full' | 'voice-only';
|
||||
}) {
|
||||
|
||||
const { credentials, voice } = engine;
|
||||
const showCredentials = mode === 'full' && !engine.isAutoLinked && credentials.type === 'api-key';
|
||||
|
||||
const handleCredentialsUpdate = React.useCallback((newCredentials: DCredentialsApiKey) => {
|
||||
onUpdate({ credentials: newCredentials });
|
||||
}, [onUpdate]);
|
||||
|
||||
const handleVoiceChange = React.useCallback((ttsVoiceId: DVoiceElevenLabs['ttsVoiceId']) => {
|
||||
const { ttsVoiceId: _, ...restVoice } = voice;
|
||||
onUpdate({
|
||||
voice: {
|
||||
...restVoice,
|
||||
...(ttsVoiceId && { ttsVoiceId }),
|
||||
},
|
||||
});
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
return <>
|
||||
|
||||
{/* Credentials (only for manually added engines in full mode) */}
|
||||
{showCredentials && (
|
||||
<CredentialsApiKeyInputs
|
||||
credentials={credentials}
|
||||
onUpdate={handleCredentialsUpdate}
|
||||
vendorType='elevenlabs'
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormChipControl<Exclude<DVoiceElevenLabs['ttsModel'], undefined>>
|
||||
title='Model'
|
||||
alignEnd
|
||||
options={[
|
||||
{ value: 'eleven_multilingual_v2', label: 'Multilingual v2', description: 'Default' },
|
||||
{ value: 'eleven_turbo_v2_5', label: 'Turbo v2.5', description: 'Fast' },
|
||||
{ value: 'eleven_flash_v2_5', label: 'Flash v2.5', description: 'Fastest' },
|
||||
{ value: 'eleven_v3', label: 'v3', description: 'Newest' },
|
||||
]}
|
||||
value={voice.ttsModel ?? SPEEX_DEFAULTS.ELEVENLABS_MODEL}
|
||||
onChange={(value) => onUpdate({ voice: { ...voice, ttsModel: value } })}
|
||||
/>
|
||||
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
|
||||
<FormLabelStart title='Voice' description={isMobile ? undefined : 'ElevenLabs voice'} />
|
||||
<SpeexVoiceSelect
|
||||
autoPreview
|
||||
engine={engine}
|
||||
voiceId={voice.ttsVoiceId ?? null}
|
||||
onVoiceChange={handleVoiceChange}
|
||||
/>
|
||||
</FormControl>
|
||||
|
||||
{/*{showCredentials && (*/}
|
||||
{/* <FormHelperText>*/}
|
||||
{/* Voice listing requires API key. Language auto-detected from preferences.*/}
|
||||
{/* </FormHelperText>*/}
|
||||
{/*)}*/}
|
||||
|
||||
</>;
|
||||
}
|
||||
|
||||
|
||||
function LocalAIConfig({ engine, onUpdate, mode, isMobile }: {
|
||||
engine: DSpeexEngine<'localai'>,
|
||||
onUpdate: (updates: Partial<DSpeexEngine<'localai'>>) => void;
|
||||
isMobile: boolean;
|
||||
mode: 'full' | 'voice-only';
|
||||
}) {
|
||||
const { credentials, voice } = engine;
|
||||
const showCredentials = mode === 'full' && !engine.isAutoLinked && credentials.type === 'api-key';
|
||||
|
||||
const handleCredentialsUpdate = React.useCallback((newCredentials: DCredentialsApiKey) => {
|
||||
onUpdate({ credentials: newCredentials });
|
||||
}, [onUpdate]);
|
||||
|
||||
const handleModelChange = React.useCallback((ttsModel: DVoiceLocalAI['ttsModel']) => {
|
||||
const { ttsModel: _, ...restVoice } = voice;
|
||||
onUpdate({
|
||||
voice: {
|
||||
...restVoice,
|
||||
...(ttsModel && { ttsModel }),
|
||||
},
|
||||
});
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
return <>
|
||||
|
||||
{/* Credentials (only for manually added engines in full mode) */}
|
||||
{showCredentials && (
|
||||
<CredentialsApiKeyInputs
|
||||
credentials={credentials}
|
||||
onUpdate={handleCredentialsUpdate}
|
||||
vendorType='localai'
|
||||
showHost
|
||||
hostRequired
|
||||
hostPlaceholder='http://localhost:8080'
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Model: autocomplete with suggestions + free-form input */}
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
|
||||
<FormLabelStart title='Model' description={isMobile ? undefined : 'Select or type'} />
|
||||
<SpeexVoiceAutocomplete
|
||||
engine={engine}
|
||||
value={voice.ttsModel}
|
||||
onValueChange={handleModelChange}
|
||||
placeholder='e.g., kokoro'
|
||||
/>
|
||||
</FormControl>
|
||||
|
||||
<FormTextField
|
||||
autoCompleteId='speex-localai-backend'
|
||||
title='Backend'
|
||||
description='Optional'
|
||||
placeholder='e.g., coqui, bark, piper'
|
||||
value={voice.ttsBackend ?? ''}
|
||||
onChange={(text) => onUpdate({ voice: { ...voice, ttsBackend: text || undefined } })}
|
||||
inputSx={{ maxWidth: 220 }}
|
||||
/>
|
||||
|
||||
</>;
|
||||
}
|
||||
|
||||
|
||||
function OpenAIConfig({ engine, onUpdate, isMobile, mode }: {
|
||||
engine: DSpeexEngine<'openai'>,
|
||||
onUpdate: (updates: Partial<DSpeexEngineAny>) => void;
|
||||
isMobile: boolean;
|
||||
mode: 'full' | 'voice-only';
|
||||
}) {
|
||||
|
||||
const { credentials, voice } = engine;
|
||||
const showCredentials = mode === 'full' && !engine.isAutoLinked && credentials.type === 'api-key';
|
||||
|
||||
const handleCredentialsUpdate = React.useCallback((newCredentials: DCredentialsApiKey) => {
|
||||
onUpdate({ credentials: newCredentials });
|
||||
}, [onUpdate]);
|
||||
|
||||
const handleVoiceChange = React.useCallback((ttsVoiceId: DVoiceOpenAI['ttsVoiceId']) => {
|
||||
const { ttsVoiceId: _, ...restVoice } = voice;
|
||||
onUpdate({
|
||||
voice: {
|
||||
...restVoice,
|
||||
...(ttsVoiceId && { ttsVoiceId }),
|
||||
},
|
||||
});
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
const handleSpeedChange = React.useCallback((value: number) => {
|
||||
onUpdate({ voice: { ...voice, ttsSpeed: value } });
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
return <>
|
||||
|
||||
{/* Credentials (only for manually added engines in full mode) */}
|
||||
{showCredentials && (
|
||||
<CredentialsApiKeyInputs
|
||||
credentials={credentials}
|
||||
onUpdate={handleCredentialsUpdate}
|
||||
vendorType='openai'
|
||||
showHost
|
||||
hostPlaceholder='https://api.openai.com (optional)'
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormChipControl<DVoiceOpenAI['ttsModel']>
|
||||
title='Model'
|
||||
alignEnd
|
||||
options={[
|
||||
{ value: 'gpt-4o-mini-tts', label: 'GPT-4o Mini', description: 'Expressive' },
|
||||
{ value: 'tts-1', label: 'TTS-1', description: 'Fast' },
|
||||
{ value: 'tts-1-hd', label: 'TTS-1-HD', description: 'Quality' },
|
||||
]}
|
||||
value={voice.ttsModel ?? SPEEX_DEFAULTS.OPENAI_MODEL}
|
||||
onChange={value => onUpdate({
|
||||
voice: {
|
||||
...voice,
|
||||
ttsModel: value,
|
||||
},
|
||||
})}
|
||||
/>
|
||||
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
|
||||
<FormLabelStart title='Voice' description={isMobile ? undefined : 'OpenAI TTS voice'} />
|
||||
<SpeexVoiceSelect
|
||||
engine={engine}
|
||||
voiceId={voice.ttsVoiceId ?? null}
|
||||
onVoiceChange={handleVoiceChange}
|
||||
/>
|
||||
</FormControl>
|
||||
|
||||
|
||||
<FormSliderControl
|
||||
title='Speed'
|
||||
description={`${voice.ttsSpeed ?? 1}x`}
|
||||
min={0.5}
|
||||
max={2}
|
||||
step={0.25}
|
||||
value={voice.ttsSpeed ?? 1}
|
||||
onChange={handleSpeedChange}
|
||||
valueLabelDisplay={voice.ttsSpeed && voice.ttsSpeed !== 1 ? 'on' : 'auto'}
|
||||
sliderSx={{ maxWidth: 220, my: -0.5 }}
|
||||
/>
|
||||
|
||||
{voice.ttsModel === 'gpt-4o-mini-tts' && (
|
||||
<FormTextField
|
||||
autoCompleteId='speex-openai-instruction'
|
||||
title='Instruction'
|
||||
description={isMobile ? undefined : '4o Mini only'}
|
||||
placeholder='e.g., Speak with joy'
|
||||
value={voice.ttsInstruction ?? ''}
|
||||
onChange={(text) => onUpdate({ voice: { ...voice, ttsInstruction: text } })}
|
||||
inputSx={{ flexGrow: 1, maxWidth: 220 }}
|
||||
/>
|
||||
)}
|
||||
|
||||
</>;
|
||||
}
|
||||
|
||||
|
||||
function WebSpeechConfig({ engine, onUpdate, isMobile }: {
|
||||
engine: DSpeexEngine<'webspeech'>
|
||||
onUpdate: (updates: Partial<DSpeexEngine<'webspeech'>>) => void;
|
||||
isMobile: boolean;
|
||||
mode: 'full' | 'voice-only';
|
||||
}) {
|
||||
|
||||
const { voice } = engine;
|
||||
|
||||
const handleVoiceChange = React.useCallback((ttsVoiceURI: DVoiceWebSpeech['ttsVoiceURI']) => {
|
||||
const { ttsVoiceURI: _, ...restVoice } = voice;
|
||||
onUpdate({
|
||||
voice: {
|
||||
...restVoice,
|
||||
...(ttsVoiceURI && { ttsVoiceURI }),
|
||||
},
|
||||
});
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
const handleSpeedChange = React.useCallback((value: number) => {
|
||||
onUpdate({ voice: { ...voice, ttsSpeed: value } });
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
const handlePitchChange = React.useCallback((value: number) => {
|
||||
onUpdate({ voice: { ...voice, ttsPitch: value } });
|
||||
}, [onUpdate, voice]);
|
||||
|
||||
return <>
|
||||
|
||||
<FormControl orientation='horizontal' sx={{ justifyContent: 'space-between', alignItems: 'center', overflow: 'hidden' }}>
|
||||
<FormLabelStart title='Voice' description={isMobile ? undefined : 'System voice'} />
|
||||
<SpeexVoiceSelect
|
||||
engine={engine}
|
||||
voiceId={voice.ttsVoiceURI ?? null}
|
||||
onVoiceChange={handleVoiceChange}
|
||||
/>
|
||||
</FormControl>
|
||||
|
||||
<FormSliderControl
|
||||
title='Speed'
|
||||
description={`${(voice.ttsSpeed ?? 1).toFixed(1)}x`}
|
||||
min={0.5}
|
||||
max={2}
|
||||
step={0.1}
|
||||
value={voice.ttsSpeed ?? 1}
|
||||
onChange={handleSpeedChange}
|
||||
valueLabelDisplay={voice.ttsSpeed && voice.ttsSpeed !== 1 ? 'on' : 'auto'}
|
||||
sliderSx={{ maxWidth: 220, my: -0.5 }}
|
||||
/>
|
||||
|
||||
<FormSliderControl
|
||||
title='Pitch'
|
||||
description={`${(voice.ttsPitch ?? 1).toFixed(1)}`}
|
||||
min={0.5}
|
||||
max={2}
|
||||
step={0.1}
|
||||
value={voice.ttsPitch ?? 1}
|
||||
onChange={handlePitchChange}
|
||||
valueLabelDisplay={voice.ttsPitch && voice.ttsPitch !== 1 ? 'on' : 'auto'}
|
||||
sliderSx={{ maxWidth: 220, my: -0.5 }}
|
||||
/>
|
||||
|
||||
</>;
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* SpeexEngineSettings - TTS engine selection and configuration
|
||||
*
|
||||
* Provides:
|
||||
* - Chip-based engine selection with visual status
|
||||
* - Add Service dropdown menu
|
||||
* - Per-engine voice configuration in a Card
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Box, Button, Chip, Dropdown, ListItemDecorator, Menu, MenuButton, MenuItem, SvgIconProps, Typography } from '@mui/joy';
|
||||
import AddIcon from '@mui/icons-material/Add';
|
||||
import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline';
|
||||
import LinkIcon from '@mui/icons-material/Link';
|
||||
|
||||
import { ConfirmationModal } from '~/common/components/modals/ConfirmationModal';
|
||||
import { ElevenLabsIcon } from '~/common/components/icons/vendors/ElevenLabsIcon';
|
||||
import { FormLabelStart } from '~/common/components/forms/FormLabelStart';
|
||||
import { LocalAIIcon } from '~/common/components/icons/vendors/LocalAIIcon';
|
||||
import { OpenAIIcon } from '~/common/components/icons/vendors/OpenAIIcon';
|
||||
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
|
||||
import { themeZIndexOverMobileDrawer } from '~/common/app.theme';
|
||||
|
||||
import type { DSpeexEngineAny, DSpeexVendorType } from '../speex.types';
|
||||
import { SpeexConfigureEngineFull } from './SpeexConfigureEngineFull';
|
||||
import { speexAreCredentialsValid, useSpeexEngines, useSpeexGlobalEngine, useSpeexStore } from '../store-module-speex';
|
||||
|
||||
|
||||
const _styles = {
|
||||
menu: {
|
||||
zIndex: themeZIndexOverMobileDrawer,
|
||||
minWidth: 220,
|
||||
'--List-padding': '0.75rem',
|
||||
borderRadius: 'xl',
|
||||
boxShadow: 'md',
|
||||
},
|
||||
menuButton: {
|
||||
ml: 'auto',
|
||||
// borderRadius: '1.5rem',
|
||||
// borderColor: 'neutral.outlinedBorder', // like ModeServiceSelector's Add button
|
||||
// minWidth: 150,
|
||||
textWrap: 'nowrap',
|
||||
// '&[aria-expanded="true"]': {
|
||||
// borderBottomRightRadius: 0,
|
||||
// borderBottomLeftRadius: 0,
|
||||
// // color: 'neutral.softColor',
|
||||
// // backgroundColor: 'neutral.softHoverBg',
|
||||
// },
|
||||
},
|
||||
menuItem: {
|
||||
py: 1,
|
||||
px: 1,
|
||||
borderRadius: 'md',
|
||||
minHeight: 56,
|
||||
},
|
||||
menuItemContent: {
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 0.125,
|
||||
},
|
||||
menuItemName: {
|
||||
fontWeight: 600,
|
||||
},
|
||||
menuItemDescription: {
|
||||
fontWeight: 400,
|
||||
},
|
||||
chipRow: {
|
||||
display: 'flex',
|
||||
flexWrap: 'wrap',
|
||||
gap: 1,
|
||||
},
|
||||
chip: {
|
||||
px: 1.5,
|
||||
minHeight: '2rem',
|
||||
// borderRadius: 'md',
|
||||
// boxShadow: 'sm',
|
||||
},
|
||||
chipUnconfigured: {
|
||||
px: 1.5,
|
||||
minHeight: '2rem',
|
||||
// borderRadius: 'md',
|
||||
// color: 'text.tertiary',
|
||||
opacity: 0.6,
|
||||
},
|
||||
chipSymbol: {
|
||||
ml: -0.75,
|
||||
mr: 0.5,
|
||||
width: 20,
|
||||
height: 20,
|
||||
borderRadius: '50%',
|
||||
backgroundColor: 'background.surface',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
},
|
||||
} as const;
|
||||
|
||||
|
||||
const ADDABLE_VENDORS: { vendorType: DSpeexVendorType; label: string; description: string, icon?: React.FunctionComponent<SvgIconProps> }[] = [
|
||||
{ vendorType: 'elevenlabs', label: 'ElevenLabs', description: 'Premium voices', icon: ElevenLabsIcon },
|
||||
{ vendorType: 'localai', label: 'LocalAI', description: 'Self-hosted TTS', icon: LocalAIIcon },
|
||||
{ vendorType: 'openai', label: 'OpenAI TTS', description: 'Reliable', icon: OpenAIIcon },
|
||||
] as const;
|
||||
|
||||
|
||||
export function SpeexConfigureEngines(_props: { isMobile: boolean }) {
|
||||
|
||||
// state
|
||||
const [confirmDeleteEngine, setConfirmDeleteEngine] = React.useState<DSpeexEngineAny | null>(null);
|
||||
|
||||
// external state - module
|
||||
const engines = useSpeexEngines();
|
||||
const activeEngine = useSpeexGlobalEngine(); // auto-select the highest priority, if the user choice (active engine) is missing
|
||||
const activeEngineId = activeEngine?.engineId ?? null;
|
||||
const activeEngineValid = !activeEngine ? false : speexAreCredentialsValid(activeEngine.credentials);
|
||||
|
||||
|
||||
// derived state
|
||||
const hasEngines = engines.length > 0;
|
||||
const canDeleteActiveEngine = activeEngine && !activeEngine.isAutoDetected && !activeEngine.isAutoLinked;
|
||||
|
||||
|
||||
// handlers
|
||||
|
||||
const handleEngineSelect = React.useCallback((engineId: string | null) => {
|
||||
useSpeexStore.getState().setActiveEngineId(engineId);
|
||||
}, []);
|
||||
|
||||
const handleEngineUpdate = React.useCallback((updates: Partial<DSpeexEngineAny>) => {
|
||||
if (activeEngineId)
|
||||
useSpeexStore.getState().updateEngine(activeEngineId, updates);
|
||||
}, [activeEngineId]);
|
||||
|
||||
const handleAddEngine = React.useCallback((vendorType: DSpeexVendorType) => {
|
||||
const newEngineId = useSpeexStore.getState().createEngine(vendorType);
|
||||
useSpeexStore.getState().setActiveEngineId(newEngineId);
|
||||
}, []);
|
||||
|
||||
const handleDeleteClick = React.useCallback((event: React.MouseEvent) => {
|
||||
if (!activeEngine || !canDeleteActiveEngine) return;
|
||||
|
||||
// shift+click skips confirmation
|
||||
if (event.shiftKey)
|
||||
return useSpeexStore.getState().deleteEngine(activeEngine.engineId);
|
||||
|
||||
setConfirmDeleteEngine(activeEngine);
|
||||
}, [activeEngine, canDeleteActiveEngine]);
|
||||
|
||||
const handleConfirmDelete = React.useCallback(() => {
|
||||
if (!confirmDeleteEngine) return;
|
||||
|
||||
useSpeexStore.getState().deleteEngine(confirmDeleteEngine.engineId);
|
||||
|
||||
setConfirmDeleteEngine(null);
|
||||
}, [confirmDeleteEngine]);
|
||||
|
||||
const handleCancelDelete = React.useCallback(() => {
|
||||
setConfirmDeleteEngine(null);
|
||||
}, []);
|
||||
|
||||
|
||||
return <>
|
||||
|
||||
{/* "Voice Engine" + Add Service dropdown */}
|
||||
<Box sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
|
||||
<FormLabelStart
|
||||
// title='Voice Engine'
|
||||
title='Active Engine'
|
||||
description={activeEngine ? activeEngine.label : 'Select a voice provider'}
|
||||
/>
|
||||
|
||||
|
||||
{/* -> Add Service */}
|
||||
<Dropdown>
|
||||
<MenuButton size='sm' variant={!activeEngine ? 'solid' : 'outlined'} startDecorator={<AddIcon />} sx={_styles.menuButton}>
|
||||
Add
|
||||
{/* Add Service*/}
|
||||
</MenuButton>
|
||||
{/*<MenuButton size='sm' color='primary' variant={!activeEngine ? 'solid' : 'outlined'} startDecorator={<AddIcon />} sx={_styles.menuButton}>*/}
|
||||
{/* Add*/}
|
||||
{/* /!* Add Service*!/*/}
|
||||
{/*</MenuButton>*/}
|
||||
<Menu placement='bottom' popperOptions={{ modifiers: [{ name: 'offset', options: { offset: [-12, -2] } }] }} sx={_styles.menu}>
|
||||
{ADDABLE_VENDORS.map(vendor => (
|
||||
<MenuItem key={vendor.vendorType} onClick={() => handleAddEngine(vendor.vendorType)} sx={_styles.menuItem}>
|
||||
<ListItemDecorator>
|
||||
{vendor.icon ? <vendor.icon /> : null}
|
||||
</ListItemDecorator>
|
||||
<Box sx={_styles.menuItemContent}>
|
||||
<Typography level='title-md' sx={_styles.menuItemName}>{vendor.label}</Typography>
|
||||
<Typography level='body-sm' sx={_styles.menuItemDescription}>{vendor.description}</Typography>
|
||||
</Box>
|
||||
</MenuItem>
|
||||
))}
|
||||
</Menu>
|
||||
</Dropdown>
|
||||
|
||||
</Box>
|
||||
|
||||
{/* Engine Chips row */}
|
||||
{hasEngines && (
|
||||
<Box sx={_styles.chipRow}>
|
||||
{engines.map(engine => {
|
||||
const isActive = engine.engineId === activeEngineId;
|
||||
const isConfigured = speexAreCredentialsValid(engine.credentials);
|
||||
|
||||
return (
|
||||
<TooltipOutlined key={engine.engineId} title={isActive ? 'Global application voice' : isConfigured ? 'Click to activate' : 'Needs configuration'}>
|
||||
<Chip
|
||||
variant={isActive ? 'solid' : 'outlined'}
|
||||
color={!isActive ? 'neutral' : !isConfigured ? 'danger' : 'neutral'}
|
||||
// startDecorator={isActive && <Box sx={_styles.chipSymbol}>
|
||||
// <CheckRoundedIcon sx={{ fontSize: 16, color: 'text.primary' }} />
|
||||
// </Box>}
|
||||
endDecorator={engine.isAutoLinked && <LinkIcon sx={{ fontSize: 16 }} />}
|
||||
onClick={event => handleEngineSelect(event.shiftKey ? null : engine.engineId)}
|
||||
sx={isConfigured ? _styles.chip : _styles.chipUnconfigured}
|
||||
>
|
||||
{engine.label}
|
||||
</Chip>
|
||||
</TooltipOutlined>
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{/* Active engine (specific) full configuration */}
|
||||
{activeEngine && (
|
||||
<SpeexConfigureEngineFull
|
||||
engine={activeEngine}
|
||||
isMobile={_props.isMobile}
|
||||
mode={activeEngine.isAutoLinked || activeEngine.isAutoDetected ? 'voice-only' : 'full'}
|
||||
bottomStart={
|
||||
!canDeleteActiveEngine ? (
|
||||
<Chip size='sm' color={!activeEngineValid ? 'danger' : undefined} variant='soft' sx={{ px: 1.5, py: 0.5 }}>
|
||||
{!activeEngineValid ? (activeEngine.isAutoLinked ? 'Linked to AI Service' : 'Invalid Configuration')
|
||||
: activeEngine.isAutoLinked ? 'Linked to AI Service'
|
||||
: activeEngine.isAutoDetected ? 'System'
|
||||
: 'Configured Manually'}
|
||||
</Chip>
|
||||
) : (
|
||||
// <GoodTooltip title='Delete this service'>
|
||||
<Button
|
||||
size='sm'
|
||||
color='neutral'
|
||||
variant='plain'
|
||||
onClick={handleDeleteClick}
|
||||
startDecorator={<DeleteOutlineIcon />}
|
||||
// sx={{ minWidth: 120 }}
|
||||
>
|
||||
Delete
|
||||
</Button>
|
||||
// </GoodTooltip>
|
||||
)}
|
||||
onUpdate={handleEngineUpdate}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{!hasEngines && (
|
||||
<Typography level='body-sm' sx={{ color: 'text.tertiary' }}>
|
||||
No voice engines available. Configure an OpenAI-compatible AI service to auto-link TTS, or add ElevenLabs for premium voices.
|
||||
</Typography>
|
||||
)}
|
||||
|
||||
{/* Delete Confirmation Modal */}
|
||||
{!!confirmDeleteEngine && (
|
||||
<ConfirmationModal
|
||||
open
|
||||
onClose={handleCancelDelete}
|
||||
onPositive={handleConfirmDelete}
|
||||
lowStakes
|
||||
noTitleBar
|
||||
confirmationText={<>Remove <strong>{confirmDeleteEngine.label}</strong>? This cannot be undone.</>}
|
||||
positiveActionText='Remove'
|
||||
/>
|
||||
)}
|
||||
|
||||
</>;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* SpeexEngineSelect - Reusable engine selection dropdown
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Option, Select, Typography } from '@mui/joy';
|
||||
|
||||
import type { DSpeexVendorType, SpeexEngineId } from '../speex.types';
|
||||
import { speexAreCredentialsValid, useSpeexEngines } from '../store-module-speex';
|
||||
|
||||
|
||||
export function speexVendorTypeLabel(vendorType: DSpeexVendorType): string {
|
||||
switch (vendorType) {
|
||||
case 'elevenlabs':
|
||||
return 'ElevenLabs';
|
||||
case 'openai':
|
||||
return 'OpenAI';
|
||||
case 'localai':
|
||||
return 'LocalAI';
|
||||
case 'webspeech':
|
||||
return 'System';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
interface SpeexEngineSelectProps {
|
||||
/** Selected engine ID (null = none selected) */
|
||||
engineId: string | null;
|
||||
/** Called when selection changes */
|
||||
onEngineChange: (engineId: string | null) => void;
|
||||
/** Disable the select */
|
||||
disabled?: boolean;
|
||||
/** Placeholder text (default: 'Select engine...') */
|
||||
placeholder?: string;
|
||||
}
|
||||
|
||||
export function SpeexEngineSelect(props: SpeexEngineSelectProps) {
|
||||
const { engineId, onEngineChange, disabled, placeholder = 'Select engine...' } = props;
|
||||
|
||||
const engines = useSpeexEngines();
|
||||
|
||||
const validEngines = React.useMemo(
|
||||
() => engines.filter(e => speexAreCredentialsValid(e.credentials)),
|
||||
[engines],
|
||||
);
|
||||
|
||||
const handleChange = React.useCallback((_event: unknown, value: SpeexEngineId | null) => {
|
||||
onEngineChange(value);
|
||||
}, [onEngineChange]);
|
||||
|
||||
return (
|
||||
<Select
|
||||
value={engineId}
|
||||
disabled={disabled || !validEngines.length}
|
||||
placeholder={placeholder}
|
||||
onChange={handleChange}
|
||||
sx={{ minWidth: 200 }}
|
||||
>
|
||||
{validEngines.map(({ engineId, label, vendorType }) => (
|
||||
<Option key={engineId} value={engineId} label={label}>
|
||||
{label}
|
||||
{!label.toLowerCase().includes(vendorType) && (
|
||||
<Typography level='body-xs' sx={{ ml: 1, color: 'text.tertiary' }}>
|
||||
({speexVendorTypeLabel(vendorType)})
|
||||
</Typography>
|
||||
)}
|
||||
</Option>
|
||||
))}
|
||||
</Select>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* SpeexVoiceAutocomplete - Combined voice/model selector + free-form input
|
||||
*
|
||||
* Uses MUI Joy Autocomplete with freeSolo to allow:
|
||||
* - Selecting from fetched voice/model list (suggestions)
|
||||
* - Typing custom value (free-form)
|
||||
*
|
||||
* Used by LocalAI where models can be selected from list or typed manually.
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Autocomplete, AutocompleteOption, Box, CircularProgress, IconButton, Typography } from '@mui/joy';
|
||||
import RefreshRoundedIcon from '@mui/icons-material/RefreshRounded';
|
||||
|
||||
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
|
||||
|
||||
import type { DSpeexEngineAny, SpeexListVoiceOption } from '../speex.types';
|
||||
import { useSpeexVoices } from './useSpeexVoices';
|
||||
|
||||
|
||||
interface SpeexVoiceAutocompleteProps {
|
||||
engine: DSpeexEngineAny;
|
||||
/** Current value (can be from list or custom) */
|
||||
value: string | undefined;
|
||||
/** Called when value changes (selection or typed). undefined = cleared */
|
||||
onValueChange: (value: string | undefined) => void;
|
||||
/** Placeholder text */
|
||||
placeholder?: string;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
|
||||
export function SpeexVoiceAutocomplete(props: SpeexVoiceAutocompleteProps) {
|
||||
const { engine, value /* e.g. ttsModel */, onValueChange, placeholder = 'Select or type...', disabled } = props;
|
||||
|
||||
// fetch voices/models
|
||||
const { voices, isLoading, error, refetch } = useSpeexVoices(engine);
|
||||
|
||||
// local input state for freeSolo
|
||||
const [inputValue, setInputValue] = React.useState(value ?? '');
|
||||
|
||||
// sync input when value prop changes externally
|
||||
React.useEffect(() => {
|
||||
setInputValue(value ?? '');
|
||||
}, [value]);
|
||||
|
||||
|
||||
// handlers
|
||||
|
||||
const handleChange = React.useCallback((_event: unknown, newValue: string | SpeexListVoiceOption | null) => {
|
||||
// newValue can be: string (typed), SpeexListVoiceOption (selected), or null (cleared)
|
||||
if (newValue === null)
|
||||
onValueChange(undefined);
|
||||
else if (typeof newValue === 'string')
|
||||
onValueChange(newValue || undefined);
|
||||
else
|
||||
onValueChange(newValue.id || undefined);
|
||||
}, [onValueChange]);
|
||||
|
||||
const handleInputChange = React.useCallback((_event: unknown, newInputValue: string, reason: string) => {
|
||||
// BUGFIX: when re-clicking on the same option on the popup, reason will be 'reset', but the inputValue
|
||||
// will be the label of the selected option and not the value. This fixes it
|
||||
if (reason !== 'input')
|
||||
return;
|
||||
|
||||
setInputValue(newInputValue);
|
||||
// For freeSolo, also update value on input change (typing)
|
||||
onValueChange(newInputValue || undefined);
|
||||
}, [onValueChange]);
|
||||
|
||||
|
||||
return (
|
||||
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
|
||||
|
||||
{/* Refresh button (only if refetch available) */}
|
||||
{refetch && (
|
||||
<TooltipOutlined color={error ? 'danger' : undefined} title={error ? <pre>{error}</pre> : 'Refresh'}>
|
||||
<IconButton
|
||||
color={error ? 'danger' : 'neutral'}
|
||||
variant='plain'
|
||||
disabled={isLoading}
|
||||
onClick={() => refetch()}
|
||||
>
|
||||
{!isLoading ? <RefreshRoundedIcon /> : <CircularProgress size='sm' />}
|
||||
</IconButton>
|
||||
</TooltipOutlined>
|
||||
)}
|
||||
|
||||
{/* Autocomplete */}
|
||||
<Autocomplete<SpeexListVoiceOption, false, false, true>
|
||||
freeSolo
|
||||
openOnFocus
|
||||
clearOnEscape
|
||||
disabled={disabled}
|
||||
placeholder={placeholder}
|
||||
|
||||
options={voices}
|
||||
getOptionKey={(option) => typeof option === 'string' ? option : option.id}
|
||||
getOptionLabel={(option) => typeof option === 'string' ? option : option.name}
|
||||
isOptionEqualToValue={(option, val) => option.id === (typeof val === 'string' ? val : val.id)}
|
||||
value={voices.find(o => o.id === value) ?? (value || null)}
|
||||
onChange={handleChange}
|
||||
|
||||
inputValue={inputValue}
|
||||
onInputChange={handleInputChange}
|
||||
|
||||
loading={isLoading}
|
||||
renderOption={(optionProps, option) => {
|
||||
const { key, ...rest } = optionProps as any;
|
||||
return (
|
||||
<AutocompleteOption key={key} {...rest} sx={{ display: 'block' }}>
|
||||
<Typography level='title-sm'>{option.name}</Typography>
|
||||
{option.description && (
|
||||
<Typography level='body-xs' sx={{ opacity: 0.6 }}>{option.description}</Typography>
|
||||
)}
|
||||
</AutocompleteOption>
|
||||
);
|
||||
}}
|
||||
slotProps={{
|
||||
root: {
|
||||
sx: { minWidth: 180, maxWidth: 220, flexGrow: 1 },
|
||||
},
|
||||
listbox: {
|
||||
sx: { maxWidth: 'min(400px, calc(100dvw - 1rem))' },
|
||||
},
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import * as React from 'react';
|
||||
|
||||
import { Box, CircularProgress, IconButton, Option, optionClasses, Select, SelectSlotsAndSlotProps } from '@mui/joy';
|
||||
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
|
||||
import RefreshRoundedIcon from '@mui/icons-material/RefreshRounded';
|
||||
|
||||
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
|
||||
import { TooltipOutlined } from '~/common/components/TooltipOutlined';
|
||||
|
||||
import type { DSpeexEngineAny } from '../speex.types';
|
||||
import { useSpeexVoices } from './useSpeexVoices';
|
||||
|
||||
|
||||
// copied from useLLMSelect.tsx - inspired by optimaSelectSlotProps.listbox
|
||||
const _selectSlotProps: SelectSlotsAndSlotProps<false>['slotProps'] = {
|
||||
root: {
|
||||
sx: {
|
||||
minWidth: 220, // 180 = 220 - 36 - 4
|
||||
},
|
||||
},
|
||||
button: {
|
||||
className: 'agi-ellipsize',
|
||||
sx: {
|
||||
// these + the ellipsize class will ellipsize the text in the button
|
||||
display: 'inline-block',
|
||||
textAlign: 'start',
|
||||
// maxWidth: 220,
|
||||
} as const,
|
||||
},
|
||||
listbox: {
|
||||
// size: 'md',
|
||||
// className: 'agi-ellipsize',
|
||||
sx: {
|
||||
boxShadow: 'xl',
|
||||
// Option: clip width to 200...360px
|
||||
[`& .${optionClasses.root}`]: {
|
||||
// minWidth: 300,
|
||||
maxWidth: 'min(640px, calc(100dvw - 0.25rem))', // the small reduction is to avoid accidental h-scrolling because of the border
|
||||
},
|
||||
},
|
||||
} as const,
|
||||
} as const;
|
||||
|
||||
|
||||
export function SpeexVoiceSelect(props: {
|
||||
engine: DSpeexEngineAny;
|
||||
voiceId: string | null;
|
||||
onVoiceChange: (voiceId: string) => void;
|
||||
disabled?: boolean;
|
||||
autoPreview?: boolean;
|
||||
}) {
|
||||
|
||||
// props
|
||||
const { engine, voiceId, onVoiceChange, disabled, autoPreview } = props;
|
||||
|
||||
// external state - module
|
||||
const { voices, isLoading, error, refetch } = useSpeexVoices(engine);
|
||||
|
||||
// track user-initiated voice changes for preview (not initial load or voice list changes)
|
||||
const [userSelectedVoiceId, setUserSelectedVoiceId] = React.useState<string | null>(null);
|
||||
|
||||
|
||||
// [effect] auto-preview: play voice sample only when user explicitly selects a voice
|
||||
const selectedVoice = userSelectedVoiceId ? voices.find(v => v.id === userSelectedVoiceId) : null;
|
||||
const previewUrl = (autoPreview && selectedVoice?.previewUrl) || null;
|
||||
React.useEffect(() => {
|
||||
if (previewUrl)
|
||||
void AudioPlayer.playUrl(previewUrl);
|
||||
}, [previewUrl]);
|
||||
|
||||
|
||||
// handlers
|
||||
|
||||
const handleVoiceChange = React.useCallback((_event: unknown, value: string | null) => {
|
||||
setUserSelectedVoiceId(value);
|
||||
value && onVoiceChange(value);
|
||||
}, [onVoiceChange]);
|
||||
|
||||
return <Box sx={{ display: 'flex', alignItems: 'center' }}>
|
||||
|
||||
{refetch && (
|
||||
<TooltipOutlined color={error ? 'danger' : undefined} title={error ? <pre>{error}</pre> : 'Refresh voices'}>
|
||||
<IconButton
|
||||
color={error ? 'danger' : 'neutral'}
|
||||
variant='plain'
|
||||
disabled={isLoading}
|
||||
onClick={() => refetch()}
|
||||
>
|
||||
{!isLoading ? <RefreshRoundedIcon /> : <CircularProgress size='sm' />}
|
||||
</IconButton>
|
||||
</TooltipOutlined>
|
||||
)}
|
||||
|
||||
<Select
|
||||
variant='outlined'
|
||||
disabled={disabled || isLoading || voices.length === 0}
|
||||
value={voiceId ?? null}
|
||||
onChange={handleVoiceChange}
|
||||
placeholder={
|
||||
error ? 'Error loading voices'
|
||||
: isLoading ? 'Loading...'
|
||||
: voices.length === 0 ? 'No voices available'
|
||||
: voiceId ? `Voice ${voiceId.slice(0, 12)}...`
|
||||
: 'Select a voice'
|
||||
}
|
||||
// startDecorator={<PhVoice />}
|
||||
// endDecorator={isLoading && <CircularProgress size='sm' />}
|
||||
indicator={<KeyboardArrowDownIcon />}
|
||||
slotProps={_selectSlotProps}
|
||||
>
|
||||
{voices.map(voice => (
|
||||
<Option key={voice.id} value={voice.id} label={voice.name.split('-')[0]}>
|
||||
<div className='agi-ellipsize'>
|
||||
{voice.name} {voice.description && <span style={{ marginLeft: '0.75rem', opacity: 0.5, fontSize: 'smaller' }}>{voice.description}</span>}
|
||||
</div>
|
||||
</Option>
|
||||
))}
|
||||
</Select>
|
||||
|
||||
</Box>;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
|
||||
import type { DSpeexEngineAny, SpeexListVoiceOption, SpeexListVoicesResult } from '../speex.types';
|
||||
import { speexListVoices_RPC_orThrow } from '../protocols/rpc/rpc.client';
|
||||
import { useSpeexWebSpeechVoices } from '../protocols/webspeech/webspeech.client';
|
||||
|
||||
|
||||
const _stableEmptyVoices: SpeexListVoiceOption[] = [] as const;
|
||||
|
||||
// returns voices given an engine
|
||||
export function useSpeexVoices(engine: DSpeexEngineAny): SpeexListVoicesResult {
|
||||
|
||||
// props
|
||||
const { vendorType, engineId } = engine;
|
||||
const isWebspeech = vendorType === 'webspeech';
|
||||
|
||||
// use browser voices
|
||||
const browserVoicesResult = useSpeexWebSpeechVoices(isWebspeech);
|
||||
|
||||
// use RPC voices
|
||||
const { data: cloudVoices, error: cloudError, isFetching: cloudIsFetching, refetch } = useQuery({
|
||||
enabled: !isWebspeech,
|
||||
queryKey: ['speex', 'listVoices', engineId, vendorType],
|
||||
queryFn: () => speexListVoices_RPC_orThrow(engine as any /* will not run for 'webspeech' */),
|
||||
staleTime: 5 * 60 * 1000, // 5 minutes - voices don't change often
|
||||
});
|
||||
|
||||
// do not refetch openai, voices are hardcoded
|
||||
const needsRefetch = vendorType !== 'openai';
|
||||
|
||||
// switch result
|
||||
return isWebspeech ? browserVoicesResult : {
|
||||
voices: cloudVoices?.length ? cloudVoices : _stableEmptyVoices,
|
||||
isLoading: cloudIsFetching,
|
||||
error: cloudError instanceof Error ? cloudError.message : null,
|
||||
refetch: needsRefetch ? refetch : undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Speex RPC Client
|
||||
*
|
||||
* Handles communication with speex.router for cloud TTS providers.
|
||||
* Resolves credentials from engine configuration and calls the streaming API.
|
||||
*/
|
||||
|
||||
import { apiAsync, apiStream } from '~/common/util/trpc.client';
|
||||
import { convert_Base64_To_UInt8Array, convert_UInt8Array_To_Base64 } from '~/common/util/blobUtils';
|
||||
import { findModelsServiceOrNull } from '~/common/stores/llms/store-llms';
|
||||
import { stripUndefined } from '~/common/util/objectUtils';
|
||||
|
||||
import type { DLocalAIServiceSettings } from '~/modules/llms/vendors/localai/localai.vendor';
|
||||
import type { DOpenAIServiceSettings } from '~/modules/llms/vendors/openai/openai.vendor';
|
||||
|
||||
import { AudioLivePlayer } from '~/common/util/audio/AudioLivePlayer';
|
||||
import { AudioPlayer } from '~/common/util/audio/AudioPlayer';
|
||||
|
||||
import type { DSpeexEngine, SpeexListVoiceOption, SpeexSpeakResult } from '../../speex.types';
|
||||
import type { SpeexWire_Access, SpeexWire_Voice } from './rpc.wiretypes';
|
||||
import { SPEEX_DEBUG } from '../../speex.config';
|
||||
|
||||
|
||||
type _DSpeexEngineRPC = DSpeexEngine<'elevenlabs'> | DSpeexEngine<'localai'> | DSpeexEngine<'openai'>;
|
||||
|
||||
|
||||
/**
|
||||
* Synthesize speech via speex.router (streaming)
|
||||
*/
|
||||
export async function speexSynthesize_RPC(
|
||||
engine: _DSpeexEngineRPC,
|
||||
text: string,
|
||||
options: {
|
||||
streaming: boolean;
|
||||
languageCode?: string;
|
||||
priority?: 'fast' | 'balanced' | 'quality';
|
||||
playback: boolean;
|
||||
returnAudio: boolean;
|
||||
},
|
||||
callbacks?: {
|
||||
onStart?: () => void;
|
||||
onChunk?: (chunk: ArrayBuffer) => void;
|
||||
onComplete?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
},
|
||||
): Promise<SpeexSpeakResult> {
|
||||
|
||||
// engine credentials (DCredentials..) -> wire Access
|
||||
if (SPEEX_DEBUG) console.log(`[Speex RPC] Synthesize request (engine: ${engine.engineId}, ${text.length} chars) - options:`, options);
|
||||
const access = stripUndefined(_buildRPCWireAccess(engine));
|
||||
if (!access) {
|
||||
const error = new Error(`Failed to resolve credentials for engine ${engine.engineId}`);
|
||||
callbacks?.onError?.(error);
|
||||
return { success: false, errorType: 'tts-unconfigured', error: error.message };
|
||||
}
|
||||
|
||||
// engine voice -> wire Voice
|
||||
// IMPORTANT: TS ensures structural compatibility here between the DVoice* and Voice*_schema types
|
||||
const voice: SpeexWire_Voice = stripUndefined(engine.voice);
|
||||
|
||||
|
||||
// audio player for streaming playback
|
||||
let audioPlayer: AudioLivePlayer | null = null;
|
||||
const audioChunks: ArrayBuffer[] = [];
|
||||
|
||||
const abortController = new AbortController();
|
||||
|
||||
try {
|
||||
|
||||
// call the streaming RPC - whether the backend will stream in chunks or as a whole
|
||||
const particleStream = await apiStream.speex.synthesize.mutate({
|
||||
access,
|
||||
text,
|
||||
voice,
|
||||
streaming: options.streaming,
|
||||
...(options.languageCode && { languageCode: options.languageCode }),
|
||||
...(options.priority && { priority: options.priority }),
|
||||
}, {
|
||||
signal: abortController.signal,
|
||||
});
|
||||
|
||||
// process streaming particles
|
||||
for await (const particle of particleStream) {
|
||||
if (SPEEX_DEBUG) console.log('[Speex RPC] <-', particle);
|
||||
switch (particle.t) {
|
||||
case 'start':
|
||||
callbacks?.onStart?.();
|
||||
if (options.playback && options.streaming)
|
||||
audioPlayer = new AudioLivePlayer();
|
||||
break;
|
||||
|
||||
case 'audio':
|
||||
// Decode base64 to ArrayBuffer
|
||||
const audioData = convert_Base64_To_UInt8Array(particle.base64, 'speex.rpc.client');
|
||||
|
||||
// Accumulate for return (copy bytes before playback may transfer/detach the buffer)
|
||||
if (options.returnAudio)
|
||||
audioChunks.push(audioData.slice().buffer);
|
||||
|
||||
// Playback: streaming uses AudioLivePlayer for chunked playback,
|
||||
// non-streaming uses AudioPlayer for single-buffer playback
|
||||
if (options.playback) {
|
||||
if (particle.chunk) {
|
||||
// create the player on-demand, however in the near future we'll migrate to
|
||||
// Northbridge AudioPlayer for all playback needs
|
||||
if (!audioPlayer)
|
||||
audioPlayer = new AudioLivePlayer();
|
||||
|
||||
audioPlayer.enqueueChunk(audioData.buffer);
|
||||
} else {
|
||||
// also consider merging LiveAudioPlayer into AudioPlayer - note this will throw on malformed base64 data
|
||||
void AudioPlayer.playBuffer(audioData.buffer); // fire-and-forget for whole audio
|
||||
}
|
||||
}
|
||||
|
||||
// Callback
|
||||
callbacks?.onChunk?.(audioData.buffer);
|
||||
break;
|
||||
|
||||
case 'log':
|
||||
// intended to be user visible
|
||||
console.log(`[Speex] (${particle.level})`, particle.message);
|
||||
break;
|
||||
|
||||
case 'done':
|
||||
const { chars, audioBytes, durationMs } = particle;
|
||||
if (SPEEX_DEBUG) console.log(`[Speex RPC] Synthesis done: ${chars} chars, ${audioBytes} bytes, ${durationMs} ms`);
|
||||
|
||||
// NOTE: calling this will end the sound abruptly if the final chunk is still playing, so we don't do it for now
|
||||
audioPlayer?.endPlayback();
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
// noinspection ExceptionCaughtLocallyJS
|
||||
throw new Error(particle.e);
|
||||
}
|
||||
}
|
||||
|
||||
callbacks?.onComplete?.();
|
||||
|
||||
// build result
|
||||
const result: SpeexSpeakResult = { success: true };
|
||||
|
||||
if (options.returnAudio && audioChunks.length > 0) {
|
||||
// Concatenate all chunks and convert to base64
|
||||
const totalLength = audioChunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of audioChunks) {
|
||||
combined.set(new Uint8Array(chunk), offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
result.audioBase64 = convert_UInt8Array_To_Base64(combined, 'speex.rpc.client');
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error: any) {
|
||||
if (SPEEX_DEBUG) console.error('[Speex RPC] Synthesis error:', { error });
|
||||
|
||||
// cleanup
|
||||
if (audioPlayer)
|
||||
void audioPlayer.stop();
|
||||
|
||||
const errorMessage = error.message || 'Synthesis failed';
|
||||
callbacks?.onError?.(new Error(errorMessage));
|
||||
return { success: false, errorType: 'tts-exception', error: errorMessage };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* List voices via speex.router
|
||||
*/
|
||||
export async function speexListVoices_RPC_orThrow(engine: _DSpeexEngineRPC): Promise<SpeexListVoiceOption[]> {
|
||||
const access = _buildRPCWireAccess(engine);
|
||||
if (!access)
|
||||
return [];
|
||||
|
||||
return (await apiAsync.speex.listVoices.query({ access })).voices;
|
||||
}
|
||||
|
||||
|
||||
// -- private helpers --
|
||||
|
||||
function _buildRPCWireAccess({ credentials: c, vendorType }: _DSpeexEngineRPC): SpeexWire_Access | null {
|
||||
switch (c.type) {
|
||||
|
||||
// resolve from inline API keys
|
||||
case 'api-key':
|
||||
switch (vendorType) {
|
||||
case 'elevenlabs':
|
||||
return {
|
||||
dialect: 'elevenlabs',
|
||||
apiKey: c.apiKey,
|
||||
...(c.apiHost && { apiHost: c.apiHost }),
|
||||
};
|
||||
|
||||
case 'localai':
|
||||
case 'openai':
|
||||
return {
|
||||
dialect: vendorType,
|
||||
...(c.apiKey && { apiKey: c.apiKey }),
|
||||
...(c.apiHost && { apiHost: c.apiHost }),
|
||||
// ...(c.apiOrgId && { apiOrgId: c.apiOrgId }),
|
||||
};
|
||||
|
||||
default:
|
||||
const _exhaustiveCheck: never = vendorType;
|
||||
return null;
|
||||
}
|
||||
|
||||
// resolve from LLMs services
|
||||
case 'llms-service':
|
||||
const service = findModelsServiceOrNull(c.serviceId);
|
||||
if (!service) return null;
|
||||
switch (vendorType) {
|
||||
case 'elevenlabs':
|
||||
// no linking for ElevenLabs - we shall NOT be here
|
||||
return null;
|
||||
|
||||
case 'openai':
|
||||
const oai = (service.setup || {}) as DOpenAIServiceSettings;
|
||||
return {
|
||||
dialect: vendorType,
|
||||
...(oai.oaiKey && { apiKey: oai.oaiKey }),
|
||||
...(oai.oaiHost && { apiHost: oai.oaiHost }),
|
||||
...(oai.oaiOrg && { apiOrgId: oai.oaiOrg }),
|
||||
};
|
||||
|
||||
case 'localai':
|
||||
const lai = (service.setup || {}) as DLocalAIServiceSettings;
|
||||
return {
|
||||
dialect: vendorType,
|
||||
...(lai.localAIKey && { apiKey: lai.localAIKey }),
|
||||
...(lai.localAIHost && { apiHost: lai.localAIHost }),
|
||||
};
|
||||
|
||||
default:
|
||||
const _exhaustiveCheck: never = vendorType;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { createTRPCRouter, edgeProcedure } from '~/server/trpc/trpc.server';
|
||||
|
||||
import { SpeexSpeechParticle, SpeexWire, SpeexWire_Access, SpeexWire_ListVoices_Output, SpeexWire_Voice } from './rpc.wiretypes';
|
||||
import { listVoicesElevenLabs, synthesizeElevenLabs } from './synthesize-elevenlabs';
|
||||
import { listVoicesLocalAIOrThrow, listVoicesOpenAI, synthesizeOpenAIProtocol } from './synthesize-openai';
|
||||
|
||||
|
||||
interface SynthesizeBackendFnParams<TSpeexAccess extends SpeexWire_Access> {
|
||||
access: TSpeexAccess;
|
||||
text: string;
|
||||
voice: SpeexWire_Voice;
|
||||
streaming: boolean;
|
||||
languageCode?: string;
|
||||
priority?: 'fast' | 'balanced' | 'quality';
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export type SynthesizeBackendFn<TSpeexAccess extends SpeexWire_Access> = (params: SynthesizeBackendFnParams<TSpeexAccess>) => AsyncGenerator<SpeexSpeechParticle>;
|
||||
|
||||
|
||||
export const speexRouter = createTRPCRouter({
|
||||
|
||||
/**
|
||||
* Speech synthesis - streaming AsyncGenerator
|
||||
* Yields SpeexParticle chunks: start, audio, done, error
|
||||
*/
|
||||
synthesize: edgeProcedure
|
||||
.input(SpeexWire.Synthesize_input_schema)
|
||||
.mutation(async function* ({ input, ctx }): AsyncGenerator<SpeexSpeechParticle> {
|
||||
const { access, text, voice, streaming, languageCode, priority } = input;
|
||||
|
||||
try {
|
||||
yield { t: 'start' };
|
||||
switch (access.dialect) {
|
||||
case 'elevenlabs':
|
||||
yield* synthesizeElevenLabs({ access, text, voice, streaming, languageCode, priority, signal: ctx.reqSignal });
|
||||
break;
|
||||
|
||||
case 'localai':
|
||||
case 'openai':
|
||||
yield* synthesizeOpenAIProtocol({ access, text, voice, streaming, languageCode, priority, signal: ctx.reqSignal });
|
||||
break;
|
||||
|
||||
default:
|
||||
const _exhaustiveCheck: never = access;
|
||||
}
|
||||
} catch (error) {
|
||||
yield { t: 'error', e: error instanceof Error ? error.message : 'Synthesis failed' };
|
||||
}
|
||||
}),
|
||||
|
||||
/**
|
||||
* List available voices for a dialect
|
||||
*/
|
||||
listVoices: edgeProcedure
|
||||
.input(SpeexWire.ListVoices_input_schema)
|
||||
.query(async ({ input }): Promise<SpeexWire_ListVoices_Output> => {
|
||||
const { access } = input;
|
||||
|
||||
switch (access.dialect) {
|
||||
case 'elevenlabs':
|
||||
return await listVoicesElevenLabs(access);
|
||||
|
||||
case 'openai':
|
||||
return { voices: listVoicesOpenAI() };
|
||||
|
||||
case 'localai':
|
||||
return await listVoicesLocalAIOrThrow(access);
|
||||
|
||||
default:
|
||||
const _exhaustiveCheck: never = access;
|
||||
return { voices: [] };
|
||||
}
|
||||
}),
|
||||
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Shared streaming utilities for Speex RPC synthesizers
|
||||
*
|
||||
* Provides common streaming chunk accumulation logic used by
|
||||
* ElevenLabs, OpenAI, and LocalAI synthesizers.
|
||||
*/
|
||||
|
||||
import type { SpeexSpeechParticle } from './rpc.wiretypes';
|
||||
|
||||
|
||||
/**
|
||||
* Streams audio chunks from a Response, accumulating to minimum size before yielding.
|
||||
*
|
||||
* @param response - Fetch Response with audio body
|
||||
* @param minChunkSize - Minimum bytes to accumulate before yielding (default 4096)
|
||||
* @param textLength - Original text length for 'done' particle
|
||||
*/
|
||||
export async function* streamAudioChunksOrThrow(
|
||||
response: Response,
|
||||
minChunkSize: number,
|
||||
textLength: number,
|
||||
): AsyncGenerator<SpeexSpeechParticle> {
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader)
|
||||
return yield { t: 'error', e: 'No stream reader available' };
|
||||
|
||||
try {
|
||||
const accumulatedChunks: Uint8Array[] = [];
|
||||
let accumulatedSize = 0;
|
||||
let totalAudioBytes = 0;
|
||||
|
||||
while (true) {
|
||||
const { value, done: readerDone } = await reader.read();
|
||||
if (readerDone) break;
|
||||
if (!value) continue;
|
||||
|
||||
// Accumulate chunks
|
||||
accumulatedChunks.push(value);
|
||||
accumulatedSize += value.length;
|
||||
|
||||
// Yield when accumulated size reaches threshold
|
||||
if (accumulatedSize >= minChunkSize) {
|
||||
yield { t: 'audio', base64: Buffer.concat(accumulatedChunks).toString('base64'), chunk: true };
|
||||
totalAudioBytes += accumulatedSize;
|
||||
accumulatedChunks.length = 0;
|
||||
accumulatedSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Yield any remaining data as final chunk
|
||||
if (accumulatedSize > 0) {
|
||||
yield { t: 'audio', base64: Buffer.concat(accumulatedChunks).toString('base64'), chunk: true /*, final: true*/ };
|
||||
totalAudioBytes += accumulatedSize;
|
||||
}
|
||||
|
||||
yield { t: 'done', chars: textLength, audioBytes: totalAudioBytes };
|
||||
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns entire audio response as a single chunk (non-streaming mode).
|
||||
* Includes optional metadata from response headers.
|
||||
*/
|
||||
export async function* returnAudioWholeOrThrow(
|
||||
response: Response,
|
||||
textLength: number,
|
||||
audioMeta?: Pick<Extract<SpeexSpeechParticle, { t: 'audio' }>, 'contentType' | 'characterCost' | 'ttsLatencyMs'>,
|
||||
): AsyncGenerator<SpeexSpeechParticle> {
|
||||
|
||||
const audioArrayBuffer = await response.arrayBuffer();
|
||||
yield {
|
||||
t: 'audio',
|
||||
base64: Buffer.from(audioArrayBuffer).toString('base64'),
|
||||
chunk: false,
|
||||
...(audioMeta?.contentType ? { contentType: audioMeta.contentType } : {}),
|
||||
...(audioMeta?.characterCost ? { characterCost: audioMeta.characterCost } : {}),
|
||||
...(audioMeta?.ttsLatencyMs ? { ttsLatencyMs: audioMeta.ttsLatencyMs } : {}),
|
||||
};
|
||||
|
||||
yield { t: 'done', chars: textLength, audioBytes: audioArrayBuffer.byteLength };
|
||||
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
import * as z from 'zod/v4';
|
||||
|
||||
|
||||
/**
|
||||
* Streaming Speech Synthesis Particle (TS-only) schema
|
||||
*/
|
||||
export type SpeexSpeechParticle =
|
||||
| { t: 'start' }
|
||||
| { t: 'audio'; base64: string; chunk: boolean; contentType?: string; characterCost?: number; ttsLatencyMs?: number }
|
||||
| { t: 'done'; durationMs?: number; chars: number; audioBytes: number }
|
||||
| { t: 'log'; level: 'info', message: string }
|
||||
| { t: 'error'; e: string }
|
||||
;
|
||||
|
||||
|
||||
export type SpeexWire_Access = z.infer<typeof SpeexWire.Access_schema>;
|
||||
export type SpeexWire_Access_ElevenLabs = z.infer<typeof SpeexWire.AccessElevenLabs_schema>;
|
||||
export type SpeexWire_Access_OpenAI = z.infer<typeof SpeexWire.AccessOpenAI_schema>;
|
||||
|
||||
export type SpeexWire_Voice = z.infer<typeof SpeexWire.Voice_schema>;
|
||||
|
||||
export type SpeexWire_Synthesize_Input = z.infer<typeof SpeexWire.Synthesize_input_schema>;
|
||||
|
||||
export type SpeexWire_VoiceOption = z.infer<typeof SpeexWire.VoiceOption_schema>;
|
||||
export type SpeexWire_ListVoices_Input = z.infer<typeof SpeexWire.ListVoices_input_schema>;
|
||||
export type SpeexWire_ListVoices_Output = z.infer<typeof SpeexWire.ListVoices_output_schema>;
|
||||
|
||||
|
||||
/**
|
||||
* Wire Protocol Schemas for Speex module
|
||||
*/
|
||||
export namespace SpeexWire {
|
||||
|
||||
// Access schemas - discriminated union by dialect
|
||||
|
||||
export const AccessElevenLabs_schema = z.object({
|
||||
dialect: z.literal('elevenlabs'),
|
||||
apiKey: z.string(),
|
||||
apiHost: z.string().optional(),
|
||||
});
|
||||
|
||||
export const AccessOpenAI_schema = z.object({
|
||||
dialect: z.enum(['localai', 'openai']),
|
||||
apiKey: z.string().optional(), // openai: required, localai: optional
|
||||
apiHost: z.string().optional(), // localai: required, openai: optional
|
||||
apiOrgId: z.string().optional(), // openai only
|
||||
});
|
||||
|
||||
export const Access_schema = z.discriminatedUnion('dialect',
|
||||
[AccessElevenLabs_schema, AccessOpenAI_schema],
|
||||
);
|
||||
|
||||
|
||||
// Voice schemas - per dialect
|
||||
|
||||
export const VoiceElevenLabs_schema = z.object({
|
||||
dialect: z.literal('elevenlabs'),
|
||||
ttsModel: z.string().optional(),
|
||||
ttsVoiceId: z.string().optional(),
|
||||
});
|
||||
|
||||
export const VoiceLocalAI_schema = z.object({
|
||||
dialect: z.literal('localai'),
|
||||
ttsBackend: z.string().optional(), // e.g., 'coqui', 'bark', 'piper', 'vall-e-x'
|
||||
ttsModel: z.string().optional(), // e.g., 'kokoro', 'tts_models/en/ljspeech/glow-tts'
|
||||
ttsLanguage: z.string().optional(), // for multilingual models like xtts_v2
|
||||
});
|
||||
|
||||
export const VoiceOpenAI_schema = z.object({
|
||||
dialect: z.literal('openai'),
|
||||
ttsModel: z.enum(['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts']).optional(),
|
||||
ttsVoiceId: z.string().optional(),
|
||||
ttsSpeed: z.number().min(0.25).max(4.0).optional(),
|
||||
ttsInstruction: z.string().optional(),
|
||||
});
|
||||
|
||||
export const Voice_schema = z.discriminatedUnion('dialect',
|
||||
[VoiceElevenLabs_schema, VoiceLocalAI_schema, VoiceOpenAI_schema],
|
||||
);
|
||||
|
||||
|
||||
// .Synthesize input schema
|
||||
|
||||
export const Synthesize_input_schema = z.object({
|
||||
access: SpeexWire.Access_schema,
|
||||
text: z.string(),
|
||||
voice: SpeexWire.Voice_schema,
|
||||
streaming: z.boolean(),
|
||||
languageCode: z.string().optional(), // ISO language code (e.g., 'en', 'fr') for model selection fallback
|
||||
priority: z.enum(['fast', 'balanced', 'quality']).optional(), // Hint for speed vs quality tradeoff
|
||||
});
|
||||
|
||||
|
||||
// .ListVoices voice schema
|
||||
|
||||
export const VoiceOption_schema = z.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
description: z.string().optional(),
|
||||
previewUrl: z.string().optional(),
|
||||
category: z.string().optional(), // e.g., 'premade', 'cloned', 'professional'
|
||||
// Voice labels (flattened for simplicity)
|
||||
// gender: z.string().optional(), // e.g., 'male', 'female', 'neutral'
|
||||
// accent: z.string().optional(), // e.g., 'american', 'british', 'australian'
|
||||
// age: z.string().optional(), // e.g., 'young', 'middle_aged', 'old'
|
||||
// language: z.string().optional(), // e.g., 'en', 'es', 'multilingual'
|
||||
});
|
||||
|
||||
export const ListVoices_input_schema = z.object({
|
||||
access: SpeexWire.Access_schema,
|
||||
});
|
||||
|
||||
export const ListVoices_output_schema = z.object({
|
||||
voices: z.array(VoiceOption_schema),
|
||||
});
|
||||
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
import * as z from 'zod/v4';
|
||||
|
||||
import { fetchJsonOrTRPCThrow, fetchResponseOrTRPCThrow } from '~/server/trpc/trpc.router.fetchers';
|
||||
|
||||
import type { SpeexSpeechParticle, SpeexWire_Access_ElevenLabs, SpeexWire_ListVoices_Output } from './rpc.wiretypes';
|
||||
import type { SynthesizeBackendFn } from './rpc.router';
|
||||
import { SPEEX_DEBUG, SPEEX_DEFAULTS } from '../../speex.config';
|
||||
import { returnAudioWholeOrThrow, streamAudioChunksOrThrow } from './rpc.streaming';
|
||||
|
||||
|
||||
// configuration
|
||||
const SAFETY_TEXT_LENGTH = 1000;
|
||||
const MIN_CHUNK_SIZE = 4096;
|
||||
|
||||
|
||||
const _selectModel = (priority: 'fast' | 'balanced' | 'quality' | undefined, languageCode: string | undefined): string => {
|
||||
const fast = SPEEX_DEFAULTS.ELEVENLABS_MODEL_FAST;
|
||||
const quality = SPEEX_DEFAULTS.ELEVENLABS_MODEL;
|
||||
return priority === 'fast' ? fast // lowest latency, best for real-time use cases like calls
|
||||
: priority === 'quality' ? quality // multilingual v2 (highest quality)
|
||||
: languageCode?.toLowerCase() === 'en' ? fast : quality; // 'balanced'/undefined: English → turbo, non-English → multilingual
|
||||
};
|
||||
|
||||
|
||||
export const synthesizeElevenLabs: SynthesizeBackendFn<SpeexWire_Access_ElevenLabs> = async function* (params) {
|
||||
|
||||
// destructure and validate
|
||||
const { access, text: inputText, voice, streaming, languageCode, priority, signal } = params;
|
||||
if (access.dialect !== 'elevenlabs' || voice.dialect !== 'elevenlabs')
|
||||
throw new Error('Mismatched dialect in ElevenLabs synthesize');
|
||||
|
||||
|
||||
// safety check: trim text that's too long
|
||||
let text = inputText;
|
||||
if (text.length > SAFETY_TEXT_LENGTH) {
|
||||
text = text.slice(0, SAFETY_TEXT_LENGTH);
|
||||
// -> log.info
|
||||
yield { t: 'log', level: 'info', message: `Text truncated to ${SAFETY_TEXT_LENGTH} characters` };
|
||||
}
|
||||
|
||||
// build request - narrow to elevenlabs dialect for type safety
|
||||
const voiceId = voice.ttsVoiceId /*|| env.ELEVENLABS_VOICE_ID*/ || SPEEX_DEFAULTS.ELEVENLABS_VOICE;
|
||||
const model = voice.ttsModel || _selectModel(priority, languageCode);
|
||||
|
||||
const path = `/v1/text-to-speech/${voiceId}${streaming ? '/stream' : ''}`;
|
||||
const { headers, url } = _elevenlabsAccess(access, path);
|
||||
|
||||
const body: ElevenLabsWire.TTS_Request = {
|
||||
text,
|
||||
model_id: model,
|
||||
} as const;
|
||||
|
||||
// Fetch
|
||||
let response: Response;
|
||||
try {
|
||||
if (SPEEX_DEBUG) console.log(`[Speex][ElevenLabs] POST (stream=${streaming})`, { url, headers, body });
|
||||
response = await fetchResponseOrTRPCThrow({
|
||||
url,
|
||||
method: 'POST',
|
||||
headers,
|
||||
body,
|
||||
signal,
|
||||
name: 'ElevenLabs',
|
||||
});
|
||||
} catch (error: any) {
|
||||
yield { t: 'error', e: `ElevenLabs fetch failed: ${error.message || 'Unknown error'}` };
|
||||
return;
|
||||
}
|
||||
|
||||
// Stream or return whole audio (with metadata for non-streaming)
|
||||
try {
|
||||
yield* streaming
|
||||
? streamAudioChunksOrThrow(response, MIN_CHUNK_SIZE, text.length)
|
||||
: returnAudioWholeOrThrow(response, text.length, _parseTTSResponseHeaders(response.headers));
|
||||
} catch (error: any) {
|
||||
yield { t: 'error', e: `ElevenLabs audio error: ${error.message || 'Unknown error'}` };
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
export async function listVoicesElevenLabs(access: SpeexWire_Access_ElevenLabs): Promise<SpeexWire_ListVoices_Output> {
|
||||
const { headers, url } = _elevenlabsAccess(access, '/v1/voices');
|
||||
|
||||
// fetch voices
|
||||
const voicesList = ElevenLabsWire.VoicesList_schema.parse(
|
||||
await fetchJsonOrTRPCThrow({
|
||||
url,
|
||||
headers,
|
||||
name: 'ElevenLabs',
|
||||
}),
|
||||
);
|
||||
|
||||
// map to output
|
||||
const voices = voicesList.voices.map(voice => ({
|
||||
id: voice.voice_id,
|
||||
name: voice.name,
|
||||
description: voice.description || undefined,
|
||||
previewUrl: voice.preview_url || undefined,
|
||||
category: voice.category,
|
||||
// Flatten labels for UI display
|
||||
// gender: voice.labels?.gender || undefined,
|
||||
// accent: voice.labels?.accent || undefined,
|
||||
// age: voice.labels?.age || undefined,
|
||||
// language: voice.labels?.language || undefined,
|
||||
}));
|
||||
|
||||
// inject Rachel (default voice) if not already in the list
|
||||
const rachelId = SPEEX_DEFAULTS.ELEVENLABS_VOICE;
|
||||
if (!voices.some(v => v.id === rachelId))
|
||||
voices.unshift({
|
||||
id: rachelId,
|
||||
name: 'Rachel',
|
||||
description: 'Matter-of-fact, personable woman. Great for conversational use cases.',
|
||||
category: 'premade',
|
||||
previewUrl: undefined,
|
||||
});
|
||||
|
||||
// sort: custom voices first, then premade
|
||||
voices.sort((a, b) => {
|
||||
if (a.category === 'premade' && b.category !== 'premade') return 1;
|
||||
if (a.category !== 'premade' && b.category === 'premade') return -1;
|
||||
return 0;
|
||||
});
|
||||
|
||||
return { voices };
|
||||
}
|
||||
|
||||
|
||||
// Helpers
|
||||
|
||||
function _parseTTSResponseHeaders(headers: Headers): Pick<Extract<SpeexSpeechParticle, { t: 'audio' }>, 'contentType' | 'characterCost' | 'ttsLatencyMs'> {
|
||||
return {
|
||||
contentType: headers.get('content-type') || 'audio/mpeg',
|
||||
characterCost: parseInt(headers.get('character-cost') || '0') || undefined,
|
||||
ttsLatencyMs: parseInt(headers.get('tts-latency-ms') || '0') || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function _elevenlabsAccess(access: SpeexWire_Access_ElevenLabs, apiPath: string): { headers: HeadersInit; url: string } {
|
||||
const apiKey = (access.apiKey /*|| env.ELEVENLABS_API_KEY */ || '').trim();
|
||||
if (!apiKey)
|
||||
throw new Error('Missing ElevenLabs API key');
|
||||
|
||||
let host = (access.apiHost /*|| env.ELEVENLABS_API_HOST*/ || 'api.elevenlabs.io').trim();
|
||||
if (!host.startsWith('http'))
|
||||
host = `https://${host}`;
|
||||
if (host.endsWith('/') && apiPath.startsWith('/'))
|
||||
host = host.slice(0, -1);
|
||||
|
||||
return {
|
||||
headers: {
|
||||
'Accept': 'audio/mpeg',
|
||||
'Content-Type': 'application/json',
|
||||
'xi-api-key': apiKey,
|
||||
},
|
||||
url: host + apiPath,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Wire types for the upstream ElevenLabs API
|
||||
|
||||
namespace ElevenLabsWire {
|
||||
|
||||
// export type VoicesList = z.infer<typeof VoicesList_schema>;
|
||||
export const VoicesList_schema = z.object({
|
||||
voices: z.array(z.object({
|
||||
voice_id: z.string(),
|
||||
name: z.string(),
|
||||
category: z.enum(['premade', 'cloned', 'professional']).or(z.string()),
|
||||
labels: z.looseObject({
|
||||
gender: z.enum(['male', 'female', 'neutral']).or(z.string()).nullish(),
|
||||
accent: z.string().nullish(),
|
||||
age: z.string().nullish(),
|
||||
language: z.string().nullish(),
|
||||
}),
|
||||
description: z.string().nullish(),
|
||||
preview_url: z.string().nullish(),
|
||||
settings: z.object({
|
||||
stability: z.number(),
|
||||
similarity_boost: z.number(),
|
||||
}).nullish(),
|
||||
// high_quality_base_model_ids: z.array(z.string()).nullish(),
|
||||
is_owner: z.boolean().nullish(),
|
||||
is_legacy: z.boolean().nullish(),
|
||||
})),
|
||||
});
|
||||
|
||||
export type TTS_Request = z.infer<typeof TTS_Request_schema>;
|
||||
export const TTS_Request_schema = z.object({
|
||||
text: z.string(),
|
||||
model_id: z.string().optional(),
|
||||
voice_settings: z.object({
|
||||
stability: z.number(),
|
||||
similarity_boost: z.number(),
|
||||
}).optional(),
|
||||
});
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user