fixes typecheck issue after rebasing

.gitignore

@@ -1,6 +1,7 @@
 .env
 .venv
 .vscode
+.venv
 build
 greeting.md
 node_modules

README.md

@@ -40,3 +40,5 @@ To run the proxy locally for development or testing, install Node.js >= 18.0.0 a
 4. Start the server in development mode with `npm run start:dev`.
 
 You can also use `npm run start:dev:tsc` to enable project-wide type checking at the cost of slower startup times. `npm run type-check` can be used to run type checking without starting the server.
+
+See the [Optional Dependencies](./docs/optional-dependencies.md) page for information on how to install the optional Claude tokenizer locally.

docker/ci/Dockerfile

@@ -0,0 +1,45 @@
+# Switched to alpine both for smaller image size and because zeromq.js provides
+# a working prebuilt binary for alpine. On Debian, the prebuild was not working
+# and a bug in libzmq's makefile was causing the build from source to fail.
+# https://github.com/zeromq/zeromq.js/issues/529#issuecomment-1370721089
+FROM node:18-alpine as builder
+
+# Install general build dependencies
+RUN apk add --no-cache autoconf automake g++ libtool zeromq-dev python3 \
+    py3-pip git curl cmake gcc musl-dev pkgconfig openssl-dev
+  
+# Install Rust (required to build huggingface/tokenizers)
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN git clone -b tokenize https://gitgud.io/khanon/oai-reverse-proxy.git /app
+WORKDIR /app
+
+RUN npm ci
+
+RUN npm run build && \
+    npm prune --production
+
+FROM node:18-alpine as runner
+
+RUN apk add --no-cache \
+  zeromq-dev \
+  python3
+
+COPY --from=builder /app/build /app/build
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/.venv /app/.venv
+COPY --from=builder /app/package.json /app/package.json
+
+WORKDIR /app
+RUN . .venv/bin/activate
+
+EXPOSE 7860
+
+ENV NODE_ENV=production
+
+# TODO: stamp with tag and git commit
+ENV RENDER=true
+ENV RENDER_GIT_COMMIT=ci-test
+
+CMD [ "npm", "start" ]

docker/huggingface/Dockerfile

@@ -1,9 +1,10 @@
-FROM node:18-bullseye-slim
+FROM node:18-bullseye
 RUN apt-get update && \
-    apt-get install -y git
+    apt-get install -y git python3 python3-pip libzmq3-dev curl cmake g++ libsodium-dev pkg-config
 RUN git clone https://gitgud.io/khanon/oai-reverse-proxy.git /app
 WORKDIR /app
-RUN npm install
+RUN pip3 install --no-cache-dir -r requirements.txt
+RUN npm ci --loglevel=verbose
 COPY Dockerfile greeting.md* .env* ./
 RUN npm run build
 EXPOSE 7860

docs/optional-dependencies.md

@@ -0,0 +1,35 @@
+# Optional Dependencies
+## Claude tokenizer
+As Anthropic does not ship a NodeJS tokenizer, the server includes a small Python script that runs alongside the proxy to tokenize Claude requests. It is automatically started when the server is launched, but requires additional dependencies to be installed. If these dependencies are not installed, the server will not be able to accurately count the number of tokens in Claude requests but will still function normally otherwise.
+
+Note: On Windows, a Windows Firewall prompt may appear when the Claude tokenizer is started. This is normal and is caused by the Python process attempting to open a socket to communicate with the NodeJS server. You can safely allow the connection.
+
+### Automatic installation (local development)
+This will create a venv and install the required dependencies. You still need to activate the venv when running the server, and you must have Python >= 3.8.0 installed.
+1. Install Python >= 3.8.0
+2. Run `npm install`, which should automatically create a venv and install the required dependencies.
+3. Activate the virtual environment with `source .venv/bin/activate` (Linux/Mac) or `.\.venv\Scripts\activate` (PowerShell/Windows)
+    - **This step is required every time you start the server from a new terminal.**
+
+### Manual installation (local development)
+1. Install Python >= 3.8.0
+2. Create a virtual environment using `python -m .venv venv`
+3. Activate the virtual environment with `source .venv/bin/activate` (Linux/Mac) or `.\.venv\Scripts\activate` (PowerShell/Windows)
+    - **This step is required every time you start the server from a new terminal.**
+4. Install dependencies with `pip install -r requirements.txt`
+5. Provided you have the virtual environment activated, the server will automatically start the tokenizer when it is launched.
+
+### Docker (production deployment)
+Refer to the reference Dockerfiles for examples on how to install the tokenizer. The Huggingface and Render Dockerfiles both include the tokenizer.
+
+Generally, you will need libzmq3-dev, cmake, g++, and Python >= 3.8.0 installed. The postinstall script will automatically install the required Python dependencies.
+
+### Troubleshooting
+Ensure that:
+- Python >= 3.8 is installed and in your PATH
+- Python dependencies are installed (re-run `npm install`)
+- Python venv is activated (see above)
+- zeromq optional dependency installed successfully
+  - This should generally be installed automatically.
+  - On Windows, you may need to install MS C++ Build Tools or set msvs_version (eg `npm config set msvs_version 2019`), then re-run npm install.
+  - On Linux, ensure you have the appropriate build tools and headers installed for your distribution; refer to the reference Dockerfiles for examples.

esbuild.js

@@ -0,0 +1,47 @@
+const esbuild = require("esbuild");
+const fs = require("fs");
+const { copy } = require("esbuild-plugin-copy");
+
+const buildDir = "build";
+
+const config = {
+  entryPoints: ["src/server.ts"],
+  bundle: true,
+  outfile: `${buildDir}/server.js`,
+  platform: "node",
+  target: "es2020",
+  format: "cjs",
+  sourcemap: true,
+  external: ["fs", "path", "zeromq", "tiktoken"],
+  plugins: [
+    copy({
+      resolveFrom: "cwd",
+      assets: {
+        from: ["src/tokenization/*.py"],
+        to: [`${buildDir}/tokenization`],
+      },
+    }),
+  ],
+};
+
+function createBundler() {
+  return {
+    build: async () => esbuild.build(config),
+    watch: async () => {
+      const watchConfig = { ...config, logLevel: "info" };
+      const ctx = await esbuild.context(watchConfig);
+      ctx.watch();
+    },
+  };
+}
+
+(async () => {
+  fs.rmSync(buildDir, { recursive: true, force: true });
+  const isDev = process.argv.includes("--dev");
+  const bundler = createBundler();
+  if (isDev) {
+    await bundler.watch();
+  } else {
+    await bundler.build();
+  }
+})();

package-lock.json

@@ -7,6 +7,7 @@
     "": {
       "name": "oai-reverse-proxy",
       "version": "1.0.0",
+      "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
         "axios": "^1.3.5",
@@ -20,6 +21,7 @@
         "pino": "^8.11.0",
         "pino-http": "^8.3.3",
         "showdown": "^2.1.0",
+        "tiktoken": "^1.0.7",
         "uuid": "^9.0.0",
         "zlib": "^1.0.5",
         "zod": "^3.21.4"
@@ -29,8 +31,11 @@
         "@types/express": "^4.17.17",
         "@types/showdown": "^2.0.0",
         "@types/uuid": "^9.0.1",
+        "@types/zeromq": "^5.2.2",
         "concurrently": "^8.0.1",
         "esbuild": "^0.17.16",
+        "esbuild-node-externals": "^1.7.0",
+        "esbuild-plugin-copy": "^2.1.1",
         "esbuild-register": "^3.4.2",
         "nodemon": "^2.0.22",
         "source-map-support": "^0.5.21",
@@ -50,6 +55,30 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/@aminya/node-gyp-build": {
+      "version": "4.5.0-aminya.4",
+      "resolved": "https://registry.npmjs.org/@aminya/node-gyp-build/-/node-gyp-build-4.5.0-aminya.4.tgz",
+      "integrity": "sha512-2c2+BqZOxfTz/m+1MNWncMyMgil2WOg8cHhKPf1qUo1t9ohOWOgSeb7TVVD4fnTxIcAcpWdmXBpFkjPRyBVS9g==",
+      "bin": {
+        "node-gyp-build": "bin.js",
+        "node-gyp-build-optional": "optional.js",
+        "node-gyp-build-test": "build-test.js"
+      },
+      "optionalDependencies": {
+        "zeromq": "^6.0.0-beta.16"
+      }
+    },
+    "node_modules/@aminya/node-gyp-build": {
+      "version": "4.5.0-aminya.4",
+      "resolved": "https://registry.npmjs.org/@aminya/node-gyp-build/-/node-gyp-build-4.5.0-aminya.4.tgz",
+      "integrity": "sha512-2c2+BqZOxfTz/m+1MNWncMyMgil2WOg8cHhKPf1qUo1t9ohOWOgSeb7TVVD4fnTxIcAcpWdmXBpFkjPRyBVS9g==",
+      "optional": true,
+      "bin": {
+        "node-gyp-build": "bin.js",
+        "node-gyp-build-optional": "optional.js",
+        "node-gyp-build-test": "build-test.js"
+      }
+    },
     "node_modules/@babel/parser": {
       "version": "7.22.7",
       "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.7.tgz",
@@ -671,6 +700,41 @@
         "node": ">=v12.0.0"
       }
     },
+    "node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/@protobufjs/aspromise": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -926,6 +990,15 @@
       "integrity": "sha512-rFT3ak0/2trgvp4yYZo5iKFEPsET7vKydKF+VRCxlQ9bpheehyAJH89dAkaLEq/j/RZXJIqcgsmPJKUP1Z28HA==",
       "dev": true
     },
+    "node_modules/@types/zeromq": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/@types/zeromq/-/zeromq-5.2.2.tgz",
+      "integrity": "sha512-BsYJhPP27BbMu3k11qo/3neYLTn2QK8cLONYgBHc/vlVpu7wqIawclt8BKTjBqEDwcFnbbrk/HBw8x0zpKZJlw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -1071,6 +1144,15 @@
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
     },
+    "node_modules/array-union": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
+      "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/arrify": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
@@ -1503,6 +1585,38 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cross-env": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz",
+      "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==",
+      "optional": true,
+      "dependencies": {
+        "cross-spawn": "^7.0.1"
+      },
+      "bin": {
+        "cross-env": "src/bin/cross-env.js",
+        "cross-env-shell": "src/bin/cross-env-shell.js"
+      },
+      "engines": {
+        "node": ">=10.14",
+        "npm": ">=6",
+        "yarn": ">=1"
+      }
+    },
+    "node_modules/cross-spawn": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "optional": true,
+      "dependencies": {
+        "path-key": "^3.1.0",
+        "shebang-command": "^2.0.0",
+        "which": "^2.0.1"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/date-fns": {
       "version": "2.29.3",
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz",
@@ -1564,6 +1678,18 @@
         "node": ">=0.3.1"
       }
     },
+    "node_modules/dir-glob": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
+      "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
+      "dev": true,
+      "dependencies": {
+        "path-type": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/dotenv": {
       "version": "16.0.3",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
@@ -1686,6 +1812,37 @@
         "@esbuild/win32-x64": "0.17.16"
       }
     },
+    "node_modules/esbuild-node-externals": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/esbuild-node-externals/-/esbuild-node-externals-1.7.0.tgz",
+      "integrity": "sha512-nfY3hxtO2anCTZ87LgfzCTfBuyG6de+NyiCNMF1mgrBufS0NgoYlBwF77HHuOInsJLxsAJf0BfLeV6ekZ3hRuA==",
+      "dev": true,
+      "dependencies": {
+        "find-up": "^5.0.0",
+        "tslib": "^2.4.1"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "peerDependencies": {
+        "esbuild": "0.12 - 0.17"
+      }
+    },
+    "node_modules/esbuild-plugin-copy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/esbuild-plugin-copy/-/esbuild-plugin-copy-2.1.1.tgz",
+      "integrity": "sha512-Bk66jpevTcV8KMFzZI1P7MZKZ+uDcrZm2G2egZ2jNIvVnivDpodZI+/KnpL3Jnap0PBdIHU7HwFGB8r+vV5CVw==",
+      "dev": true,
+      "dependencies": {
+        "chalk": "^4.1.2",
+        "chokidar": "^3.5.3",
+        "fs-extra": "^10.0.1",
+        "globby": "^11.0.3"
+      },
+      "peerDependencies": {
+        "esbuild": ">= 0.14.0"
+      }
+    },
     "node_modules/esbuild-register": {
       "version": "3.4.2",
       "resolved": "https://registry.npmjs.org/esbuild-register/-/esbuild-register-3.4.2.tgz",
@@ -1916,6 +2073,22 @@
       "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
       "optional": true
     },
+    "node_modules/fast-glob": {
+      "version": "3.2.12",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz",
+      "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==",
+      "dev": true,
+      "dependencies": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.4"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
     "node_modules/fast-levenshtein": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
@@ -1935,6 +2108,15 @@
       "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz",
       "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w=="
     },
+    "node_modules/fastq": {
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
+      "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==",
+      "dev": true,
+      "dependencies": {
+        "reusify": "^1.0.4"
+      }
+    },
     "node_modules/faye-websocket": {
       "version": "0.11.4",
       "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz",
@@ -1974,6 +2156,22 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/find-up": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
+      "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
+      "dev": true,
+      "dependencies": {
+        "locate-path": "^6.0.0",
+        "path-exists": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/firebase-admin": {
       "version": "11.9.0",
       "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-11.9.0.tgz",
@@ -2044,6 +2242,20 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/fs-extra": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
+      "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
+      "dev": true,
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -2174,6 +2386,26 @@
         "node": ">=10"
       }
     },
+    "node_modules/globby": {
+      "version": "11.1.0",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz",
+      "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==",
+      "dev": true,
+      "dependencies": {
+        "array-union": "^2.1.0",
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.2.9",
+        "ignore": "^5.2.0",
+        "merge2": "^1.4.1",
+        "slash": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/google-auth-library": {
       "version": "8.7.0",
       "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz",
@@ -2299,7 +2531,7 @@
       "version": "4.2.11",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
       "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
-      "optional": true
+      "devOptional": true
     },
     "node_modules/gtoken": {
       "version": "6.1.2",
@@ -2515,6 +2747,15 @@
         }
       ]
     },
+    "node_modules/ignore": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz",
+      "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/ignore-by-default": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
@@ -2536,6 +2777,15 @@
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
+    "node_modules/interpret": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
+      "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
+      "optional": true,
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -2556,6 +2806,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/is-core-module": {
+      "version": "2.12.1",
+      "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz",
+      "integrity": "sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==",
+      "optional": true,
+      "dependencies": {
+        "has": "^1.0.3"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/is-extglob": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -2620,6 +2882,12 @@
       "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==",
       "optional": true
     },
+    "node_modules/isexe": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+      "optional": true
+    },
     "node_modules/jose": {
       "version": "4.14.4",
       "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz",
@@ -2674,6 +2942,18 @@
         "bignumber.js": "^9.0.0"
       }
     },
+    "node_modules/jsonfile": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
+      "dev": true,
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "node_modules/jsonwebtoken": {
       "version": "9.0.0",
       "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
@@ -2805,6 +3085,21 @@
         "uc.micro": "^1.0.1"
       }
     },
+    "node_modules/locate-path": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
+      "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
+      "dev": true,
+      "dependencies": {
+        "p-locate": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/lodash": {
       "version": "4.17.21",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -2924,6 +3219,15 @@
       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
       "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
     },
+    "node_modules/merge2": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/methods": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
@@ -3020,6 +3324,12 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/node-addon-api": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
+      "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==",
+      "optional": true
+    },
     "node_modules/node-fetch": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz",
@@ -3202,7 +3512,7 @@
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
       "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
-      "optional": true,
+      "devOptional": true,
       "dependencies": {
         "yocto-queue": "^0.1.0"
       },
@@ -3213,6 +3523,21 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-locate": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+      "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+      "dev": true,
+      "dependencies": {
+        "p-limit": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -3221,6 +3546,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/path-exists": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -3230,11 +3564,35 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/path-key": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+      "optional": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/path-parse": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
+      "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
+      "optional": true
+    },
     "node_modules/path-to-regexp": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
       "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
     },
+    "node_modules/path-type": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+      "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/picomatch": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
@@ -3426,6 +3784,26 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/queue-microtask": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
+    },
     "node_modules/quick-format-unescaped": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
@@ -3487,6 +3865,18 @@
         "node": ">= 12.13.0"
       }
     },
+    "node_modules/rechoir": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
+      "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==",
+      "optional": true,
+      "dependencies": {
+        "resolve": "^1.1.6"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -3510,6 +3900,23 @@
         "lodash": "^4.17.21"
       }
     },
+    "node_modules/resolve": {
+      "version": "1.22.2",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz",
+      "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==",
+      "optional": true,
+      "dependencies": {
+        "is-core-module": "^2.11.0",
+        "path-parse": "^1.0.7",
+        "supports-preserve-symlinks-flag": "^1.0.0"
+      },
+      "bin": {
+        "resolve": "bin/resolve"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/retry": {
       "version": "0.13.1",
       "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
@@ -3555,6 +3962,16 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
       "optional": true
     },
+    "node_modules/reusify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+      "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "dev": true,
+      "engines": {
+        "iojs": ">=1.0.0",
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/rimraf": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -3590,6 +4007,29 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/run-parallel": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+      "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "dependencies": {
+        "queue-microtask": "^1.2.2"
+      }
+    },
     "node_modules/rxjs": {
       "version": "7.8.0",
       "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz",
@@ -3692,6 +4132,27 @@
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
       "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
     },
+    "node_modules/shebang-command": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+      "optional": true,
+      "dependencies": {
+        "shebang-regex": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/shebang-regex": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+      "optional": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/shell-quote": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz",
@@ -3701,6 +4162,43 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/shelljs": {
+      "version": "0.8.5",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz",
+      "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==",
+      "optional": true,
+      "dependencies": {
+        "glob": "^7.0.0",
+        "interpret": "^1.0.0",
+        "rechoir": "^0.6.2"
+      },
+      "bin": {
+        "shjs": "bin/shjs"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/shelljs/node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "optional": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/showdown": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz",
@@ -3716,6 +4214,22 @@
         "url": "https://www.paypal.me/tiviesantos"
       }
     },
+    "node_modules/shx": {
+      "version": "0.3.4",
+      "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz",
+      "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==",
+      "optional": true,
+      "dependencies": {
+        "minimist": "^1.2.3",
+        "shelljs": "^0.8.5"
+      },
+      "bin": {
+        "shx": "lib/cli.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/side-channel": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
@@ -3741,6 +4255,24 @@
         "node": ">=8.10.0"
       }
     },
+    "node_modules/simple-update-notifier/node_modules/semver": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz",
+      "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver.js"
+      }
+    },
+    "node_modules/slash": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
+      "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==",
+      "dev": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/sonic-boom": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.3.0.tgz",
@@ -3870,6 +4402,18 @@
         "node": ">=4"
       }
     },
+    "node_modules/supports-preserve-symlinks-flag": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz",
+      "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==",
+      "optional": true,
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/teeny-request": {
       "version": "8.0.3",
       "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.3.tgz",
@@ -3899,6 +4443,11 @@
         "real-require": "^0.2.0"
       }
     },
+    "node_modules/tiktoken": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.7.tgz",
+      "integrity": "sha512-UoEWsxJssI8sKUJrUXLm0IlVdRHrng1BTHdmz6arbbmy6WY6LZZHXIpR/9c5pOYDsT2e24v04uBTFwh+AYXLiw=="
+    },
     "node_modules/tmp": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
@@ -4071,6 +4620,15 @@
       "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==",
       "optional": true
     },
+    "node_modules/universalify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
+      "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==",
+      "dev": true,
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -4155,6 +4713,30 @@
         "webidl-conversions": "^3.0.0"
       }
     },
+    "node_modules/which": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+      "optional": true,
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "node-which": "bin/node-which"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/word-wrap": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+      "optional": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@@ -4238,7 +4820,7 @@
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
       "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
-      "optional": true,
+      "devOptional": true,
       "engines": {
         "node": ">=10"
       },
@@ -4246,6 +4828,23 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/zeromq": {
+      "version": "6.0.0-beta.16",
+      "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-6.0.0-beta.16.tgz",
+      "integrity": "sha512-taPr+V2synMrybR4H4YBJkjQ1tIi0CPuXsO6Jm2O1IbgnfJ0o3qYqi0QuhT9/oFwiTNr/yQiCze9OU2szGlp7w==",
+      "hasInstallScript": true,
+      "optional": true,
+      "dependencies": {
+        "@aminya/node-gyp-build": "4.5.0-aminya.4",
+        "cross-env": "^7.0.3",
+        "node-addon-api": "^5.0.0",
+        "shelljs": "^0.8.5",
+        "shx": "^0.3.4"
+      },
+      "engines": {
+        "node": ">= 10.2"
+      }
+    },
     "node_modules/zlib": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",

package.json

@@ -3,12 +3,13 @@
   "version": "1.0.0",
   "description": "Reverse proxy for the OpenAI API",
   "scripts": {
-    "build:watch": "esbuild src/server.ts --outfile=build/server.js --platform=node --target=es2020 --format=cjs --bundle --sourcemap --watch",
-    "build": "tsc",
-    "start:dev": "concurrently \"npm run build:watch\" \"npm run start:watch\"",
-    "start:dev:tsc": "nodemon --watch src --exec ts-node --transpile-only src/server.ts",
-    "start:watch": "nodemon --require source-map-support/register build/server.js",
+    "build:dev": "node esbuild.js --dev",
+    "build": "node esbuild.js",
+    "postinstall": "node scripts/install-python-deps.js",
+    "start:dev:tsc": "nodemon --watch src --exec ts-node src/server.ts",
+    "start:dev": "concurrently \"npm run build:dev\" \"npm run start:watch\"",
     "start:replit": "tsc && node build/server.js",
+    "start:watch": "nodemon --require source-map-support/register build/server.js",
     "start": "node build/server.js",
     "type-check": "tsc --noEmit"
   },
@@ -29,6 +30,7 @@
     "pino": "^8.11.0",
     "pino-http": "^8.3.3",
     "showdown": "^2.1.0",
+    "tiktoken": "^1.0.7",
     "uuid": "^9.0.0",
     "zlib": "^1.0.5",
     "zod": "^3.21.4"
@@ -38,8 +40,11 @@
     "@types/express": "^4.17.17",
     "@types/showdown": "^2.0.0",
     "@types/uuid": "^9.0.1",
+    "@types/zeromq": "^5.2.2",
     "concurrently": "^8.0.1",
     "esbuild": "^0.17.16",
+    "esbuild-node-externals": "^1.7.0",
+    "esbuild-plugin-copy": "^2.1.1",
     "esbuild-register": "^3.4.2",
     "nodemon": "^2.0.22",
     "source-map-support": "^0.5.21",
@@ -49,5 +54,8 @@
   "overrides": {
     "optionator": "^0.9.3",
     "semver": "^7.5.3"
+  },
+  "optionalDependencies": {
+    "zeromq": "^6.0.0-beta.16"
   }
 }

requirements.txt

@@ -0,0 +1,2 @@
+pyzmq==25.1.0
+anthropic==0.2.9

scripts/install-python-deps.js

@@ -0,0 +1,68 @@
+const fs = require("fs");
+const spawn = require("child_process").spawn;
+
+const IS_WINDOWS = process.platform === "win32";
+const IS_DEV = process.env.NODE_ENV !== "production";
+
+const installDeps = async () => {
+  try {
+    console.log("Installing additional optional dependencies...");
+    console.log("Creating venv...");
+    await maybeCreateVenv();
+    console.log("Installing python dependencies...");
+    await installPythonDependencies();
+  } catch (error) {
+    console.error("Error installing additional optional dependencies", error);
+    process.exit(0); // don't fail the build
+  }
+};
+
+installDeps();
+
+async function maybeCreateVenv() {
+  if (!IS_DEV) {
+    console.log("Skipping venv creation in production");
+    return true;
+  }
+  if (fs.existsSync(".venv")) {
+    console.log("Skipping venv creation, already exists");
+    return true;
+  }
+  const python = IS_WINDOWS ? "python" : "python3";
+  await runCommand(`${python} -m venv .venv`);
+  return true;
+}
+
+async function installPythonDependencies() {
+  const commands = [];
+  if (IS_DEV) {
+    commands.push(
+      IS_WINDOWS ? ".venv\\Scripts\\activate.bat" : "source .venv/bin/activate"
+    );
+  }
+  const pip = IS_WINDOWS ? "pip" : "pip3";
+  commands.push(`${pip} install -r requirements.txt`);
+
+  const command = commands.join(" && ");
+  await runCommand(command);
+  return true;
+}
+
+async function runCommand(command) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, [], { shell: true });
+    child.stdout.on("data", (data) => {
+      console.log(data.toString());
+    });
+    child.stderr.on("data", (data) => {
+      console.error(data.toString());
+    });
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject();
+      }
+    });
+  });
+}

src/info-page.ts

@@ -52,7 +52,17 @@ function cacheInfoPageHtml(baseUrl: string) {
   };
 
   const title = getServerTitle();
-  const headerHtml = buildInfoPageHeader(new showdown.Converter(), title);
+  let headerHtml = buildInfoPageHeader(new showdown.Converter(), title);
+
+  if (process.env.MISSING_PYTHON_WARNING) {
+    headerHtml +=
+      `<p style="color: red;">Python is not installed; the Claude tokenizer ` +
+      `cannot start. Your Dockerfile may be out of date; see <a ` +
+      `href="https://gitgud.io/khanon/oai-reverse-proxy">the docs</a> for an ` +
+      `updated Huggingface Dockerfile.</p><p>You can disable this warning by ` +
+      `setting <code>DISABLE_MISSING_PYTHON_WARNING=true</code> in your ` +
+      `environment.</p>`;
+  }
 
   const pageBody = `<!DOCTYPE html>
 <html lang="en">
@@ -193,7 +203,7 @@ Logs are anonymous and do not contain IP addresses or timestamps. [You can see t
   }
 
   if (config.queueMode !== "none") {
-    const waits = [];
+    const waits: string[] = [];
     infoBody += `\n## Estimated Wait Times\nIf the AI is busy, your prompt will processed when a slot frees up.`;
 
     if (config.openaiKey) {

src/proxy/anthropic.ts

@@ -108,10 +108,16 @@ const anthropicResponseHandler: ProxyResHandlerWithBody = async (
     body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
   }
 
-  if (!req.originalUrl.includes("/v1/complete") && !req.originalUrl.includes("/complete")) {
+  if (req.inboundApi === "openai") {
     req.log.info("Transforming Anthropic response to OpenAI format");
     body = transformAnthropicResponse(body);
   }
+
+  // TODO: Remove once tokenization is stable
+  if (req.debug) {
+    body.proxy_tokenizer_debug_info = req.debug;
+  }
+
   res.status(200).json(body);
 };
 

src/proxy/middleware/common.ts

@@ -45,6 +45,9 @@ export function writeErrorResponse(
     res.write(`data: [DONE]\n\n`);
     res.end();
   } else {
+    if (req.debug) {
+      errorPayload.error.proxy_tokenizer_debug_info = req.debug;
+    }
     res.status(statusCode).json(errorPayload);
   }
 }
@@ -57,7 +60,8 @@ export const handleProxyError: httpProxy.ErrorCallback = (err, req, res) => {
 export const handleInternalError = (
   err: Error,
   req: Request,
-  res: Response
+  res: Response,
+  errorType: string = "proxy_internal_error"
 ) => {
   try {
     const isZod = err instanceof ZodError;
@@ -86,7 +90,7 @@ export const handleInternalError = (
     } else {
       writeErrorResponse(req, res, 500, {
         error: {
-          type: "proxy_rewriter_error",
+          type: errorType,
           proxy_note: `Reverse proxy encountered an error before it could reach the upstream API.`,
           message: err.message,
           stack: err.stack,

src/proxy/middleware/request/add-key.ts

@@ -41,8 +41,6 @@ export const addKey: ProxyRequestMiddleware = (proxyReq, req) => {
   // For such cases, ignore the requested model entirely.
   if (req.inboundApi === "openai" && req.outboundApi === "anthropic") {
     req.log.debug("Using an Anthropic key for an OpenAI-compatible request");
-    // We don't assign the model here, that will happen when transforming the
-    // request body.
     assignedKey = keyPool.get("claude-v1");
   } else {
     assignedKey = keyPool.get(req.body.model);

src/proxy/middleware/request/check-prompt-size.ts

@@ -0,0 +1,36 @@
+import { countTokens } from "../../../tokenization";
+import { RequestPreprocessor } from ".";
+import { openAIMessagesToClaudePrompt } from "./transform-outbound-payload";
+
+export const checkPromptSize: RequestPreprocessor = async (req) => {
+  const prompt =
+    req.inboundApi === "openai" ? req.body.messages : req.body.prompt;
+
+  let result;
+  if (req.outboundApi === "openai") {
+    result = await countTokens({ req, prompt, service: "openai" });
+  } else {
+    // If we're doing OpenAI-to-Anthropic, we need to convert the messages to a
+    // prompt first before counting tokens, as that process affects the token
+    // count.
+    let promptStr =
+      req.inboundApi === "anthropic"
+        ? prompt
+        : openAIMessagesToClaudePrompt(prompt);
+    result = await countTokens({
+      req,
+      prompt: promptStr,
+      service: "anthropic",
+    });
+  }
+
+  req.promptTokens = result.token_count;
+
+  // TODO: Remove once token counting is stable
+  req.log.debug({ result }, "Counted prompt tokens");
+  req.debug = req.debug ?? {};
+  req.debug = {
+    ...req.debug,
+    ...result,
+  };
+};

src/proxy/middleware/request/index.ts

@@ -4,6 +4,7 @@ import type { ProxyReqCallback } from "http-proxy";
 
 // Express middleware (runs before http-proxy-middleware, can be async)
 export { createPreprocessorMiddleware } from "./preprocess";
+export { checkPromptSize } from "./check-prompt-size";
 export { setApiFormat } from "./set-api-format";
 export { transformOutboundPayload } from "./transform-outbound-payload";
 

src/proxy/middleware/request/preprocess.ts

@@ -1,6 +1,11 @@
 import { RequestHandler } from "express";
 import { handleInternalError } from "../common";
-import { RequestPreprocessor, setApiFormat, transformOutboundPayload } from ".";
+import {
+  RequestPreprocessor,
+  checkPromptSize,
+  setApiFormat,
+  transformOutboundPayload,
+} from ".";
 
 /**
  * Returns a middleware function that processes the request body into the given
@@ -12,6 +17,7 @@ export const createPreprocessorMiddleware = (
 ): RequestHandler => {
   const preprocessors: RequestPreprocessor[] = [
     setApiFormat(apiFormat),
+    checkPromptSize,
     transformOutboundPayload,
     ...(additionalPreprocessors ?? []),
   ];
@@ -24,7 +30,7 @@ export const createPreprocessorMiddleware = (
       next();
     } catch (error) {
       req.log.error(error, "Error while executing request preprocessor");
-      handleInternalError(error as Error, req, res);
+      handleInternalError(error as Error, req, res, "proxy_preprocessor_error");
     }
   };
 };

src/proxy/middleware/request/transform-outbound-payload.ts

@@ -2,7 +2,13 @@ import { Request } from "express";
 import { z } from "zod";
 import { isCompletionRequest } from "../common";
 import { RequestPreprocessor } from ".";
-// import { countTokens } from "../../../tokenization";
+import { OpenAIPromptMessage } from "../../../tokenization/openai";
+
+/**
+ * The maximum number of tokens an Anthropic prompt can have before we switch to
+ * the larger claude-100k context model.
+ */
+const CLAUDE_100K_TOKEN_THRESHOLD = 8200;
 
 // https://console.anthropic.com/docs/api/reference#-v1-complete
 const AnthropicV1CompleteSchema = z.object({
@@ -55,10 +61,9 @@ const OpenAIV1ChatCompletionSchema = z.object({
 /** Transforms an incoming request body to one that matches the target API. */
 export const transformOutboundPayload: RequestPreprocessor = async (req) => {
   const sameService = req.inboundApi === req.outboundApi;
-  const alreadyTransformed = req.retryCount > 0;
   const notTransformable = !isCompletionRequest(req);
 
-  if (alreadyTransformed || notTransformable) {
+  if (notTransformable) {
     return;
   }
 
@@ -69,6 +74,7 @@ export const transformOutboundPayload: RequestPreprocessor = async (req) => {
         ? OpenAIV1ChatCompletionSchema
         : AnthropicV1CompleteSchema;
     const result = validator.safeParse(req.body);
+
     if (!result.success) {
       req.log.error(
         { issues: result.error.issues, body: req.body },
@@ -76,11 +82,14 @@ export const transformOutboundPayload: RequestPreprocessor = async (req) => {
       );
       throw result.error;
     }
+
+    validatePromptSize(req);
     return;
   }
 
   if (req.inboundApi === "openai" && req.outboundApi === "anthropic") {
     req.body = openaiToAnthropic(req.body, req);
+    validatePromptSize(req);
     return;
   }
 
@@ -107,24 +116,7 @@ function openaiToAnthropic(body: any, req: Request) {
   req.headers["anthropic-version"] = "2023-01-01";
 
   const { messages, ...rest } = result.data;
-  const prompt =
-    result.data.messages
-      .map((m) => {
-        let role: string = m.role;
-        if (role === "assistant") {
-          role = "Assistant";
-        } else if (role === "system") {
-          role = "System";
-        } else if (role === "user") {
-          role = "Human";
-        }
-        // https://console.anthropic.com/docs/prompt-design
-        // `name` isn't supported by Anthropic but we can still try to use it.
-        return `\n\n${role}: ${m.name?.trim() ? `(as ${m.name}) ` : ""}${
-          m.content
-        }`;
-      })
-      .join("") + "\n\nAssistant: ";
+  const prompt = openAIMessagesToClaudePrompt(messages);
 
   // No longer defaulting to `claude-v1.2` because it seems to be in the process
   // of being deprecated. `claude-v1` is the new default.
@@ -135,9 +127,16 @@ function openaiToAnthropic(body: any, req: Request) {
   const CLAUDE_BIG = process.env.CLAUDE_BIG_MODEL || "claude-v1-100k";
   const CLAUDE_SMALL = process.env.CLAUDE_SMALL_MODEL || "claude-v1";
 
-  // TODO: Finish implementing tokenizer for more accurate model selection.
-  // This currently uses _character count_, not token count.
-  const model = prompt.length > 25000 ? CLAUDE_BIG : CLAUDE_SMALL;
+  const contextTokens = Number(req.promptTokens ?? 0) + Number(rest.max_tokens);
+  const model =
+    (contextTokens ?? 0) > CLAUDE_100K_TOKEN_THRESHOLD
+      ? CLAUDE_BIG
+      : CLAUDE_SMALL;
+
+  req.log.debug(
+    { contextTokens, model, CLAUDE_100K_TOKEN_THRESHOLD },
+    "Selected Claude model"
+  );
 
   let stops = rest.stop
     ? Array.isArray(rest.stop)
@@ -160,3 +159,63 @@ function openaiToAnthropic(body: any, req: Request) {
     stop_sequences: stops,
   };
 }
+
+export function openAIMessagesToClaudePrompt(messages: OpenAIPromptMessage[]) {
+  return (
+    messages
+      .map((m) => {
+        let role: string = m.role;
+        if (role === "assistant") {
+          role = "Assistant";
+        } else if (role === "system") {
+          role = "System";
+        } else if (role === "user") {
+          role = "Human";
+        }
+        // https://console.anthropic.com/docs/prompt-design
+        // `name` isn't supported by Anthropic but we can still try to use it.
+        return `\n\n${role}: ${m.name?.trim() ? `(as ${m.name}) ` : ""}${
+          m.content
+        }`;
+      })
+      .join("") + "\n\nAssistant:"
+  );
+}
+
+function validatePromptSize(req: Request) {
+  const promptTokens = req.promptTokens || 0;
+  const model = req.body.model;
+  let maxTokensForModel = 0;
+
+  if (model.match(/gpt-3.5/)) {
+    maxTokensForModel = 4096;
+  } else if (model.match(/gpt-4/)) {
+    maxTokensForModel = 8192;
+  } else if (model.match(/gpt-4-32k/)) {
+    maxTokensForModel = 32768;
+  } else if (model.match(/claude-(?:instant-)?v1(?:\.\d)?(?:-100k)/)) {
+    // Claude models don't throw an error if you exceed the token limit and
+    // instead just become extremely slow and give schizo results, so we will be
+    // more conservative with the token limit for them.
+    maxTokensForModel = 100000 * 0.98;
+  } else if (model.match(/claude-(?:instant-)?v1(?:\.\d)?$/)) {
+    maxTokensForModel = 9000 * 0.98;
+  } else {
+    // I don't trust my regular expressions enough to throw an error here so
+    // we just log a warning and allow 100k tokens.
+    req.log.warn({ model }, "Unknown model, using 100k token limit.");
+    maxTokensForModel = 100000;
+  }
+
+  if (req.debug) {
+    req.debug.calculated_max_tokens = maxTokensForModel;
+  }
+
+  z.number()
+    .max(
+      maxTokensForModel,
+      `Prompt is too long for model ${model} (${promptTokens} tokens, max ${maxTokensForModel})`
+    )
+    .parse(promptTokens);
+  req.log.debug({ promptTokens, maxTokensForModel }, "Prompt size validated");
+}

src/proxy/openai.ts

@@ -125,6 +125,11 @@ const openaiResponseHandler: ProxyResHandlerWithBody = async (
     body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
   }
 
+  // TODO: Remove once tokenization is stable
+  if (req.debug) {
+    body.proxy_tokenizer_debug_info = req.debug;
+  }
+
   res.status(200).json(body);
 };
 

src/server.ts

@@ -12,6 +12,7 @@ import { handleInfoPage } from "./info-page";
 import { logQueue } from "./prompt-logging";
 import { start as startRequestQueue } from "./proxy/queue";
 import { init as initUserStore } from "./proxy/auth/user-store";
+import { init as initTokenizers } from "./tokenization";
 import { checkOrigin } from "./proxy/check-origin";
 
 const PORT = config.port;
@@ -99,6 +100,8 @@ async function start() {
 
   keyPool.init();
 
+  await initTokenizers();
+
   if (config.gatekeeper === "user_token") {
     await initUserStore();
   }

src/tokenization/claude-ipc.ts

@@ -0,0 +1,160 @@
+import { spawn, ChildProcess } from "child_process";
+import { join } from "path";
+import { logger } from "../logger";
+
+const TOKENIZER_SOCKET = "tcp://localhost:5555";
+const log = logger.child({ module: "claude-ipc" });
+const pythonLog = logger.child({ module: "claude-python" });
+
+let tokenizer: ChildProcess;
+let initialized = false;
+let socket: any; // zeromq.Dealer, not sure how to import it safely as it is optional
+
+export async function init() {
+  log.info("Initializing Claude tokenizer IPC");
+  try {
+    tokenizer = await launchTokenizer();
+    const zmq = await import("zeromq");
+    socket = new zmq.Dealer({ sendTimeout: 500 });
+    socket.connect(TOKENIZER_SOCKET);
+
+    await socket.send(["init"]);
+    const response = await socket.receive();
+    if (response.toString() !== "ok") {
+      throw new Error("Unexpected init response");
+    }
+
+    // Start message pump
+    processMessages();
+
+    // Test tokenizer
+    const result = await requestTokenCount({
+      requestId: "init-test",
+      prompt: "test prompt",
+    });
+    if (result !== 2) {
+      log.error({ result }, "Unexpected test token count");
+      throw new Error("Unexpected test token count");
+    }
+
+    initialized = true;
+  } catch (err) {
+    log.error({ err: err.message }, "Failed to initialize Claude tokenizer");
+    if (process.env.NODE_ENV !== "production") {
+      console.error(
+        `\nClaude tokenizer failed to initialize.\nIf you want to use the tokenizer, see the Optional Dependencies documentation.\n`
+      );
+    }
+    return false;
+  }
+  log.info("Claude tokenizer IPC ready");
+  return true;
+}
+
+const pendingRequests = new Map<
+  string,
+  { resolve: (tokens: number) => void }
+>();
+
+export async function requestTokenCount({
+  requestId,
+  prompt,
+}: {
+  requestId: string;
+  prompt: string;
+}) {
+  if (!socket) {
+    throw new Error("Claude tokenizer is not initialized");
+  }
+
+  log.debug({ requestId, chars: prompt.length }, "Requesting token count");
+  await socket.send(["tokenize", requestId, prompt]);
+
+  log.debug({ requestId }, "Waiting for socket response");
+  return new Promise<number>(async (resolve, reject) => {
+    const resolveFn = (tokens: number) => {
+      log.debug({ requestId, tokens }, "Received token count");
+      pendingRequests.delete(requestId);
+      resolve(tokens);
+    };
+
+    pendingRequests.set(requestId, { resolve: resolveFn });
+
+    const timeout = initialized ? 500 : 10000;
+    setTimeout(() => {
+      if (pendingRequests.has(requestId)) {
+        pendingRequests.delete(requestId);
+        const err = "Tokenizer deadline exceeded";
+        log.warn({ requestId }, err);
+        reject(new Error(err));
+      }
+    }, timeout);
+  });
+}
+
+async function processMessages() {
+  if (!socket) {
+    throw new Error("Claude tokenizer is not initialized");
+  }
+  log.debug("Starting message loop");
+  for await (const [requestId, tokens] of socket) {
+    const request = pendingRequests.get(requestId.toString());
+    if (!request) {
+      log.error({ requestId }, "No pending request found for incoming message");
+      continue;
+    }
+    request.resolve(Number(tokens.toString()));
+  }
+}
+
+async function launchTokenizer() {
+  return new Promise<ChildProcess>((resolve, reject) => {
+    let resolved = false;
+
+    const python = process.platform === "win32" ? "python" : "python3";
+    const proc = spawn(python, [
+      "-u",
+      join(__dirname, "tokenization", "claude-tokenizer.py"),
+    ]);
+    if (!proc) {
+      reject(new Error("Failed to spawn Claude tokenizer"));
+    }
+
+    function cleanup() {
+      socket?.close();
+      socket = undefined!;
+      tokenizer = undefined!;
+    }
+
+    proc.stdout!.on("data", (data) => {
+      pythonLog.info(data.toString().trim());
+    });
+    proc.stderr!.on("data", (data) => {
+      pythonLog.error(data.toString().trim());
+    });
+    proc.on("error", (err) => {
+      pythonLog.error({ err }, "Claude tokenizer error");
+      cleanup();
+      if (!resolved) {
+        resolved = true;
+        reject(err);
+      }
+    });
+    proc.on("close", (code) => {
+      pythonLog.info(`Claude tokenizer exited with code ${code}`);
+      cleanup();
+      if (code !== 0 && !resolved) {
+        resolved = true;
+        reject(new Error("Claude tokenizer exited immediately"));
+      }
+    });
+
+    // Wait a moment to catch any immediate errors (missing imports, etc)
+    setTimeout(() => {
+      if (!resolved) {
+        resolved = true;
+        resolve(proc);
+      }
+    }, 200);
+  });
+}

src/tokenization/claude-tokenizer.py

@@ -0,0 +1,54 @@
+""" 
+This is a small process running alongside the main NodeJS server intended to
+tokenize prompts for Claude, as currently Anthropic only ships a Python
+implemetnation for their tokenizer.
+ZeroMQ is used for IPC between the NodeJS server and this process.
+"""
+import zmq
+import anthropic
+
+def create_socket():
+    context = zmq.Context()
+    socket = context.socket(zmq.ROUTER)
+    socket.bind("tcp://*:5555")
+    return context, socket
+
+def init(socket):
+    print("claude-tokenizer.py: starting")
+    try:
+        while True:
+            message = socket.recv_multipart()
+            routing_id, command = message
+            if command == b"init":
+                print("claude-tokenizer.py: initialized")
+                socket.send_multipart([routing_id, b"ok"])
+                break
+    except Exception as e:
+        print("claude-tokenizer.py: failed to initialize")
+        return
+
+    message_processor(socket)
+
+def message_processor(socket):
+    while True:
+        try:
+            message = socket.recv_multipart()
+            routing_id, command, request_id, payload = message
+            payload = payload.decode("utf-8")
+            if command == b"exit":
+                print("claude-tokenizer.py: exiting")
+                break
+            elif command == b"tokenize":                
+                token_count = anthropic.count_tokens(payload)
+                socket.send_multipart([routing_id, request_id, str(token_count).encode("utf-8")])
+            else:
+                print("claude-tokenizer.py: unknown message type")
+        except Exception as e:
+            print(f"claude-tokenizer.py: failed to process message ({e})")
+            break
+
+if __name__ == "__main__":
+    context, socket = create_socket()
+    init(socket)
+    socket.close()
+    context.term()

src/tokenization/index.ts

@@ -0,0 +1 @@
+export { init, countTokens } from "./tokenizer";

src/tokenization/openai.ts

@@ -0,0 +1,57 @@
+import { Tiktoken } from "tiktoken/lite";
+import cl100k_base from "tiktoken/encoders/cl100k_base.json";
+
+let encoder: Tiktoken;
+
+export function init() {
+  encoder = new Tiktoken(
+    cl100k_base.bpe_ranks,
+    cl100k_base.special_tokens,
+    cl100k_base.pat_str
+  );
+  return true;
+}
+
+// Implmentation based and tested against:
+// https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+
+export function getTokenCount(messages: any[], model: string) {
+  const gpt4 = model.startsWith("gpt-4");
+
+  const tokensPerMessage = gpt4 ? 3 : 4;
+  const tokensPerName = gpt4 ? 1 : -1; // turbo omits role if name is present
+
+  let numTokens = 0;
+
+  for (const message of messages) {
+    numTokens += tokensPerMessage;
+    for (const key of Object.keys(message)) {
+      {
+        const value = message[key];
+        // Break if we get a huge message or exceed the token limit to prevent DoS
+        // 100k tokens allows for future 100k GPT-4 models and 250k characters is
+        // just a sanity check
+        if (value.length > 250000 || numTokens > 100000) {
+          numTokens = 100000;
+          return {
+            tokenizer: "tiktoken (prompt length limit exceeded)",
+            token_count: numTokens,
+          };
+        }
+
+        numTokens += encoder.encode(message[key]).length;
+        if (key === "name") {
+          numTokens += tokensPerName;
+        }
+      }
+    }
+  }
+  numTokens += 3; // every reply is primed with <|start|>assistant<|message|>
+  return { tokenizer: "tiktoken", token_count: numTokens };
+}
+
+export type OpenAIPromptMessage = {
+  name?: string;
+  content: string;
+  role: string;
+};

src/tokenization/tokenizer.ts

@@ -0,0 +1,129 @@
+import { Request } from "express";
+import childProcess from "child_process";
+import { config } from "../config";
+import { logger } from "../logger";
+import {
+  init as initIpc,
+  requestTokenCount as requestClaudeTokenCount,
+} from "./claude-ipc";
+import {
+  init as initEncoder,
+  getTokenCount as getOpenAITokenCount,
+  OpenAIPromptMessage,
+} from "./openai";
+
+let canTokenizeClaude = false;
+
+export async function init() {
+  if (config.anthropicKey) {
+    if (!isPythonInstalled()) {
+      const skipWarning = !!process.env.DISABLE_MISSING_PYTHON_WARNING;
+      process.env.MISSING_PYTHON_WARNING = skipWarning ? "" : "true";
+    } else {
+      canTokenizeClaude = await initIpc();
+      if (!canTokenizeClaude) {
+        logger.warn(
+          "Anthropic key is set, but tokenizer is not available. Claude prompts will use a naive estimate for token count."
+        );
+      }
+    }
+  }
+  if (config.openaiKey) {
+    initEncoder();
+  }
+}
+
+type TokenCountResult = {
+  token_count: number;
+  tokenizer: string;
+  tokenization_duration_ms: number;
+};
+type TokenCountRequest = {
+  req: Request;
+} & (
+  | { prompt: string; service: "anthropic" }
+  | { prompt: OpenAIPromptMessage[]; service: "openai" }
+);
+export async function countTokens({
+  req,
+  service,
+  prompt,
+}: TokenCountRequest): Promise<TokenCountResult> {
+  const time = process.hrtime();
+
+  switch (service) {
+    case "anthropic":
+      if (!canTokenizeClaude) {
+        const result = guesstimateTokens(prompt);
+        return {
+          token_count: result,
+          tokenizer: "guesstimate (claude-ipc disabled)",
+          tokenization_duration_ms: getElapsedMs(time),
+        };
+      }
+
+      // If the prompt is absolutely massive (possibly malicious) don't even try
+      if (prompt.length > 500000) {
+        return {
+          token_count: guesstimateTokens(JSON.stringify(prompt)),
+          tokenizer: "guesstimate (prompt too long)",
+          tokenization_duration_ms: getElapsedMs(time),
+        };
+      }
+
+      try {
+        const result = await requestClaudeTokenCount({
+          requestId: String(req.id),
+          prompt,
+        });
+        return {
+          token_count: result,
+          tokenizer: "claude-ipc",
+          tokenization_duration_ms: getElapsedMs(time),
+        };
+      } catch (e: any) {
+        req.log.error("Failed to tokenize with claude_tokenizer", e);
+        const result = guesstimateTokens(prompt);
+        return {
+          token_count: result,
+          tokenizer: `guesstimate (claude-ipc failed: ${e.message})`,
+          tokenization_duration_ms: getElapsedMs(time),
+        };
+      }
+
+    case "openai":
+      const result = getOpenAITokenCount(prompt, req.body.model);
+      return {
+        ...result,
+        tokenization_duration_ms: getElapsedMs(time),
+      };
+    default:
+      throw new Error(`Unknown service: ${service}`);
+  }
+}
+
+function getElapsedMs(time: [number, number]) {
+  const diff = process.hrtime(time);
+  return diff[0] * 1000 + diff[1] / 1e6;
+}
+
+function guesstimateTokens(prompt: string) {
+  // From Anthropic's docs:
+  // The maximum length of prompt that Claude can see is its context window.
+  // Claude's context window is currently ~6500 words / ~8000 tokens /
+  // ~28000 Unicode characters.
+  // This suggests 0.28 tokens per character but in practice this seems to be
+  // a substantial underestimate in some cases.
+  return Math.ceil(prompt.length * 0.325);
+}
+
+function isPythonInstalled() {
+  try {
+    const python = process.platform === "win32" ? "python" : "python3";
+    childProcess.execSync(`${python} --version`, { stdio: "ignore" });
+    return true;
+  } catch (err) {
+    logger.debug({ err: err.message }, "Python not installed.");
+    return false;
+  }
+}

src/types/custom.d.ts

@@ -18,6 +18,9 @@ declare global {
       onAborted?: () => void;
       proceed: () => void;
       heartbeatInterval?: NodeJS.Timeout;
+      promptTokens?: number;
+      // TODO: remove later
+      debug: Record<string, any>;
     }
   }
 }

tsconfig.json

@@ -9,7 +9,9 @@
     "skipLibCheck": true,
     "skipDefaultLibCheck": true,
     "outDir": "build",
-    "sourceMap": true
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "useUnknownInCatchVariables": false
   },
   "include": ["src"],
   "exclude": ["node_modules"],