uses accurate Claude tokenization

2023-08-08 17:29:36 -05:00
parent 81ceee7897
commit d1d83b41fa
4 changed files with 18 additions and 30 deletions
@@ -9,6 +9,7 @@
      "version": "1.0.0",
      "license": "MIT",
      "dependencies": {
+        "@anthropic-ai/tokenizer": "^0.0.4",
        "axios": "^1.3.5",
        "cookie-parser": "^1.4.6",
        "copyfiles": "^2.4.1",
@@ -47,6 +48,15 @@
        "node": ">=18.0.0"
      }
    },
+    "node_modules/@anthropic-ai/tokenizer": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/@anthropic-ai/tokenizer/-/tokenizer-0.0.4.tgz",
+      "integrity": "sha512-EHRKbxlxlc8W4KCBEseByJ7YwyYCmgu9OyN59H9+IYIGPoKv8tXyQXinkeGDI+cI8Tiuz9wk2jZb/kK7AyvL7g==",
+      "dependencies": {
+        "@types/node": "^18.11.18",
+        "tiktoken": "^1.0.10"
+      }
+    },
    "node_modules/@babel/parser": {
      "version": "7.22.7",
      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.7.tgz",