uses accurate Claude tokenization

2023-08-08 17:29:36 -05:00
parent 81ceee7897
commit d1d83b41fa
4 changed files with 18 additions and 30 deletions
@@ -16,6 +16,7 @@
  "author": "",
  "license": "MIT",
  "dependencies": {
+    "@anthropic-ai/tokenizer": "^0.0.4",
    "axios": "^1.3.5",
    "cookie-parser": "^1.4.6",
    "copyfiles": "^2.4.1",