From 8a0e7a4e3d5a20710fe086a7be92f9b814dbf59b Mon Sep 17 00:00:00 2001
From: Enrico Ros <enrico.ros@gmail.com>
Date: Thu, 16 May 2024 00:55:58 -0700
Subject: [PATCH] Tiktoken: in the future, show tokens

(cherry picked from commit d5c3f5012b316bd5d93080060e5f6790b734c2fa)
---
 src/common/util/token-counter.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts
index b6429fef5..de21026b6 100644
--- a/src/common/util/token-counter.ts
+++ b/src/common/util/token-counter.ts
@@ -6,6 +6,9 @@ import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms';
 // Do not set this to true in production, it's very verbose
 const DEBUG_TOKEN_COUNT = false;
 
+// Globals
+// const tokenEncodings: string[] = ['gpt2', 'r50k_base', 'p50k_base', 'p50k_edit', 'cl100k_base', 'o200k_base'] satisfies TiktokenEncoding[];
+
 // Global symbols to dynamically load the Tiktoken library
 let get_encoding: ((encoding: TiktokenEncoding) => Tiktoken) | null = null;
 let encoding_for_model: ((model: TiktokenModel) => Tiktoken) | null = null;