exempt 'special' token type from context size limits

src/shared/tokenization/claude.ts

@@ -19,7 +19,9 @@ export function init() {
   return true;
 }
 
-export async function getTokenCount(prompt: string | AnthropicChatMessage[]) {
+export async function getTokenCount(
+  prompt: string | { system: string; messages: AnthropicChatMessage[] }
+) {
   if (typeof prompt !== "string") {
     return getTokenCountForMessages(prompt);
   }
@@ -34,9 +36,17 @@ export async function getTokenCount(prompt: string | AnthropicChatMessage[]) {
   };
 }
 
-async function getTokenCountForMessages(messages: AnthropicChatMessage[]) {
+async function getTokenCountForMessages({
+  system,
+  messages,
+}: {
+  system: string;
+  messages: AnthropicChatMessage[];
+}) {
   let numTokens = 0;
 
+  numTokens += (await getTokenCount(system)).token_count;
+
   for (const message of messages) {
     const { content, role } = message;
     numTokens += role === "user" ? userRoleCount : assistantRoleCount;

src/shared/tokenization/tokenizer.ts

@@ -35,7 +35,7 @@ type OpenAIChatTokenCountRequest = {
 };
 
 type AnthropicChatTokenCountRequest = {
-  prompt: AnthropicChatMessage[];
+  prompt: { system: string; messages: AnthropicChatMessage[] };
   completion?: never;
   service: "anthropic-chat";
 };