exempt 'special' token type from context size limits

src/proxy/middleware/request/preprocessors/count-prompt-tokens.ts

@@ -31,7 +31,10 @@ export const countPromptTokens: RequestPreprocessor = async (req) => {
     }
     case "anthropic-chat": {
       req.outputTokens = req.body.max_tokens;
-      const prompt: AnthropicChatMessage[] = req.body.messages;
+      const prompt = {
+        system: req.body.system ?? "",
+        messages: req.body.messages,
+      };
       result = await countTokens({ req, prompt, service });
       break;
     }

src/proxy/middleware/request/preprocessors/validate-context-size.ts

@@ -46,6 +46,11 @@ export const validateContextSize: RequestPreprocessor = async (req) => {
   }
   proxyMax ||= Number.MAX_SAFE_INTEGER;
 
+  if (req.user?.type === "special") {
+    req.log.debug("Special user, not enforcing proxy context limit.");
+    proxyMax = Number.MAX_SAFE_INTEGER;
+  }
+
   let modelMax: number;
   if (model.match(/gpt-3.5-turbo-16k/)) {
     modelMax = 16384;

src/proxy/middleware/response/index.ts

@@ -370,7 +370,7 @@ async function handleAnthropicBadRequestError(
       "Anthropic key has been disabled."
     );
     keyPool.disable(req.key!, "revoked");
-    errorPayload.proxy_note = `Assigned key has been disabled. ${error?.message}`;
+    errorPayload.proxy_note = `Assigned key has been disabled. (${error?.message})`;
     return;
   }