Adjust release o1 context to 200K

src/config.ts

@@ -415,18 +415,18 @@ export const config: Config = {
   firebaseKey: getEnvWithDefault("FIREBASE_KEY", undefined),
   textModelRateLimit: getEnvWithDefault("TEXT_MODEL_RATE_LIMIT", 4),
   imageModelRateLimit: getEnvWithDefault("IMAGE_MODEL_RATE_LIMIT", 4),
-  maxContextTokensOpenAI: getEnvWithDefault("MAX_CONTEXT_TOKENS_OPENAI", 16384),
+  maxContextTokensOpenAI: getEnvWithDefault("MAX_CONTEXT_TOKENS_OPENAI", 32768),
   maxContextTokensAnthropic: getEnvWithDefault(
     "MAX_CONTEXT_TOKENS_ANTHROPIC",
-    0
+    32768
   ),
   maxOutputTokensOpenAI: getEnvWithDefault(
     ["MAX_OUTPUT_TOKENS_OPENAI", "MAX_OUTPUT_TOKENS"],
-    400
+    1024
   ),
   maxOutputTokensAnthropic: getEnvWithDefault(
     ["MAX_OUTPUT_TOKENS_ANTHROPIC", "MAX_OUTPUT_TOKENS"],
-    400
+    1024
   ),
   allowedModelFamilies: getEnvWithDefault(
     "ALLOWED_MODEL_FAMILIES",

src/proxy/anthropic.ts

@@ -46,7 +46,7 @@ const getModelsResponse = () => {
     "claude-3-haiku-20240307",
     "claude-3-opus-20240229",
     "claude-3-sonnet-20240229",
-    "claude-3-5-sonnet-20240620"
+    "claude-3-5-sonnet-20240620",
   ];
 
   const models = claudeVariants.map((id) => ({
@@ -70,7 +70,7 @@ const handleModelRequest: RequestHandler = (_req, res) => {
 };
 
 /** Only used for non-streaming requests. */
-const anthropicResponseHandler: ProxyResHandlerWithBody = async (
+const anthropicBlockingResponseHandler: ProxyResHandlerWithBody = async (
   _proxyRes,
   req,
   res,
@@ -179,6 +179,28 @@ export function transformAnthropicChatResponseToOpenAI(
   };
 }
 
+/**
+ * If a client using the OpenAI compatibility endpoint requests an actual OpenAI
+ * model, reassigns it to Claude 3 Sonnet.
+ */
+function maybeReassignModel(req: Request) {
+  const model = req.body.model;
+  if (!model.startsWith("gpt-")) return;
+  req.body.model = "claude-3-sonnet-20240229";
+}
+
+/**
+ * If client requests more than 4096 output tokens the request must have a
+ * particular version header.
+ * https://docs.anthropic.com/en/release-notes/api#july-15th-2024
+ */
+function setAnthropicBetaHeader(req: Request) {
+  const { max_tokens_to_sample } = req.body;
+  if (max_tokens_to_sample > 4096) {
+    req.headers["anthropic-beta"] = "max-tokens-3-5-sonnet-2024-07-15";
+  }
+}
+
 const anthropicProxy = createQueueMiddleware({
   proxyMiddleware: createProxyMiddleware({
     target: "https://api.anthropic.com",
@@ -189,7 +211,7 @@ const anthropicProxy = createQueueMiddleware({
       proxyReq: createOnProxyReqHandler({
         pipeline: [addKey, addAnthropicPreamble, finalizeBody],
       }),
-      proxyRes: createOnProxyResHandler([anthropicResponseHandler]),
+      proxyRes: createOnProxyResHandler([anthropicBlockingResponseHandler]),
       error: handleProxyError,
     },
     // Abusing pathFilter to rewrite the paths dynamically.
@@ -213,6 +235,11 @@ const anthropicProxy = createQueueMiddleware({
   }),
 });
 
+const nativeAnthropicChatPreprocessor = createPreprocessorMiddleware(
+  { inApi: "anthropic-chat", outApi: "anthropic-chat", service: "anthropic" },
+  { afterTransform: [setAnthropicBetaHeader] }
+);
+
 const nativeTextPreprocessor = createPreprocessorMiddleware({
   inApi: "anthropic-text",
   outApi: "anthropic-text",
@@ -268,11 +295,7 @@ anthropicRouter.get("/v1/models", handleModelRequest);
 anthropicRouter.post(
   "/v1/messages",
   ipLimiter,
-  createPreprocessorMiddleware({
-    inApi: "anthropic-chat",
-    outApi: "anthropic-chat",
-    service: "anthropic",
-  }),
+  nativeAnthropicChatPreprocessor,
   anthropicProxy
 );
 // Anthropic text completion endpoint. Translates to Anthropic chat completion
@@ -292,65 +315,5 @@ anthropicRouter.post(
   preprocessOpenAICompatRequest,
   anthropicProxy
 );
-// Temporarily force Anthropic Text to Anthropic Chat for frontends which do not
-// yet support the new model. Forces claude-3. Will be removed once common
-// frontends have been updated.
-anthropicRouter.post(
-  "/v1/:type(sonnet|opus)/:action(complete|messages)",
-  ipLimiter,
-  handleAnthropicTextCompatRequest,
-  createPreprocessorMiddleware({
-    inApi: "anthropic-text",
-    outApi: "anthropic-chat",
-    service: "anthropic",
-  }),
-  anthropicProxy
-);
-
-function handleAnthropicTextCompatRequest(
-  req: Request,
-  res: Response,
-  next: any
-) {
-  const type = req.params.type;
-  const action = req.params.action;
-  const alreadyInChatFormat = Boolean(req.body.messages);
-  const compatModel = `claude-3-${type}-20240229`;
-  req.log.info(
-    { type, inputModel: req.body.model, compatModel, alreadyInChatFormat },
-    "Handling Anthropic compatibility request"
-  );
-
-  if (action === "messages" || alreadyInChatFormat) {
-    return sendErrorToClient({
-      req,
-      res,
-      options: {
-        title: "Unnecessary usage of compatibility endpoint",
-        message: `Your client seems to already support the new Claude API format. This endpoint is intended for clients that do not yet support the new format.\nUse the normal \`/anthropic\` proxy endpoint instead.`,
-        format: "unknown",
-        statusCode: 400,
-        reqId: req.id,
-        obj: {
-          requested_endpoint: "/anthropic/" + type,
-          correct_endpoint: "/anthropic",
-        },
-      },
-    });
-  }
-
-  req.body.model = compatModel;
-  next();
-}
-
-/**
- * If a client using the OpenAI compatibility endpoint requests an actual OpenAI
- * model, reassigns it to Claude 3 Sonnet.
- */
-function maybeReassignModel(req: Request) {
-  const model = req.body.model;
-  if (!model.startsWith("gpt-")) return;
-  req.body.model = "claude-3-sonnet-20240229";
-}
 
 export const anthropic = anthropicRouter;

src/proxy/middleware/request/onproxyreq/check-model-family.ts

@@ -1,14 +1,16 @@
-import { HPMRequestCallback } from "../index";
 import { config } from "../../../../config";
 import { ForbiddenError } from "../../../../shared/errors";
 import { getModelFamilyForRequest } from "../../../../shared/models";
+import { HPMRequestCallback } from "../index";
 
 /**
  * Ensures the selected model family is enabled by the proxy configuration.
- **/
-export const checkModelFamily: HPMRequestCallback = (_proxyReq, req, res) => {
+ */
+export const checkModelFamily: HPMRequestCallback = (_proxyReq, req) => {
   const family = getModelFamilyForRequest(req);
   if (!config.allowedModelFamilies.includes(family)) {
-    throw new ForbiddenError(`Model family '${family}' is not enabled on this proxy`);
+    throw new ForbiddenError(
+      `Model family '${family}' is not enabled on this proxy`
+    );
   }
 };

src/proxy/middleware/request/preprocessors/sign-aws-request.ts

@@ -35,6 +35,7 @@ export const signAwsRequest: RequestPreprocessor = async (req) => {
 
   const credential = getCredentialParts(req);
   const host = AMZ_HOST.replace("%REGION%", credential.region);
+
   // AWS only uses 2023-06-01 and does not actually check this header, but we
   // set it so that the stream adapter always selects the correct transformer.
   req.headers["anthropic-version"] = "2023-06-01";

src/proxy/middleware/request/preprocessors/transform-outbound-payload.ts

@@ -67,11 +67,13 @@ function applyMistralPromptFixes(req: Request): void {
     );
 
     // If the prompt relies on `prefix: true` for the last message, we need to
-    // convert it to a text completions request because Mistral support for
-    // this feature is limited (and completely broken on AWS Mistral).
+    // convert it to a text completions request because AWS Mistral support for
+    // this feature is broken.
+    // On Mistral La Plateforme, we can't do this because they don't expose
+    // a text completions endpoint.
     const { messages } = req.body;
     const lastMessage = messages && messages[messages.length - 1];
-    if (lastMessage && lastMessage.role === "assistant") {
+    if (lastMessage?.role === "assistant" && req.service === "aws") {
       // enable prefix if client forgot, otherwise the template will insert an
       // eos token which is very unlikely to be what the client wants.
       lastMessage.prefix = true;

src/proxy/middleware/request/preprocessors/validate-context-size.ts

@@ -7,7 +7,7 @@ import { RequestPreprocessor } from "../index";
 const CLAUDE_MAX_CONTEXT = config.maxContextTokensAnthropic;
 const OPENAI_MAX_CONTEXT = config.maxContextTokensOpenAI;
 // todo: make configurable
-const GOOGLE_AI_MAX_CONTEXT = 1024000;
+const GOOGLE_AI_MAX_CONTEXT = 2048000;
 const MISTRAL_AI_MAX_CONTENT = 131072;
 
 /**
@@ -58,6 +58,8 @@ export const validateContextSize: RequestPreprocessor = async (req) => {
     modelMax = 16384;
   } else if (model.match(/^gpt-4o/)) {
     modelMax = 128000;
+  } else if (model.match(/^chatgpt-4o/)) {
+    modelMax = 128000;
   } else if (model.match(/gpt-4-turbo(-\d{4}-\d{2}-\d{2})?$/)) {
     modelMax = 131072;
   } else if (model.match(/gpt-4-turbo(-preview)?$/)) {
@@ -66,6 +68,12 @@ export const validateContextSize: RequestPreprocessor = async (req) => {
     modelMax = 131072;
   } else if (model.match(/^gpt-4(-\d{4})?-vision(-preview)?$/)) {
     modelMax = 131072;
+  } else if (model.match(/^o1(-\d{4}-\d{2}-\d{2})?$/)) {
+    modelMax = 200000;
+  } else if (model.match(/^o1-mini(-\d{4}-\d{2}-\d{2})?$/)) {
+    modelMax = 128000;
+  } else if (model.match(/^o1-preview(-\d{4}-\d{2}-\d{2})?$/)) {
+    modelMax = 128000;
   } else if (model.match(/gpt-3.5-turbo/)) {
     modelMax = 16384;
   } else if (model.match(/gpt-4-32k/)) {

src/proxy/mistral-ai.ts

@@ -21,6 +21,7 @@ import {
   createOnProxyResHandler,
   ProxyResHandlerWithBody,
 } from "./middleware/response";
+import { BadRequestError } from "../shared/errors";
 
 // Mistral can't settle on a single naming scheme and deprecates models within
 // months of releasing them so this list is hard to keep up to date. 2024-07-28
@@ -170,7 +171,12 @@ export function detectMistralInputApi(req: Request) {
   if (messages) {
     req.inboundApi = "mistral-ai";
     req.outboundApi = "mistral-ai";
-  } else if (prompt) {
+  } else if (prompt && req.service === "mistral-ai") {
+    // Mistral La Plateforme doesn't expose a text completions endpoint.
+    throw new BadRequestError(
+      "Mistral (via La Plateforme API) does not support text completions. This format is only supported on Mistral via the AWS API."
+    );
+  } else if (prompt && req.service === "aws") {
     req.inboundApi = "mistral-text";
     req.outboundApi = "mistral-text";
   }

src/proxy/openai.ts

@@ -35,6 +35,8 @@ export const KNOWN_OPENAI_MODELS = [
   // GPT4o Mini
   "gpt-4o-mini",
   "gpt-4o-mini-2024-07-18",
+  // GPT4o (ChatGPT)
+  "chatgpt-4o-latest",
   // GPT4 Turbo (superceded by GPT4o)
   "gpt-4-turbo",
   "gpt-4-turbo-2024-04-09", // gpt4-turbo stable, with vision

src/shared/api-schemas/mistral-ai.ts

@@ -45,7 +45,9 @@ const BaseMistralAIV1CompletionsSchema = z.object({
     .default([])
     .transform((v) => (Array.isArray(v) ? v : [v])),
   random_seed: z.number().int().min(0).optional(),
-  response_format: z.enum(["text", "json_object"]).optional().default("text"),
+  response_format: z
+    .object({ type: z.enum(["text", "json_object"]) })
+    .optional(),
   safe_prompt: z.boolean().optional().default(false),
 });
 

src/shared/api-schemas/openai.ts

@@ -52,7 +52,7 @@ export const OpenAIV1ChatCompletionSchema = z
       .number()
       .int()
       .nullish()
-      .default(Math.min(OPENAI_OUTPUT_MAX, 4096))
+      .default(Math.min(OPENAI_OUTPUT_MAX, 16384))
       .transform((v) => Math.min(v ?? OPENAI_OUTPUT_MAX, OPENAI_OUTPUT_MAX)),
     frequency_penalty: z.number().optional().default(0),
     presence_penalty: z.number().optional().default(0),

src/shared/models.ts

@@ -130,6 +130,7 @@ export const IMAGE_GEN_MODELS: ModelFamily[] = ["dall-e", "azure-dall-e"];
 
 export const OPENAI_MODEL_FAMILY_MAP: { [regex: string]: OpenAIModelFamily } = {
   "^gpt-4o(-\\d{4}-\\d{2}-\\d{2})?$": "gpt4o",
+  "^chatgpt-4o": "gpt4o",
   "^gpt-4o-mini(-\\d{4}-\\d{2}-\\d{2})?$": "turbo", // closest match
   "^gpt-4-turbo(-\\d{4}-\\d{2}-\\d{2})?$": "gpt4-turbo",
   "^gpt-4-turbo(-preview)?$": "gpt4-turbo",