diff --git a/src/proxy/aws.ts b/src/proxy/aws.ts index 98f073e..9356e11 100644 --- a/src/proxy/aws.ts +++ b/src/proxy/aws.ts @@ -211,7 +211,7 @@ awsRouter.post("/v1/claude-3/messages", (req, res) => { options: { title: "Proxy error (wrong endpoint)", message: - "Your client is attempting to use the /anthropic/v1/claude-3 compatibility endpoint, but supports the new API format and should use the normal /anthropic/v1 endpoint instead.", + "Your client is attempting to use the /aws/claude/claude-3 compatibility endpoint, but supports the new API format and should use the normal /aws/claude endpoint instead.", format: "unknown", statusCode: 404, reqId: req.id, diff --git a/src/proxy/middleware/common.ts b/src/proxy/middleware/common.ts index e3c0be1..1194ff8 100644 --- a/src/proxy/middleware/common.ts +++ b/src/proxy/middleware/common.ts @@ -5,6 +5,7 @@ import { generateErrorMessage } from "zod-error"; import { assertNever } from "../../shared/utils"; import { QuotaExceededError } from "./request/preprocessors/apply-quota-limits"; import { buildSpoofedSSE, sendErrorToClient } from "./response/error-generator"; +import { HttpError } from "../../shared/errors"; const OPENAI_CHAT_COMPLETION_ENDPOINT = "/v1/chat/completions"; const OPENAI_TEXT_COMPLETION_ENDPOINT = "/v1/completions"; @@ -111,6 +112,15 @@ function classifyError(err: Error): { }; switch (err.constructor.name) { + case "HttpError": + if ((err as HttpError).status === 402) { + return { + statusCode: 402, + statusMessage: "No Keys Available", + userMessage: err.message, + type: "proxy_no_keys_available", + }; + } else return defaultError; case "ZodError": const userMessage = generateErrorMessage((err as ZodError).issues, { prefix: "Request validation failed. ", diff --git a/src/proxy/middleware/response/error-generator.ts b/src/proxy/middleware/response/error-generator.ts index 1ba038b..9bfe61f 100644 --- a/src/proxy/middleware/response/error-generator.ts +++ b/src/proxy/middleware/response/error-generator.ts @@ -40,10 +40,8 @@ function getMessageContent({ } ``` */ - - const friendlyMessage = obj?.proxy_note - ? `${message}\n\n***\n\n*${obj.proxy_note}*` - : message; + const note = obj?.proxy_note || obj?.error?.message || ""; + const friendlyMessage = note ? `${message}\n\n***\n\n*${note}*` : message; const details = JSON.parse(JSON.stringify(obj ?? {})); let stack = ""; if (details.stack) { diff --git a/src/proxy/middleware/response/index.ts b/src/proxy/middleware/response/index.ts index e1b7c40..bd02b68 100644 --- a/src/proxy/middleware/response/index.ts +++ b/src/proxy/middleware/response/index.ts @@ -330,9 +330,8 @@ const handleUpstreamErrors: ProxyResHandlerWithBody = async ( errorPayload.proxy_note = `API key is invalid or revoked. ${tryAgainMessage}`; break; case "AccessDeniedException": - const isModelAccessError = errorPayload.error?.message?.includes( - `access to the model with the specified model ID` - ); + const isModelAccessError = + errorPayload.error?.message?.includes(`specified model ID`); if (!isModelAccessError) { req.log.error( { key: req.key?.hash, model: req.body?.model }, @@ -451,7 +450,7 @@ async function handleAnthropicBadRequestError( return; } - errorPayload.proxy_note = `Unrecognized 400 Bad Request error from the API.`; + errorPayload.proxy_note = `Unrecognized error from the API. (${error?.message})`; } async function handleAnthropicRateLimitError( diff --git a/src/proxy/middleware/response/streaming/sse-stream-adapter.ts b/src/proxy/middleware/response/streaming/sse-stream-adapter.ts index 487be16..d7e5832 100644 --- a/src/proxy/middleware/response/streaming/sse-stream-adapter.ts +++ b/src/proxy/middleware/response/streaming/sse-stream-adapter.ts @@ -21,6 +21,7 @@ type SSEStreamAdapterOptions = TransformOptions & { export class SSEStreamAdapter extends Transform { private readonly isAwsStream; private readonly isGoogleStream; + private api: APIFormat; private partialMessage = ""; private textDecoder = new TextDecoder("utf8"); private log: pino.Logger; @@ -30,6 +31,7 @@ export class SSEStreamAdapter extends Transform { this.isAwsStream = options?.contentType === "application/vnd.amazon.eventstream"; this.isGoogleStream = options?.api === "google-ai"; + this.api = options.api; this.log = options.logger.child({ module: "sse-stream-adapter" }); } @@ -51,13 +53,10 @@ export class SSEStreamAdapter extends Transform { const event = Buffer.from(bytes, "base64").toString("utf8"); const eventObj = JSON.parse(event); - if ('completion' in eventObj) { + if ("completion" in eventObj) { return ["event: completion", `data: ${event}`].join(`\n`); } else { - return [ - `event: ${eventObj.type}`, - `data: ${event}`, - ].join(`\n`); + return [`event: ${eventObj.type}`, `data: ${event}`].join(`\n`); } } // Intentional fallthrough, as non-JSON events may as well be errors @@ -75,15 +74,10 @@ export class SSEStreamAdapter extends Transform { throw new RetryableError("AWS request throttled mid-stream"); default: this.log.error({ message, type }, "Received bad AWS stream event"); - return buildSpoofedSSE({ - format: "anthropic-text", - title: "Proxy stream error", - message: - "The proxy received an unrecognized error from AWS while streaming.", - obj: message, - reqId: "proxy-sse-adapter-message", - model: "", - }); + const error: any = new Error(`Got mysterious error chunk: ${type}`); + error.lastEvent = message; + this.emit("error", error); + return null; } default: // Amazon says this can't ever happen... diff --git a/src/shared/errors.ts b/src/shared/errors.ts index ec6e122..d9a1a46 100644 --- a/src/shared/errors.ts +++ b/src/shared/errors.ts @@ -1,6 +1,7 @@ export class HttpError extends Error { constructor(public status: number, message: string) { super(message); + this.name = "HttpError"; } } diff --git a/src/shared/key-management/anthropic/provider.ts b/src/shared/key-management/anthropic/provider.ts index 643245e..96d2c2f 100644 --- a/src/shared/key-management/anthropic/provider.ts +++ b/src/shared/key-management/anthropic/provider.ts @@ -4,6 +4,7 @@ import { config } from "../../../config"; import { logger } from "../../../logger"; import { AnthropicModelFamily, getClaudeModelFamily } from "../../models"; import { AnthropicKeyChecker } from "./checker"; +import { HttpError } from "../../errors"; // https://docs.anthropic.com/claude/reference/selecting-a-model export type AnthropicModel = @@ -130,7 +131,7 @@ export class AnthropicKeyProvider implements KeyProvider { // certainly change when they move out of beta later this year. const availableKeys = this.keys.filter((k) => !k.isDisabled); if (availableKeys.length === 0) { - throw new Error("No Anthropic keys available."); + throw new HttpError(402, "No Anthropic keys available."); } // (largely copied from the OpenAI provider, without trial key support) diff --git a/src/shared/key-management/aws/provider.ts b/src/shared/key-management/aws/provider.ts index c874138..8590a9b 100644 --- a/src/shared/key-management/aws/provider.ts +++ b/src/shared/key-management/aws/provider.ts @@ -4,6 +4,7 @@ import { config } from "../../../config"; import { logger } from "../../../logger"; import type { AwsBedrockModelFamily } from "../../models"; import { AwsKeyChecker } from "./checker"; +import { HttpError } from "../../errors"; // https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html export type AwsBedrockModel = @@ -109,7 +110,8 @@ export class AwsBedrockKeyProvider implements KeyProvider { ); }); if (availableKeys.length === 0) { - throw new Error( + throw new HttpError( + 402, "No keys available for this model. If you are requesting Sonnet, use Claude-2 instead." ); } diff --git a/src/shared/key-management/azure/provider.ts b/src/shared/key-management/azure/provider.ts index c38f79c..f2522d0 100644 --- a/src/shared/key-management/azure/provider.ts +++ b/src/shared/key-management/azure/provider.ts @@ -6,6 +6,7 @@ import type { AzureOpenAIModelFamily } from "../../models"; import { getAzureOpenAIModelFamily } from "../../models"; import { OpenAIModel } from "../openai/provider"; import { AzureOpenAIKeyChecker } from "./checker"; +import { HttpError } from "../../errors"; export type AzureOpenAIModel = Exclude; @@ -100,7 +101,10 @@ export class AzureOpenAIKeyProvider implements KeyProvider { (k) => !k.isDisabled && k.modelFamilies.includes(neededFamily) ); if (availableKeys.length === 0) { - throw new Error(`No keys available for model family '${neededFamily}'.`); + throw new HttpError( + 402, + `No keys available for model family '${neededFamily}'.` + ); } // (largely copied from the OpenAI provider, without trial key support) diff --git a/src/shared/key-management/google-ai/provider.ts b/src/shared/key-management/google-ai/provider.ts index f25e075..97bbd07 100644 --- a/src/shared/key-management/google-ai/provider.ts +++ b/src/shared/key-management/google-ai/provider.ts @@ -3,6 +3,7 @@ import { Key, KeyProvider } from ".."; import { config } from "../../../config"; import { logger } from "../../../logger"; import type { GoogleAIModelFamily } from "../../models"; +import { HttpError } from "../../errors"; // Note that Google AI is not the same as Vertex AI, both are provided by Google // but Vertex is the GCP product for enterprise. while Google AI is the @@ -95,7 +96,7 @@ export class GoogleAIKeyProvider implements KeyProvider { public get(_model: GoogleAIModel) { const availableKeys = this.keys.filter((k) => !k.isDisabled); if (availableKeys.length === 0) { - throw new Error("No Google AI keys available"); + throw new HttpError(402, "No Google AI keys available"); } // (largely copied from the OpenAI provider, without trial key support) diff --git a/src/shared/key-management/mistral-ai/provider.ts b/src/shared/key-management/mistral-ai/provider.ts index 94856fa..f09f5e4 100644 --- a/src/shared/key-management/mistral-ai/provider.ts +++ b/src/shared/key-management/mistral-ai/provider.ts @@ -4,6 +4,7 @@ import { config } from "../../../config"; import { logger } from "../../../logger"; import { MistralAIModelFamily, getMistralAIModelFamily } from "../../models"; import { MistralAIKeyChecker } from "./checker"; +import { HttpError } from "../../errors"; type MistralAIKeyUsage = { [K in MistralAIModelFamily as `${K}Tokens`]: number; @@ -94,7 +95,7 @@ export class MistralAIKeyProvider implements KeyProvider { public get(_model: Model) { const availableKeys = this.keys.filter((k) => !k.isDisabled); if (availableKeys.length === 0) { - throw new Error("No Mistral AI keys available"); + throw new HttpError(402, "No Mistral AI keys available"); } // (largely copied from the OpenAI provider, without trial key support) diff --git a/src/shared/key-management/openai/provider.ts b/src/shared/key-management/openai/provider.ts index c61f767..ecca3d6 100644 --- a/src/shared/key-management/openai/provider.ts +++ b/src/shared/key-management/openai/provider.ts @@ -8,6 +8,7 @@ import { config } from "../../../config"; import { logger } from "../../../logger"; import { OpenAIKeyChecker } from "./checker"; import { getOpenAIModelFamily, OpenAIModelFamily } from "../../models"; +import { HttpError } from "../../errors"; export type OpenAIModel = | "gpt-3.5-turbo" @@ -17,7 +18,7 @@ export type OpenAIModel = | "gpt-4-1106" | "text-embedding-ada-002" | "dall-e-2" - | "dall-e-3" + | "dall-e-3"; // Flattening model families instead of using a nested object for easier // cloning. @@ -167,7 +168,10 @@ export class OpenAIKeyProvider implements KeyProvider { ); if (availableKeys.length === 0) { - throw new Error(`No keys available for model family '${neededFamily}'.`); + throw new HttpError( + 402, + `No keys available for model family '${neededFamily}'.` + ); } // Select a key, from highest priority to lowest priority: