diff --git a/.env.example b/.env.example index 99666e4..0c084e3 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,9 @@ # Use production mode unless you are developing locally. NODE_ENV=production +# Detail level of diagnostic logging. (trace | debug | info | warn | error) +# LOG_LEVEL=info + # ------------------------------------------------------------------------------ # General settings: @@ -24,23 +27,22 @@ NODE_ENV=production # Max number of context tokens a user can request at once. # Increase this if your proxy allow GPT 32k or 128k context -# MAX_CONTEXT_TOKENS_OPENAI=16384 +# MAX_CONTEXT_TOKENS_OPENAI=32768 +# MAX_CONTEXT_TOKENS_ANTHROPIC=32768 # Max number of output tokens a user can request at once. -# MAX_OUTPUT_TOKENS_OPENAI=400 -# MAX_OUTPUT_TOKENS_ANTHROPIC=400 +# MAX_OUTPUT_TOKENS_OPENAI=1024 +# MAX_OUTPUT_TOKENS_ANTHROPIC=1024 # Whether to show the estimated cost of consumed tokens on the info page. # SHOW_TOKEN_COSTS=false # Whether to automatically check API keys for validity. -# Note: CHECK_KEYS is disabled by default in local development mode, but enabled -# by default in production mode. +# Disabled by default in local development mode, but enabled in production. # CHECK_KEYS=true # Which model types users are allowed to access. # The following model families are recognized: - # turbo | gpt4 | gpt4-32k | gpt4-turbo | gpt4o | o1 | dall-e | claude # | claude-opus | gemini-flash | gemini-pro | gemini-ultra | mistral-tiny | # | mistral-small | mistral-medium | mistral-large | aws-claude | @@ -60,6 +62,42 @@ NODE_ENV=production # By default, no image services are allowed and image prompts are rejected. # ALLOWED_VISION_SERVICES= +# Whether prompts should be logged to Google Sheets. +# Requires additional setup. See `docs/google-sheets.md` for more information. +# PROMPT_LOGGING=false + +# Specifies the number of proxies or load balancers in front of the server. +# For Cloudflare or Hugging Face deployments, the default of 1 is correct. +# For any other deployments, please see config.ts as the correct configuration +# depends on your setup. Misconfiguring this value can result in problems +# accurately tracking IP addresses and enforcing rate limits. +# TRUSTED_PROXIES=1 + +# Whether cookies should be set without the Secure flag, for hosts that don't +# support SSL. True by default in development, false in production. +# USE_INSECURE_COOKIES=false + +# Reorganizes requests in the queue according to their token count, placing +# larger prompts further back. The penalty is determined by (promptTokens * +# TOKENS_PUNISHMENT_FACTOR). A value of 1.0 adds one second per 1000 tokens. +# When there is no queue or it is very short, the effect is negligible (this +# setting only reorders the queue, it does not artificially delay requests). +# TOKENS_PUNISHMENT_FACTOR=0.0 + +# Captcha verification settings. Refer to docs/pow-captcha.md for guidance. +# CAPTCHA_MODE=none +# POW_TOKEN_HOURS=24 +# POW_TOKEN_MAX_IPS=2 +# POW_DIFFICULTY_LEVEL=low +# POW_CHALLENGE_TIMEOUT=30 + +# ------------------------------------------------------------------------------- +# Blocking settings: +# Allows blocking requests depending on content, referers, or IP addresses. +# This is a convenience feature; if you need more robust functionality it is +# highly recommended to put this application behind nginx or Cloudflare, as they +# will have better performance. + # IP addresses or CIDR blocks from which requests will be blocked. # IP_BLACKLIST=10.0.0.1/24 # URLs from which requests will be blocked. @@ -68,35 +106,13 @@ NODE_ENV=production # BLOCK_MESSAGE="You must be over the age of majority in your country to use this service." # Destination to redirect blocked requests to. # BLOCK_REDIRECT="https://roblox.com/" - -# Comma-separated list of phrases that will be rejected. Only whole words are matched. -# Surround phrases with quotes if they contain commas. -# Avoid short or common phrases as this tests the entire prompt. +# Comma-separated list of phrases that will be rejected. Surround phrases with +# quotes if they contain commas. You can use regular expression tokens. +# Avoid overly broad phrases as will trigger on any match in the entire prompt. # REJECT_PHRASES="phrase one,phrase two,"phrase three, which has a comma",phrase four" # Message to show when requests are rejected. # REJECT_MESSAGE="You can't say that here." -# Whether prompts should be logged to Google Sheets. -# Requires additional setup. See `docs/google-sheets.md` for more information. -# PROMPT_LOGGING=false - -# The port and network interface to listen on. -# PORT=7860 -# BIND_ADDRESS=0.0.0.0 - -# Whether cookies should be set without the Secure flag, for hosts that don't support SSL. -# USE_INSECURE_COOKIES=false - -# Detail level of logging. (trace | debug | info | warn | error) -# LOG_LEVEL=info - -# Captcha verification settings. Refer to docs/pow-captcha.md for guidance. -# CAPTCHA_MODE=none -# POW_TOKEN_HOURS=24 -# POW_TOKEN_MAX_IPS=2 -# POW_DIFFICULTY_LEVEL=low -# POW_CHALLENGE_TIMEOUT=30 - # ------------------------------------------------------------------------------ # Optional settings for user management, access control, and quota enforcement: # See `docs/user-management.md` for more information and setup instructions. @@ -116,15 +132,8 @@ NODE_ENV=production # ALLOW_NICKNAME_CHANGES=true # Default token quotas for each model family. (0 for unlimited) -# Specify as TOKEN_QUOTA_MODEL_FAMILY=value, replacing dashes with underscores. -# TOKEN_QUOTA_TURBO=0 -# TOKEN_QUOTA_GPT4=0 -# TOKEN_QUOTA_GPT4_32K=0 -# TOKEN_QUOTA_GPT4_TURBO=0 -# TOKEN_QUOTA_CLAUDE=0 -# TOKEN_QUOTA_GEMINI_PRO=0 -# TOKEN_QUOTA_AWS_CLAUDE=0 -# TOKEN_QUOTA_GCP_CLAUDE=0 +# Specify as TOKEN_QUOTA_MODEL_FAMILY=value (replacing dashes with underscores). +# eg. TOKEN_QUOTA_TURBO=0, TOKEN_QUOTA_GPT4=1000000, TOKEN_QUOTA_GPT4_32K=100000 # "Tokens" for image-generation models are counted at a rate of 100000 tokens # per US$1.00 generated, which is similar to the cost of GPT-4 Turbo. # DALL-E 3 costs around US$0.10 per image (10000 tokens). @@ -135,12 +144,22 @@ NODE_ENV=production # Leave unset to never automatically refresh quotas. # QUOTA_REFRESH_PERIOD=daily -# Specifies the number of proxies or load balancers in front of the server. -# For Cloudflare or Hugging Face deployments, the default of 1 is correct. -# For any other deployments, please see config.ts as the correct configuration -# depends on your setup. Misconfiguring this value can result in problems -# accurately tracking IP addresses and enforcing rate limits. -# TRUSTED_PROXIES=1 +# ------------------------------------------------------------------------------- +# HTTP agent settings: +# If you need to change how the proxy makes requests to other servers, such +# as when checking keys or forwarding users' requests to external services, +# you can configure an alternative HTTP agent. Otherwise the default OS settings +# will be used. + +# The name of the network interface to use. The first external IPv4 address +# belonging to this interface will be used for outgoing requests. +# HTTP_AGENT_INTERFACE=enp0s3 + +# The URL of a proxy server to use. Supports SOCKS4, SOCKS5, HTTP, and HTTPS. +# Note that if your proxy server issues a self-signed certificate, you may need +# NODE_EXTRA_CA_CERTS set to the path to your certificate. You will need to set +# that variable in your environment, not in this file. +# HTTP_AGENT_PROXY_URL=http://test:test@127.0.0.1:8000 # ------------------------------------------------------------------------------ # Secrets and keys: @@ -164,11 +183,10 @@ GCP_CREDENTIALS=project-id:client-email:region:private-key # With user_token gatekeeper, the admin password used to manage users. # ADMIN_KEY=your-very-secret-key -# To restrict access to the admin interface to specific IP addresses, set the -# ADMIN_WHITELIST environment variable to a comma-separated list of CIDR blocks. +# Restrict access to the admin interface to specific IP addresses, specified +# as a comma-separated list of CIDR ranges. # ADMIN_WHITELIST=0.0.0.0/0 - # With firebase_rtdb gatekeeper storage, the Firebase project credentials. # FIREBASE_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx # FIREBASE_RTDB_URL=https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.firebaseio.com @@ -176,10 +194,3 @@ GCP_CREDENTIALS=project-id:client-email:region:private-key # With prompt logging, the Google Sheets credentials. # GOOGLE_SHEETS_SPREADSHEET_ID=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx # GOOGLE_SHEETS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - -# Prioritize requests in the queue according to their -# token count, placing larger requests further back. -# -# Punishes requests with a second's delay per 1k tokens -# when the value is 1.0, two seconds when it's 2, etc. -# TOKENS_PUNISHMENT_FACTOR=0.0 diff --git a/package-lock.json b/package-lock.json index f3d331f..aa9da87 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "oai-reverse-proxy", "version": "1.0.0", + "hasInstallScript": true, "license": "MIT", "dependencies": { "@anthropic-ai/tokenizer": "^0.0.4", @@ -27,18 +28,21 @@ "csrf-csrf": "^2.3.0", "dotenv": "^16.3.1", "ejs": "^3.1.10", - "express": "^4.18.2", + "express": "^4.19.3", "express-session": "^1.17.3", - "firebase-admin": "^12.3.1", + "firebase-admin": "^12.5.0", "glob": "^10.3.12", "googleapis": "^122.0.0", - "http-proxy-middleware": "^3.0.0-beta.1", + "http-proxy": "1.18.1", + "http-proxy-middleware": "^3.0.2", "ipaddr.js": "^2.1.0", "memorystore": "^1.6.7", "multer": "^1.4.5-lts.1", "node-schedule": "^2.1.1", + "patch-package": "^8.0.0", "pino": "^8.11.0", "pino-http": "^8.3.3", + "proxy-agent": "^6.4.0", "sanitize-html": "^2.13.0", "sharp": "^0.32.6", "showdown": "^2.1.0", @@ -1551,6 +1555,11 @@ "node": ">= 10" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -1665,9 +1674,9 @@ } }, "node_modules/@types/http-proxy": { - "version": "1.17.10", - "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.10.tgz", - "integrity": "sha512-Qs5aULi+zV1bwKAg5z1PWnDXWmsn+LxIvUGv6E2+OOMYhclZMO+OXd9pYVf2gLykf2I7IV2u7oTHwChPNsvJ7g==", + "version": "1.17.15", + "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.15.tgz", + "integrity": "sha512-25g5atgiVNTIv0LBDTg1H74Hvayx0ajtJPLLcYE3whFv75J0pWNtOBzaXJQgDTmrX1bx5U9YC2w/n65BN1HwRQ==", "dependencies": { "@types/node": "*" } @@ -1805,6 +1814,11 @@ "integrity": "sha512-rFT3ak0/2trgvp4yYZo5iKFEPsET7vKydKF+VRCxlQ9bpheehyAJH89dAkaLEq/j/RZXJIqcgsmPJKUP1Z28HA==", "dev": true }, + "node_modules/@yarnpkg/lockfile": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", + "integrity": "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==" + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1946,6 +1960,17 @@ "node": ">=8" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", @@ -1965,6 +1990,14 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "node_modules/at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -2012,6 +2045,14 @@ } ] }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/better-sqlite3": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-10.0.0.tgz", @@ -2094,9 +2135,9 @@ } }, "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -2106,7 +2147,7 @@ "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.11.0", + "qs": "6.13.0", "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" @@ -2189,12 +2230,18 @@ } }, "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -2274,6 +2321,20 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "engines": { + "node": ">=8" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -2626,6 +2687,14 @@ "http-errors": "^2.0.0" } }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/date-fns": { "version": "2.29.3", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", @@ -2686,6 +2755,35 @@ "node": ">=0.10.0" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -2865,9 +2963,9 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "engines": { "node": ">= 0.8" } @@ -2886,6 +2984,25 @@ "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==", "optional": true }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.17.16", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.16.tgz", @@ -2971,6 +3088,54 @@ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -3009,36 +3174,36 @@ } }, "node_modules/express": { - "version": "4.19.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", - "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", + "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.2", + "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", + "finalhandler": "1.3.1", "fresh": "0.5.2", "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", + "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", + "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", + "send": "0.19.0", + "serve-static": "1.16.2", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -3201,12 +3366,12 @@ } }, "node_modules/finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "dependencies": { "debug": "2.6.9", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "2.4.1", "parseurl": "~1.3.3", @@ -3217,10 +3382,18 @@ "node": ">= 0.8" } }, + "node_modules/find-yarn-workspace-root": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz", + "integrity": "sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ==", + "dependencies": { + "micromatch": "^4.0.2" + } + }, "node_modules/firebase-admin": { - "version": "12.3.1", - "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-12.3.1.tgz", - "integrity": "sha512-vEr3s3esl8nPIA9r/feDT4nzIXCfov1CyyCSpMQWp6x63Q104qke0MEGZlrHUZVROtl8FLus6niP/M9I1s4VBA==", + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-12.5.0.tgz", + "integrity": "sha512-ad8vnlPcuuZN9scSgY8UnAxPI4mzP2/Q+dsrVLTf+j3h7bIq0FOelDCDGz4StgKJdk244v2kpOxqJjPG3grBHg==", "dependencies": { "@fastify/busboy": "^3.0.0", "@firebase/database-compat": "^1.0.2", @@ -3328,6 +3501,19 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -3348,9 +3534,12 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -3393,18 +3582,58 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", - "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-uri": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", + "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4", + "fs-extra": "^11.2.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/get-uri/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -3669,6 +3898,22 @@ "node": ">=12.0.0" } }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, "node_modules/gtoken": { "version": "6.1.2", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", @@ -3682,17 +3927,6 @@ "node": ">=12.0.0" } }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -3702,6 +3936,28 @@ "node": ">=4" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", @@ -3713,6 +3969,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/help-me": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/help-me/-/help-me-4.2.0.tgz", @@ -3877,27 +4144,27 @@ "optional": true }, "node_modules/http-proxy-middleware": { - "version": "3.0.0-beta.1", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-3.0.0-beta.1.tgz", - "integrity": "sha512-hdiTlVVoaxncf239csnEpG5ew2lRWnoNR1PMWOO6kYulSphlrfLs5JFZtFVH3R5EUWSZNMkeUqvkvfctuWaK8A==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-3.0.2.tgz", + "integrity": "sha512-fBLFpmvDzlxdckwZRjM0wWtwDZ4KBtQ8NFqhrFKoEtK4myzuiumBuNTxD+F4cVbXfOZljIbrynmvByofDzT7Ag==", "dependencies": { - "@types/http-proxy": "^1.17.10", - "debug": "^4.3.4", + "@types/http-proxy": "^1.17.15", + "debug": "^4.3.6", "http-proxy": "^1.18.1", - "is-glob": "^4.0.1", - "is-plain-obj": "^3.0.0", - "micromatch": "^4.0.5" + "is-glob": "^4.0.3", + "is-plain-object": "^5.0.0", + "micromatch": "^4.0.8" }, "engines": { - "node": ">=12.0.0" + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/http-proxy-middleware/node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -3909,9 +4176,9 @@ } }, "node_modules/http-proxy-middleware/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/https-proxy-agent": { "version": "5.0.1", @@ -4016,6 +4283,18 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", @@ -4041,6 +4320,20 @@ "node": ">=8" } }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -4076,17 +4369,6 @@ "node": ">=0.12.0" } }, - "node_modules/is-plain-obj": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", - "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-plain-object": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", @@ -4106,6 +4388,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -4167,6 +4460,11 @@ "node": ">=10" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -4175,6 +4473,47 @@ "bignumber.js": "^9.0.0" } }, + "node_modules/json-stable-stringify": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.1.1.tgz", + "integrity": "sha512-SU/971Kt5qVQfJpyDveVhQ/vya+5hvrjClFOcr8c0Fq5aODJjMwutrOfCU+eCnVD5gpx1Q3fEqkyom77zH1iIg==", + "dependencies": { + "call-bind": "^1.0.5", + "isarray": "^2.0.5", + "jsonify": "^0.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/json-stable-stringify/node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.1.tgz", + "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/jsonwebtoken": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz", @@ -4270,6 +4609,14 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/klaw-sync": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", + "integrity": "sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ==", + "dependencies": { + "graceful-fs": "^4.1.11" + } + }, "node_modules/limiter": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", @@ -4392,9 +4739,12 @@ "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" }, "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/methods": { "version": "1.1.2", @@ -4405,11 +4755,11 @@ } }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -4563,6 +4913,14 @@ "node": ">= 0.6" } }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-abi": { "version": "3.51.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.51.0.tgz", @@ -4734,13 +5092,24 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz", @@ -4773,6 +5142,29 @@ "wrappy": "1" } }, + "node_modules/open": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", + "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", + "dependencies": { + "is-docker": "^2.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -4788,6 +5180,92 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pac-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", + "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.5", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/pac-proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parse-srcset": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz", @@ -4801,6 +5279,49 @@ "node": ">= 0.8" } }, + "node_modules/patch-package": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-8.0.0.tgz", + "integrity": "sha512-da8BVIhzjtgScwDJ2TtKsfT5JFWz1hYoBl9rUQ1f38MC2HwnEIkK8VN3dKMKcP7P7bvvgzNDbfNHtx3MsQb5vA==", + "dependencies": { + "@yarnpkg/lockfile": "^1.1.0", + "chalk": "^4.1.2", + "ci-info": "^3.7.0", + "cross-spawn": "^7.0.3", + "find-yarn-workspace-root": "^2.0.0", + "fs-extra": "^9.0.0", + "json-stable-stringify": "^1.0.2", + "klaw-sync": "^6.0.0", + "minimist": "^1.2.6", + "open": "^7.4.2", + "rimraf": "^2.6.3", + "semver": "^7.5.3", + "slash": "^2.0.0", + "tmp": "^0.0.33", + "yaml": "^2.2.2" + }, + "bin": { + "patch-package": "index.js" + }, + "engines": { + "node": ">=14", + "npm": ">5" + } + }, + "node_modules/patch-package/node_modules/fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -4841,9 +5362,9 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==" }, "node_modules/picocolors": { "version": "1.0.0", @@ -5121,6 +5642,88 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -5146,12 +5749,20 @@ "once": "^1.3.1" } }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "engines": { + "node": ">=6" + } + }, "node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -5292,6 +5903,38 @@ "node": ">=14" } }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/rxjs": { "version": "7.8.0", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz", @@ -5378,9 +6021,9 @@ } }, "node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -5400,25 +6043,49 @@ "node": ">= 0.8.0" } }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/send/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "dependencies": { - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.18.0" + "send": "0.19.0" }, "engines": { "node": ">= 0.8.0" } }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -5490,13 +6157,17 @@ } }, "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -5576,6 +6247,81 @@ "node": ">=10" } }, + "node_modules/slash": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", + "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socks-proxy-agent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/sonic-boom": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.3.0.tgz", @@ -5628,6 +6374,11 @@ "node": ">= 10.x" } }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -5860,6 +6611,17 @@ "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.14.tgz", "integrity": "sha512-g5zd5r/DoH8Kw0fiYbYpVhb6WO8BHO1unXqmBBWKwoT17HwSounnDtMDFUKm2Pko8U47sjQarOe+9aUrnqmmTg==" }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -5892,9 +6654,15 @@ } }, "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz", + "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } }, "node_modules/tree-kill": { "version": "1.2.2", @@ -6016,6 +6784,14 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.13.0.tgz", "integrity": "sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg==" }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -6077,9 +6853,12 @@ } }, "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } }, "node_modules/websocket-driver": { "version": "0.7.4", @@ -6103,12 +6882,15 @@ } }, "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz", + "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==", "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" + "tr46": "^5.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" } }, "node_modules/which": { @@ -6184,6 +6966,17 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/yaml": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.1.tgz", + "integrity": "sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", diff --git a/package.json b/package.json index 01abfab..db72180 100644 --- a/package.json +++ b/package.json @@ -5,9 +5,11 @@ "scripts": { "build": "tsc && copyfiles -u 1 src/**/*.ejs build", "database:migrate": "ts-node scripts/migrate.ts", + "postinstall": "patch-package", "prepare": "husky install", - "start": "node build/server.js", + "start": "node --trace-deprecation --trace-warnings build/server.js", "start:dev": "nodemon --watch src --exec ts-node --transpile-only src/server.ts", + "start:debug": "ts-node --inspect --transpile-only src/server.ts", "start:replit": "tsc && node build/server.js", "start:watch": "nodemon --require source-map-support/register build/server.js", "type-check": "tsc --noEmit" @@ -36,18 +38,21 @@ "csrf-csrf": "^2.3.0", "dotenv": "^16.3.1", "ejs": "^3.1.10", - "express": "^4.18.2", + "express": "^4.19.3", "express-session": "^1.17.3", - "firebase-admin": "^12.3.1", + "firebase-admin": "^12.5.0", "glob": "^10.3.12", "googleapis": "^122.0.0", - "http-proxy-middleware": "^3.0.0-beta.1", + "http-proxy": "1.18.1", + "http-proxy-middleware": "^3.0.2", "ipaddr.js": "^2.1.0", "memorystore": "^1.6.7", "multer": "^1.4.5-lts.1", "node-schedule": "^2.1.1", + "patch-package": "^8.0.0", "pino": "^8.11.0", "pino-http": "^8.3.3", + "proxy-agent": "^6.4.0", "sanitize-html": "^2.13.0", "sharp": "^0.32.6", "showdown": "^2.1.0", @@ -84,8 +89,8 @@ "typescript": "^5.4.2" }, "overrides": { - "braces": "^3.0.3", - "fast-xml-parser": "^4.4.1", - "follow-redirects": "^1.15.4" + "node-fetch@2.x": { + "whatwg-url": "14.x" + } } } diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000..9dfa8d9 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,23 @@ +# Patches +Contains monkey patches for certain packages, applied using `patch-package`. + +## `http-proxy+1.18.1.patch` +Modifies the `http-proxy` package to work around an incompatibility with +body-parser and SOCKS5 proxies due to some esoteric stream handling behavior +when `socks-proxy-agent` is used instead of a generic http.Agent. + +Modification involves adjusting the `buffer` property on ProxyServer's `options` +object to be a function that returns a stream instead of a stream itself. This +allows us to give it a function which produces a new Readable from the already- +parsed request body. + +With the old implementation we would need to create an entirely new ProxyServer +instance for each request, which is not ideal under heavy load. + +`http-proxy` hasn't been updated in six years so it's unlikely that this patch +will be broken by future updates, but it's stil pinned to 1.18.1 for now. + +### See also +https://github.com/chimurai/http-proxy-middleware/issues/40 +https://github.com/chimurai/http-proxy-middleware/issues/299 +https://github.com/http-party/node-http-proxy/pull/1027 diff --git a/patches/http-proxy+1.18.1.patch b/patches/http-proxy+1.18.1.patch new file mode 100644 index 0000000..e4192e9 --- /dev/null +++ b/patches/http-proxy+1.18.1.patch @@ -0,0 +1,13 @@ +diff --git a/node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js b/node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js +index 7ae7355..c825c27 100644 +--- a/node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js ++++ b/node_modules/http-proxy/lib/http-proxy/passes/web-incoming.js +@@ -167,7 +167,7 @@ module.exports = { + } + } + +- (options.buffer || req).pipe(proxyReq); ++ (options.buffer(req) || req).pipe(proxyReq); + + proxyReq.on('response', function(proxyRes) { + if(server) { server.emit('proxyRes', proxyRes, req, res); } diff --git a/src/config.ts b/src/config.ts index e7651e5..db38ffa 100644 --- a/src/config.ts +++ b/src/config.ts @@ -385,6 +385,36 @@ type Config = { * Accepts floats. */ tokensPunishmentFactor: number; + /** + * Configuration for HTTP requests made by the proxy to other servers, such + * as when checking keys or forwarding users' requests to external services. + * If not set, all requests will be made using the default agent. + * + * If set, the proxy may make requests to other servers using the specified + * settings. This is useful if you wish to route users' requests through + * another proxy or VPN, or if you have multiple network interfaces and want + * to use a specific one for outgoing requests. + */ + httpAgent?: { + /** + * The name of the network interface to use. The first external IPv4 address + * belonging to this interface will be used for outgoing requests. + */ + interface?: string; + /** + * The URL of a proxy server to use. Supports SOCKS4, SOCKS5, HTTP, and + * HTTPS. If not set, the proxy will be made using the default agent. + * - SOCKS4: `socks4://some-socks-proxy.com:9050` + * - SOCKS5: `socks5://username:password@some-socks-proxy.com:9050` + * - HTTP: `http://proxy-server-over-tcp.com:3128` + * - HTTPS: `https://proxy-server-over-tls.com:3129` + * + * **Note:** If your proxy server issues a certificate, you may need to set + * `NODE_EXTRA_CA_CERTS` to the path to your certificate, otherwise this + * application will reject TLS connections. + */ + proxyUrl?: string; + }; }; // To change configs, create a file called .env in the root directory. @@ -491,6 +521,10 @@ export const config: Config = { ), ipBlacklist: parseCsv(getEnvWithDefault("IP_BLACKLIST", "")), tokensPunishmentFactor: getEnvWithDefault("TOKENS_PUNISHMENT_FACTOR", 0.0), + httpAgent: { + interface: getEnvWithDefault("HTTP_AGENT_INTERFACE", undefined), + proxyUrl: getEnvWithDefault("HTTP_AGENT_PROXY_URL", undefined), + }, } as const; function generateSigningKey() { @@ -610,6 +644,16 @@ export async function assertConfigIsValid() { ); } + if (Object.values(config.httpAgent || {}).filter(Boolean).length === 0) { + delete config.httpAgent; + } else if (config.httpAgent) { + if (config.httpAgent.interface && config.httpAgent.proxyUrl) { + throw new Error( + "Cannot set both `HTTP_AGENT_INTERFACE` and `HTTP_AGENT_PROXY_URL`." + ); + } + } + // Ensure forks which add new secret-like config keys don't unwittingly expose // them to users. for (const key of getKeys(config)) { @@ -623,15 +667,16 @@ export async function assertConfigIsValid() { `Config key "${key}" may be sensitive but is exposed. Add it to SENSITIVE_KEYS or OMITTED_KEYS.` ); } - - await maybeInitializeFirebase(); } /** * Config keys that are masked on the info page, but not hidden as their * presence may be relevant to the user due to privacy implications. */ -export const SENSITIVE_KEYS: (keyof Config)[] = ["googleSheetsSpreadsheetId"]; +export const SENSITIVE_KEYS: (keyof Config)[] = [ + "googleSheetsSpreadsheetId", + "httpAgent", +]; /** * Config keys that are not displayed on the info page at all, generally because @@ -755,32 +800,6 @@ function getEnvWithDefault(env: string | string[], defaultValue: T): T { } } -let firebaseApp: firebase.app.App | undefined; - -async function maybeInitializeFirebase() { - if (!config.gatekeeperStore.startsWith("firebase")) { - return; - } - - const firebase = await import("firebase-admin"); - const firebaseKey = Buffer.from(config.firebaseKey!, "base64").toString(); - const app = firebase.initializeApp({ - credential: firebase.credential.cert(JSON.parse(firebaseKey)), - databaseURL: config.firebaseRtdbUrl, - }); - - await app.database().ref("connection-test").set(Date.now()); - - firebaseApp = app; -} - -export function getFirebaseApp(): firebase.app.App { - if (!firebaseApp) { - throw new Error("Firebase app not initialized."); - } - return firebaseApp; -} - function parseCsv(val: string): string[] { if (!val) return []; diff --git a/src/proxy/anthropic.ts b/src/proxy/anthropic.ts index bc7e627..155f574 100644 --- a/src/proxy/anthropic.ts +++ b/src/proxy/anthropic.ts @@ -1,22 +1,14 @@ -import { Request, Response, RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; +import { Request, RequestHandler, Router } from "express"; import { config } from "../config"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { addKey, - addAnthropicPreamble, createPreprocessorMiddleware, finalizeBody, - createOnProxyReqHandler, } from "./middleware/request"; -import { - ProxyResHandlerWithBody, - createOnProxyResHandler, -} from "./middleware/response"; -import { sendErrorToClient } from "./middleware/response/error-generator"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; +import { ProxyReqManager } from "./middleware/request/proxy-req-manager"; let modelsCache: any = null; let modelsCacheTime = 0; @@ -69,7 +61,6 @@ const handleModelRequest: RequestHandler = (_req, res) => { res.status(200).json(getModelsResponse()); }; -/** Only used for non-streaming requests. */ const anthropicBlockingResponseHandler: ProxyResHandlerWithBody = async ( _proxyRes, req, @@ -123,13 +114,7 @@ export function transformAnthropicChatResponseToAnthropicText( }; } -/** - * Transforms a model response from the Anthropic API to match those from the - * OpenAI API, for users using Claude via the OpenAI-compatible endpoint. This - * is only used for non-streaming requests as streaming requests are handled - * on-the-fly. - */ -export function transformAnthropicTextResponseToOpenAI( +function transformAnthropicTextResponseToOpenAI( anthropicBody: Record, req: Request ): Record { @@ -201,38 +186,30 @@ function setAnthropicBetaHeader(req: Request) { } } -const anthropicProxy = createQueueMiddleware({ - proxyMiddleware: createProxyMiddleware({ - target: "https://api.anthropic.com", - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ - pipeline: [addKey, addAnthropicPreamble, finalizeBody], - }), - proxyRes: createOnProxyResHandler([anthropicBlockingResponseHandler]), - error: handleProxyError, - }, - // Abusing pathFilter to rewrite the paths dynamically. - pathFilter: (pathname, req) => { - const isText = req.outboundApi === "anthropic-text"; - const isChat = req.outboundApi === "anthropic-chat"; - if (isChat && pathname === "/v1/complete") { - req.url = "/v1/messages"; - } - if (isText && pathname === "/v1/chat/completions") { - req.url = "/v1/complete"; - } - if (isChat && pathname === "/v1/chat/completions") { - req.url = "/v1/messages"; - } - if (isChat && ["sonnet", "opus"].includes(req.params.type)) { - req.url = "/v1/messages"; - } - return true; - }, - }), +function selectUpstreamPath(manager: ProxyReqManager) { + const req = manager.request; + const pathname = req.url.split("?")[0]; + req.log.debug({ pathname }, "Anthropic path filter"); + const isText = req.outboundApi === "anthropic-text"; + const isChat = req.outboundApi === "anthropic-chat"; + if (isChat && pathname === "/v1/complete") { + manager.setPath("/v1/messages"); + } + if (isText && pathname === "/v1/chat/completions") { + manager.setPath("/v1/complete"); + } + if (isChat && pathname === "/v1/chat/completions") { + manager.setPath("/v1/messages"); + } + if (isChat && ["sonnet", "opus"].includes(req.params.type)) { + manager.setPath("/v1/messages"); + } +} + +const anthropicProxy = createQueuedProxyMiddleware({ + target: "https://api.anthropic.com", + mutations: [selectUpstreamPath, addKey, finalizeBody], + blockingResponseHandler: anthropicBlockingResponseHandler, }); const nativeAnthropicChatPreprocessor = createPreprocessorMiddleware( diff --git a/src/proxy/aws-claude.ts b/src/proxy/aws-claude.ts index a00e4e6..7648b1b 100644 --- a/src/proxy/aws-claude.ts +++ b/src/proxy/aws-claude.ts @@ -1,27 +1,19 @@ import { Request, RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; import { v4 } from "uuid"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; -import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; -import { - createPreprocessorMiddleware, - signAwsRequest, - finalizeSignedRequest, - createOnProxyReqHandler, -} from "./middleware/request"; -import { - ProxyResHandlerWithBody, - createOnProxyResHandler, -} from "./middleware/response"; import { transformAnthropicChatResponseToAnthropicText, transformAnthropicChatResponseToOpenAI, } from "./anthropic"; +import { ipLimiter } from "./rate-limit"; +import { + createPreprocessorMiddleware, + finalizeSignedRequest, + signAwsRequest, +} from "./middleware/request"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; -/** Only used for non-streaming requests. */ -const awsResponseHandler: ProxyResHandlerWithBody = async ( +const awsBlockingResponseHandler: ProxyResHandlerWithBody = async ( _proxyRes, req, res, @@ -55,12 +47,6 @@ const awsResponseHandler: ProxyResHandlerWithBody = async ( res.status(200).json({ ...newBody, proxy: body.proxy }); }; -/** - * Transforms a model response from the Anthropic API to match those from the - * OpenAI API, for users using Claude via the OpenAI-compatible endpoint. This - * is only used for non-streaming requests as streaming requests are handled - * on-the-fly. - */ function transformAwsTextResponseToOpenAI( awsBody: Record, req: Request @@ -89,23 +75,13 @@ function transformAwsTextResponseToOpenAI( }; } -const awsClaudeProxy = createQueueMiddleware({ - beforeProxy: signAwsRequest, - proxyMiddleware: createProxyMiddleware({ - target: "bad-target-will-be-rewritten", - router: ({ signedRequest }) => { - if (!signedRequest) throw new Error("Must sign request before proxying"); - return `${signedRequest.protocol}//${signedRequest.hostname}`; - }, - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [finalizeSignedRequest] }), - proxyRes: createOnProxyResHandler([awsResponseHandler]), - error: handleProxyError, - }, - }), +const awsClaudeProxy = createQueuedProxyMiddleware({ + target: ({ signedRequest }) => { + if (!signedRequest) throw new Error("Must sign request before proxying"); + return `${signedRequest.protocol}//${signedRequest.hostname}`; + }, + mutations: [signAwsRequest,finalizeSignedRequest], + blockingResponseHandler: awsBlockingResponseHandler, }); const nativeTextPreprocessor = createPreprocessorMiddleware( diff --git a/src/proxy/aws-mistral.ts b/src/proxy/aws-mistral.ts index 446658b..285ad8b 100644 --- a/src/proxy/aws-mistral.ts +++ b/src/proxy/aws-mistral.ts @@ -1,21 +1,16 @@ -import { Request } from "express"; +import { Request, Router } from "express"; import { - createOnProxyResHandler, - ProxyResHandlerWithBody, -} from "./middleware/response"; -import { createQueueMiddleware } from "./queue"; + detectMistralInputApi, + transformMistralTextToMistralChat, +} from "./mistral-ai"; +import { ipLimiter } from "./rate-limit"; +import { ProxyResHandlerWithBody } from "./middleware/response"; import { - createOnProxyReqHandler, createPreprocessorMiddleware, finalizeSignedRequest, signAwsRequest, } from "./middleware/request"; -import { createProxyMiddleware } from "http-proxy-middleware"; -import { logger } from "../logger"; -import { handleProxyError } from "./middleware/common"; -import { Router } from "express"; -import { ipLimiter } from "./rate-limit"; -import { detectMistralInputApi, transformMistralTextToMistralChat } from "./mistral-ai"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; const awsMistralBlockingResponseHandler: ProxyResHandlerWithBody = async ( _proxyRes, @@ -39,23 +34,13 @@ const awsMistralBlockingResponseHandler: ProxyResHandlerWithBody = async ( res.status(200).json({ ...newBody, proxy: body.proxy }); }; -const awsMistralProxy = createQueueMiddleware({ - beforeProxy: signAwsRequest, - proxyMiddleware: createProxyMiddleware({ - target: "bad-target-will-be-rewritten", - router: ({ signedRequest }) => { - if (!signedRequest) throw new Error("Must sign request before proxying"); - return `${signedRequest.protocol}//${signedRequest.hostname}`; - }, - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [finalizeSignedRequest] }), - proxyRes: createOnProxyResHandler([awsMistralBlockingResponseHandler]), - error: handleProxyError, - }, - }), +const awsMistralProxy = createQueuedProxyMiddleware({ + target: ({ signedRequest }) => { + if (!signedRequest) throw new Error("Must sign request before proxying"); + return `${signedRequest.protocol}//${signedRequest.hostname}`; + }, + mutations: [signAwsRequest,finalizeSignedRequest], + blockingResponseHandler: awsMistralBlockingResponseHandler, }); function maybeReassignModel(req: Request) { diff --git a/src/proxy/azure.ts b/src/proxy/azure.ts index c7fe20d..a4f9ea2 100644 --- a/src/proxy/azure.ts +++ b/src/proxy/azure.ts @@ -1,21 +1,14 @@ import { RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; import { config } from "../config"; -import { logger } from "../logger"; import { generateModelList } from "./openai"; -import { createQueueMiddleware } from "./queue"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { addAzureKey, - createOnProxyReqHandler, createPreprocessorMiddleware, finalizeSignedRequest, } from "./middleware/request"; -import { - createOnProxyResHandler, - ProxyResHandlerWithBody, -} from "./middleware/response"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; let modelsCache: any = null; let modelsCacheTime = 0; @@ -47,26 +40,17 @@ const azureOpenaiResponseHandler: ProxyResHandlerWithBody = async ( res.status(200).json({ ...body, proxy: body.proxy }); }; -const azureOpenAIProxy = createQueueMiddleware({ - beforeProxy: addAzureKey, - proxyMiddleware: createProxyMiddleware({ - target: "will be set by router", - router: (req) => { - if (!req.signedRequest) throw new Error("signedRequest not set"); - const { hostname, path } = req.signedRequest; - return `https://${hostname}${path}`; - }, - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [finalizeSignedRequest] }), - proxyRes: createOnProxyResHandler([azureOpenaiResponseHandler]), - error: handleProxyError, - }, - }), +const azureOpenAIProxy = createQueuedProxyMiddleware({ + target: ({ signedRequest }) => { + if (!signedRequest) throw new Error("Must sign request before proxying"); + const { hostname, path } = signedRequest; + return `https://${hostname}${path}`; + }, + mutations: [addAzureKey, finalizeSignedRequest], + blockingResponseHandler: azureOpenaiResponseHandler, }); + const azureOpenAIRouter = Router(); azureOpenAIRouter.get("/v1/models", handleModelRequest); azureOpenAIRouter.post( diff --git a/src/proxy/gcp.ts b/src/proxy/gcp.ts index 99a5ef6..d439e57 100644 --- a/src/proxy/gcp.ts +++ b/src/proxy/gcp.ts @@ -1,21 +1,15 @@ import { Request, RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; import { config } from "../config"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; +import { transformAnthropicChatResponseToOpenAI } from "./anthropic"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { createPreprocessorMiddleware, - signGcpRequest, finalizeSignedRequest, - createOnProxyReqHandler, + signGcpRequest, } from "./middleware/request"; -import { - ProxyResHandlerWithBody, - createOnProxyResHandler, -} from "./middleware/response"; -import { transformAnthropicChatResponseToOpenAI } from "./anthropic"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; + const LATEST_GCP_SONNET_MINOR_VERSION = "20240229"; let modelsCache: any = null; @@ -56,8 +50,7 @@ const handleModelRequest: RequestHandler = (_req, res) => { res.status(200).json(getModelsResponse()); }; -/** Only used for non-streaming requests. */ -const gcpResponseHandler: ProxyResHandlerWithBody = async ( +const gcpBlockingResponseHandler: ProxyResHandlerWithBody = async ( _proxyRes, req, res, @@ -78,23 +71,13 @@ const gcpResponseHandler: ProxyResHandlerWithBody = async ( res.status(200).json({ ...newBody, proxy: body.proxy }); }; -const gcpProxy = createQueueMiddleware({ - beforeProxy: signGcpRequest, - proxyMiddleware: createProxyMiddleware({ - target: "bad-target-will-be-rewritten", - router: ({ signedRequest }) => { - if (!signedRequest) throw new Error("Must sign request before proxying"); - return `${signedRequest.protocol}//${signedRequest.hostname}`; - }, - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [finalizeSignedRequest] }), - proxyRes: createOnProxyResHandler([gcpResponseHandler]), - error: handleProxyError, - }, - }), +const gcpProxy = createQueuedProxyMiddleware({ + target: ({ signedRequest }) => { + if (!signedRequest) throw new Error("Must sign request before proxying"); + return `${signedRequest.protocol}//${signedRequest.hostname}`; + }, + mutations: [signGcpRequest, finalizeSignedRequest], + blockingResponseHandler: gcpBlockingResponseHandler, }); const oaiToChatPreprocessor = createPreprocessorMiddleware( @@ -138,7 +121,7 @@ gcpRouter.post( * - frontends sending Anthropic model names that GCP doesn't recognize * - frontends sending OpenAI model names because they expect the proxy to * translate them - * + * * If client sends GCP model ID it will be used verbatim. Otherwise, various * strategies are used to try to map a non-GCP model name to GCP model ID. */ @@ -167,7 +150,7 @@ function maybeReassignModel(req: Request) { } const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, _rev] = match; - + const ver = minor ? `${major}.${minor}` : major; switch (ver) { case "3": diff --git a/src/proxy/google-ai.ts b/src/proxy/google-ai.ts index 6c9abca..94b245b 100644 --- a/src/proxy/google-ai.ts +++ b/src/proxy/google-ai.ts @@ -1,22 +1,15 @@ import { Request, RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; import { v4 } from "uuid"; +import { GoogleAIKey, keyPool } from "../shared/key-management"; import { config } from "../config"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { - createOnProxyReqHandler, createPreprocessorMiddleware, finalizeSignedRequest, } from "./middleware/request"; -import { - createOnProxyResHandler, - ProxyResHandlerWithBody, -} from "./middleware/response"; -import { addGoogleAIKey } from "./middleware/request/preprocessors/add-google-ai-key"; -import { GoogleAIKey, keyPool } from "../shared/key-management"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { addGoogleAIKey } from "./middleware/request/mutators/add-google-ai-key"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; let modelsCache: any = null; let modelsCacheTime = 0; @@ -63,8 +56,7 @@ const handleModelRequest: RequestHandler = (_req, res) => { res.status(200).json(getModelsResponse()); }; -/** Only used for non-streaming requests. */ -const googleAIResponseHandler: ProxyResHandlerWithBody = async ( +const googleAIBlockingResponseHandler: ProxyResHandlerWithBody = async ( _proxyRes, req, res, @@ -110,33 +102,14 @@ function transformGoogleAIResponse( }; } -const googleAIProxy = createQueueMiddleware({ - beforeProxy: addGoogleAIKey, - proxyMiddleware: createProxyMiddleware({ - target: "bad-target-will-be-rewritten", - router: ({ signedRequest }) => { - const { protocol, hostname, path } = signedRequest; - return `${protocol}//${hostname}${path}`; - }, - changeOrigin: true, - selfHandleResponse: true, - // Prevent logging of the API key by HPM - logger: logger.child( - {}, - { - redact: { - paths: ["*"], - censor: (v) => - typeof v === "string" ? v.replace(/key=\S+/g, "key=xxxxxxx") : v, - }, - } - ), - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [finalizeSignedRequest] }), - proxyRes: createOnProxyResHandler([googleAIResponseHandler]), - error: handleProxyError, - }, - }), +const googleAIProxy = createQueuedProxyMiddleware({ + target: ({ signedRequest }) => { + if (!signedRequest) throw new Error("Must sign request before proxying"); + const { protocol, hostname, path } = signedRequest; + return `${protocol}//${hostname}${path}`; + }, + mutations: [addGoogleAIKey, finalizeSignedRequest], + blockingResponseHandler: googleAIBlockingResponseHandler, }); const googleAIRouter = Router(); diff --git a/src/proxy/middleware/common.ts b/src/proxy/middleware/common.ts index 8ea71ff..3f5dab1 100644 --- a/src/proxy/middleware/common.ts +++ b/src/proxy/middleware/common.ts @@ -1,6 +1,6 @@ import { Request, Response } from "express"; import http from "http"; -import httpProxy from "http-proxy"; +import { Socket } from "net"; import { ZodError } from "zod"; import { generateErrorMessage } from "zod-error"; import { HttpError } from "../../shared/errors"; @@ -72,16 +72,23 @@ export function sendProxyError( }); } -export const handleProxyError: httpProxy.ErrorCallback = (err, req, res) => { - req.log.error(err, `Error during http-proxy-middleware request`); - classifyErrorAndSend(err, req as Request, res as Response); -}; - +/** + * Handles errors thrown during preparation of a proxy request (before it is + * sent to the upstream API), typically due to validation, quota, or other + * pre-flight checks. Depending on the error class, this function will send an + * appropriate error response to the client, streaming it if necessary. + */ export const classifyErrorAndSend = ( err: Error, req: Request, - res: Response + res: Response | Socket ) => { + if (res instanceof Socket) { + // We should always have an Express response object here, but http-proxy's + // ErrorCallback type says it could be just a Socket. + req.log.error(err, "Caught error while proxying request to target but cannot send error response to client."); + return res.destroy(); + } try { const { statusCode, statusMessage, userMessage, ...errorDetails } = classifyError(err); diff --git a/src/proxy/middleware/request/index.ts b/src/proxy/middleware/request/index.ts index e2aba44..94c313c 100644 --- a/src/proxy/middleware/request/index.ts +++ b/src/proxy/middleware/request/index.ts @@ -1,44 +1,38 @@ import type { Request } from "express"; -import type { ClientRequest } from "http"; -import type { ProxyReqCallback } from "http-proxy"; -export { createOnProxyReqHandler } from "./onproxyreq-factory"; +import { ProxyReqManager } from "./proxy-req-manager"; export { createPreprocessorMiddleware, createEmbeddingsPreprocessorMiddleware, } from "./preprocessor-factory"; -// Express middleware (runs before http-proxy-middleware, can be async) -export { addAzureKey } from "./preprocessors/add-azure-key"; +// Preprocessors (runs before request is queued, usually body transformation/validation) export { applyQuotaLimits } from "./preprocessors/apply-quota-limits"; +export { blockZoomerOrigins } from "./preprocessors/block-zoomer-origins"; export { countPromptTokens } from "./preprocessors/count-prompt-tokens"; export { languageFilter } from "./preprocessors/language-filter"; export { setApiFormat } from "./preprocessors/set-api-format"; -export { signAwsRequest } from "./preprocessors/sign-aws-request"; -export { signGcpRequest } from "./preprocessors/sign-vertex-ai-request"; export { transformOutboundPayload } from "./preprocessors/transform-outbound-payload"; export { validateContextSize } from "./preprocessors/validate-context-size"; +export { validateModelFamily } from "./preprocessors/validate-model-family"; export { validateVision } from "./preprocessors/validate-vision"; -// http-proxy-middleware callbacks (runs on onProxyReq, cannot be async) -export { addAnthropicPreamble } from "./onproxyreq/add-anthropic-preamble"; -export { addKey, addKeyForEmbeddingsRequest } from "./onproxyreq/add-key"; -export { blockZoomerOrigins } from "./onproxyreq/block-zoomer-origins"; -export { checkModelFamily } from "./onproxyreq/check-model-family"; -export { finalizeBody } from "./onproxyreq/finalize-body"; -export { finalizeSignedRequest } from "./onproxyreq/finalize-signed-request"; -export { stripHeaders } from "./onproxyreq/strip-headers"; +// Proxy request mutators (runs every time request is dequeued, before proxying, usually for auth/signing) +export { addKey, addKeyForEmbeddingsRequest } from "./mutators/add-key"; +export { addAzureKey } from "./mutators/add-azure-key"; +export { finalizeBody } from "./mutators/finalize-body"; +export { finalizeSignedRequest } from "./mutators/finalize-signed-request"; +export { signAwsRequest } from "./mutators/sign-aws-request"; +export { signGcpRequest } from "./mutators/sign-vertex-ai-request"; +export { stripHeaders } from "./mutators/strip-headers"; /** - * Middleware that runs prior to the request being handled by http-proxy- - * middleware. + * Middleware that runs prior to the request being queued or handled by + * http-proxy-middleware. You will not have access to the proxied + * request/response objects since they have not yet been sent to the API. * - * Async functions can be used here, but you will not have access to the proxied - * request/response objects, nor the data set by ProxyRequestMiddleware - * functions as they have not yet been run. - * - * User will have been authenticated by the time this middleware runs, but your - * request won't have been assigned an API key yet. + * User will have been authenticated by the proxy's gatekeeper, but the request + * won't have been assigned an upstream API key yet. * * Note that these functions only run once ever per request, even if the request * is automatically retried by the request queue middleware. @@ -46,17 +40,14 @@ export { stripHeaders } from "./onproxyreq/strip-headers"; export type RequestPreprocessor = (req: Request) => void | Promise; /** - * Callbacks that run immediately before the request is sent to the API in - * response to http-proxy-middleware's `proxyReq` event. + * Middleware that runs immediately before the request is proxied to the + * upstream API, after dequeueing the request from the request queue. * - * Async functions cannot be used here as HPM's event emitter is not async and - * will not wait for the promise to resolve before sending the request. - * - * Note that these functions may be run multiple times per request if the - * first attempt is rate limited and the request is automatically retried by the - * request queue middleware. + * Because these middleware may be run multiple times per request if a retryable + * error occurs and the request put back in the queue, they must be idempotent. + * A change manager is provided to allow the middleware to make changes to the + * request which can be automatically reverted. */ -export type HPMRequestCallback = ProxyReqCallback; - -export const forceModel = (model: string) => (req: Request) => - void (req.body.model = model); +export type ProxyReqMutator = ( + changeManager: ProxyReqManager +) => void | Promise; diff --git a/src/proxy/middleware/request/preprocessors/add-azure-key.ts b/src/proxy/middleware/request/mutators/add-azure-key.ts similarity index 83% rename from src/proxy/middleware/request/preprocessors/add-azure-key.ts rename to src/proxy/middleware/request/mutators/add-azure-key.ts index a0a0c19..bc493de 100644 --- a/src/proxy/middleware/request/preprocessors/add-azure-key.ts +++ b/src/proxy/middleware/request/mutators/add-azure-key.ts @@ -3,14 +3,16 @@ import { AzureOpenAIKey, keyPool, } from "../../../../shared/key-management"; -import { RequestPreprocessor } from "../index"; +import { ProxyReqMutator } from "../index"; -export const addAzureKey: RequestPreprocessor = (req) => { +export const addAzureKey: ProxyReqMutator = async (manager) => { + const req = manager.request; const validAPIs: APIFormat[] = ["openai", "openai-image"]; const apisValid = [req.outboundApi, req.inboundApi].every((api) => validAPIs.includes(api) ); const serviceValid = req.service === "azure"; + if (!apisValid || !serviceValid) { throw new Error("addAzureKey called on invalid request"); } @@ -22,11 +24,15 @@ export const addAzureKey: RequestPreprocessor = (req) => { const model = req.body.model.startsWith("azure-") ? req.body.model : `azure-${req.body.model}`; - - req.key = keyPool.get(model, "azure"); + // TODO: untracked mutation to body, I think this should just be a + // RequestPreprocessor because we don't need to do it every dequeue. req.body.model = model; + const key = keyPool.get(model, "azure"); + manager.setKey(key); + // Handles the sole Azure API deviation from the OpenAI spec (that I know of) + // TODO: this should also probably be a RequestPreprocessor const notNullOrUndefined = (x: any) => x !== null && x !== undefined; if ([req.body.logprobs, req.body.top_logprobs].some(notNullOrUndefined)) { // OpenAI wants logprobs: true/false and top_logprobs: number @@ -43,7 +49,7 @@ export const addAzureKey: RequestPreprocessor = (req) => { } req.log.info( - { key: req.key.hash, model }, + { key: key.hash, model }, "Assigned Azure OpenAI key to request" ); @@ -55,7 +61,7 @@ export const addAzureKey: RequestPreprocessor = (req) => { const apiVersion = req.outboundApi === "openai" ? "2023-09-01-preview" : "2024-02-15-preview"; - req.signedRequest = { + manager.setSignedRequest({ method: "POST", protocol: "https:", hostname: `${resourceName}.openai.azure.com`, @@ -66,7 +72,7 @@ export const addAzureKey: RequestPreprocessor = (req) => { ["api-key"]: apiKey, }, body: JSON.stringify(req.body), - }; + }); }; function getCredentialsFromKey(key: AzureOpenAIKey) { diff --git a/src/proxy/middleware/request/preprocessors/add-google-ai-key.ts b/src/proxy/middleware/request/mutators/add-google-ai-key.ts similarity index 67% rename from src/proxy/middleware/request/preprocessors/add-google-ai-key.ts rename to src/proxy/middleware/request/mutators/add-google-ai-key.ts index b2dd1d3..4643a79 100644 --- a/src/proxy/middleware/request/preprocessors/add-google-ai-key.ts +++ b/src/proxy/middleware/request/mutators/add-google-ai-key.ts @@ -1,7 +1,8 @@ import { keyPool } from "../../../../shared/key-management"; -import { RequestPreprocessor } from "../index"; +import { ProxyReqMutator} from "../index"; -export const addGoogleAIKey: RequestPreprocessor = (req) => { +export const addGoogleAIKey: ProxyReqMutator = (manager) => { + const req = manager.request; const inboundValid = req.inboundApi === "openai" || req.inboundApi === "google-ai"; const outboundValid = req.outboundApi === "google-ai"; @@ -12,10 +13,11 @@ export const addGoogleAIKey: RequestPreprocessor = (req) => { } const model = req.body.model; - req.isStreaming = req.isStreaming || req.body.stream; - req.key = keyPool.get(model, "google-ai"); + const key = keyPool.get(model, "google-ai"); + manager.setKey(key); + req.log.info( - { key: req.key.hash, model, stream: req.isStreaming }, + { key: key.hash, model, stream: req.isStreaming }, "Assigned Google AI API key to request" ); @@ -23,17 +25,20 @@ export const addGoogleAIKey: RequestPreprocessor = (req) => { // https://generativelanguage.googleapis.com/v1beta/models/$MODEL_ID:streamGenerateContent?key=${API_KEY} const payload = { ...req.body, stream: undefined, model: undefined }; - req.signedRequest = { + // TODO: this isn't actually signed, so the manager api is a little unclear + // with the ProxyReqManager refactor, it's probably no longer necesasry to + // do this because we can modify the path using Manager.setPath. + manager.setSignedRequest({ method: "POST", protocol: "https:", hostname: "generativelanguage.googleapis.com", path: `/v1beta/models/${model}:${ req.isStreaming ? "streamGenerateContent" : "generateContent" - }?key=${req.key.key}`, + }?key=${key.key}`, headers: { ["host"]: `generativelanguage.googleapis.com`, ["content-type"]: "application/json", }, body: JSON.stringify(payload), - }; + }); }; diff --git a/src/proxy/middleware/request/onproxyreq/add-key.ts b/src/proxy/middleware/request/mutators/add-key.ts similarity index 80% rename from src/proxy/middleware/request/onproxyreq/add-key.ts rename to src/proxy/middleware/request/mutators/add-key.ts index 5333496..db2a3af 100644 --- a/src/proxy/middleware/request/onproxyreq/add-key.ts +++ b/src/proxy/middleware/request/mutators/add-key.ts @@ -2,10 +2,12 @@ import { AnthropicChatMessage } from "../../../../shared/api-schemas"; import { containsImageContent } from "../../../../shared/api-schemas/anthropic"; import { Key, OpenAIKey, keyPool } from "../../../../shared/key-management"; import { isEmbeddingsRequest } from "../../common"; -import { HPMRequestCallback } from "../index"; import { assertNever } from "../../../../shared/utils"; +import { ProxyReqMutator } from "../index"; + +export const addKey: ProxyReqMutator = (manager) => { + const req = manager.request; -export const addKey: HPMRequestCallback = (proxyReq, req) => { let assignedKey: Key; const { service, inboundApi, outboundApi, body } = req; @@ -58,7 +60,7 @@ export const addKey: HPMRequestCallback = (proxyReq, req) => { } } - req.key = assignedKey; + manager.setKey(assignedKey); req.log.info( { key: assignedKey.hash, model: body.model, inboundApi, outboundApi }, "Assigned key to request" @@ -67,21 +69,21 @@ export const addKey: HPMRequestCallback = (proxyReq, req) => { // TODO: KeyProvider should assemble all necessary headers switch (assignedKey.service) { case "anthropic": - proxyReq.setHeader("X-API-Key", assignedKey.key); + manager.setHeader("X-API-Key", assignedKey.key); break; case "openai": const key: OpenAIKey = assignedKey as OpenAIKey; if (key.organizationId && !key.key.includes("svcacct")) { - proxyReq.setHeader("OpenAI-Organization", key.organizationId); + manager.setHeader("OpenAI-Organization", key.organizationId); } - proxyReq.setHeader("Authorization", `Bearer ${assignedKey.key}`); + manager.setHeader("Authorization", `Bearer ${assignedKey.key}`); break; case "mistral-ai": - proxyReq.setHeader("Authorization", `Bearer ${assignedKey.key}`); + manager.setHeader("Authorization", `Bearer ${assignedKey.key}`); break; case "azure": const azureKey = assignedKey.key; - proxyReq.setHeader("api-key", azureKey); + manager.setHeader("api-key", azureKey); break; case "aws": case "gcp": @@ -96,10 +98,8 @@ export const addKey: HPMRequestCallback = (proxyReq, req) => { * Special case for embeddings requests which don't go through the normal * request pipeline. */ -export const addKeyForEmbeddingsRequest: HPMRequestCallback = ( - proxyReq, - req -) => { +export const addKeyForEmbeddingsRequest: ProxyReqMutator = (manager) => { + const req = manager.request; if (!isEmbeddingsRequest(req)) { throw new Error( "addKeyForEmbeddingsRequest called on non-embeddings request" @@ -110,18 +110,18 @@ export const addKeyForEmbeddingsRequest: HPMRequestCallback = ( throw new Error("Embeddings requests must be from OpenAI"); } - req.body = { input: req.body.input, model: "text-embedding-ada-002" }; + manager.setBody({ input: req.body.input, model: "text-embedding-ada-002" }); const key = keyPool.get("text-embedding-ada-002", "openai") as OpenAIKey; - req.key = key; + manager.setKey(key); req.log.info( { key: key.hash, toApi: req.outboundApi }, "Assigned Turbo key to embeddings request" ); - proxyReq.setHeader("Authorization", `Bearer ${key.key}`); + manager.setHeader("Authorization", `Bearer ${key.key}`); if (key.organizationId) { - proxyReq.setHeader("OpenAI-Organization", key.organizationId); + manager.setHeader("OpenAI-Organization", key.organizationId); } }; diff --git a/src/proxy/middleware/request/mutators/finalize-body.ts b/src/proxy/middleware/request/mutators/finalize-body.ts new file mode 100644 index 0000000..658510d --- /dev/null +++ b/src/proxy/middleware/request/mutators/finalize-body.ts @@ -0,0 +1,22 @@ +import type { ProxyReqMutator } from "../index"; + +/** Finalize the rewritten request body. Must be the last mutator. */ +export const finalizeBody: ProxyReqMutator = (manager) => { + const req = manager.request; + + if (["POST", "PUT", "PATCH"].includes(req.method ?? "") && req.body) { + // For image generation requests, remove stream flag. + if (req.outboundApi === "openai-image") { + delete req.body.stream; + } + // For anthropic text to chat requests, remove undefined prompt. + if (req.outboundApi === "anthropic-chat") { + delete req.body.prompt; + } + + const serialized = + typeof req.body === "string" ? req.body : JSON.stringify(req.body); + manager.setHeader("Content-Length", String(Buffer.byteLength(serialized))); + manager.setBody(serialized); + } +}; diff --git a/src/proxy/middleware/request/mutators/finalize-signed-request.ts b/src/proxy/middleware/request/mutators/finalize-signed-request.ts new file mode 100644 index 0000000..f7f39b3 --- /dev/null +++ b/src/proxy/middleware/request/mutators/finalize-signed-request.ts @@ -0,0 +1,32 @@ +import { ProxyReqMutator } from "../index"; + +/** + * For AWS/GCP/Azure/Google requests, the body is signed earlier in the request + * pipeline, before the proxy middleware. This function just assigns the path + * and headers to the proxy request. + */ +export const finalizeSignedRequest: ProxyReqMutator = (manager) => { + const req = manager.request; + if (!req.signedRequest) { + throw new Error("Expected req.signedRequest to be set"); + } + + // The path depends on the selected model and the assigned key's region. + manager.setPath(req.signedRequest.path); + + // Amazon doesn't want extra headers, so we need to remove all of them and + // reassign only the ones specified in the signed request. + const headers = req.signedRequest.headers; + Object.keys(headers).forEach((key) => { + manager.removeHeader(key); + }); + Object.entries(req.signedRequest.headers).forEach(([key, value]) => { + manager.setHeader(key, value); + }); + const serialized = + typeof req.signedRequest.body === "string" + ? req.signedRequest.body + : JSON.stringify(req.signedRequest.body); + manager.setHeader("Content-Length", String(Buffer.byteLength(serialized))); + manager.setBody(serialized); +}; diff --git a/src/proxy/middleware/request/preprocessors/sign-aws-request.ts b/src/proxy/middleware/request/mutators/sign-aws-request.ts similarity index 80% rename from src/proxy/middleware/request/preprocessors/sign-aws-request.ts rename to src/proxy/middleware/request/mutators/sign-aws-request.ts index 5a937bc..28aeb72 100644 --- a/src/proxy/middleware/request/preprocessors/sign-aws-request.ts +++ b/src/proxy/middleware/request/mutators/sign-aws-request.ts @@ -7,11 +7,11 @@ import { AnthropicV1MessagesSchema, } from "../../../../shared/api-schemas"; import { AwsBedrockKey, keyPool } from "../../../../shared/key-management"; -import { RequestPreprocessor } from "../index"; import { AWSMistralV1ChatCompletionsSchema, AWSMistralV1TextCompletionsSchema, } from "../../../../shared/api-schemas/mistral-ai"; +import { ProxyReqMutator } from "../index"; const AMZ_HOST = process.env.AMZ_HOST || "bedrock-runtime.%REGION%.amazonaws.com"; @@ -21,32 +21,24 @@ const AMZ_HOST = * request object in place to fix the path. * This happens AFTER request transformation. */ -export const signAwsRequest: RequestPreprocessor = async (req) => { +export const signAwsRequest: ProxyReqMutator = async (manager) => { + const req = manager.request; const { model, stream } = req.body; - req.key = keyPool.get(model, "aws"); - - req.isStreaming = stream === true || stream === "true"; - - // same as addAnthropicPreamble for non-AWS requests, but has to happen here - if (req.outboundApi === "anthropic-text") { - let preamble = req.body.prompt.startsWith("\n\nHuman:") ? "" : "\n\nHuman:"; - req.body.prompt = preamble + req.body.prompt; - } + const key = keyPool.get(model, "aws") as AwsBedrockKey; + manager.setKey(key); const credential = getCredentialParts(req); const host = AMZ_HOST.replace("%REGION%", credential.region); // AWS only uses 2023-06-01 and does not actually check this header, but we // set it so that the stream adapter always selects the correct transformer. - req.headers["anthropic-version"] = "2023-06-01"; + manager.setHeader("anthropic-version", "2023-06-01"); // If our key has an inference profile compatible with the requested model, // we want to use the inference profile instead of the model ID when calling // InvokeModel as that will give us higher rate limits. const profile = - (req.key as AwsBedrockKey).inferenceProfileIds.find((p) => - p.includes(model) - ) || model; + key.inferenceProfileIds.find((p) => p.includes(model)) || model; // Uses the AWS SDK to sign a request, then modifies our HPM proxy request // with the headers generated by the SDK. @@ -59,7 +51,7 @@ export const signAwsRequest: RequestPreprocessor = async (req) => { ["Host"]: host, ["content-type"]: "application/json", }, - body: JSON.stringify(applyAwsStrictValidation(req)), + body: JSON.stringify(getStrictlyValidatedBodyForAws(req)), }); if (stream) { @@ -68,19 +60,13 @@ export const signAwsRequest: RequestPreprocessor = async (req) => { newRequest.headers["accept"] = "*/*"; } - const { key, body, inboundApi, outboundApi } = req; + const { body, inboundApi, outboundApi } = req; req.log.info( - { - key: key.hash, - model: body.model, - inferenceProfile: profile, - inboundApi, - outboundApi, - }, + { key: key.hash, model: body.model, profile, inboundApi, outboundApi }, "Assigned AWS credentials to request" ); - req.signedRequest = await sign(newRequest, getCredentialParts(req)); + manager.setSignedRequest(await sign(newRequest, getCredentialParts(req))); }; type Credential = { @@ -116,7 +102,7 @@ async function sign(request: HttpRequest, credential: Credential) { return signer.sign(request); } -function applyAwsStrictValidation(req: Request): unknown { +function getStrictlyValidatedBodyForAws(req: Readonly): unknown { // AWS uses vendor API formats but imposes additional (more strict) validation // rules, namely that extraneous parameters are not allowed. We will validate // using the vendor's zod schema but apply `.strip` to ensure that any diff --git a/src/proxy/middleware/request/preprocessors/sign-vertex-ai-request.ts b/src/proxy/middleware/request/mutators/sign-vertex-ai-request.ts similarity index 81% rename from src/proxy/middleware/request/preprocessors/sign-vertex-ai-request.ts rename to src/proxy/middleware/request/mutators/sign-vertex-ai-request.ts index afc6fb7..1eb90c8 100644 --- a/src/proxy/middleware/request/preprocessors/sign-vertex-ai-request.ts +++ b/src/proxy/middleware/request/mutators/sign-vertex-ai-request.ts @@ -1,12 +1,16 @@ -import express from "express"; +import { Request } from "express"; import crypto from "crypto"; -import { keyPool } from "../../../../shared/key-management"; -import { RequestPreprocessor } from "../index"; import { AnthropicV1MessagesSchema } from "../../../../shared/api-schemas"; +import { keyPool } from "../../../../shared/key-management"; +import { getAxiosInstance } from "../../../../shared/network"; +import { ProxyReqMutator } from "../index"; + +const axios = getAxiosInstance(); const GCP_HOST = process.env.GCP_HOST || "%REGION%-aiplatform.googleapis.com"; -export const signGcpRequest: RequestPreprocessor = async (req) => { +export const signGcpRequest: ProxyReqMutator = async (manager) => { + const req = manager.request; const serviceValid = req.service === "gcp"; if (!serviceValid) { throw new Error("addVertexAIKey called on invalid request"); @@ -16,12 +20,11 @@ export const signGcpRequest: RequestPreprocessor = async (req) => { throw new Error("You must specify a model with your request."); } - const { model, stream } = req.body; - req.key = keyPool.get(model, "gcp"); + const { model } = req.body; + const key = keyPool.get(model, "gcp"); + manager.setKey(key); - req.log.info({ key: req.key.hash, model }, "Assigned GCP key to request"); - - req.isStreaming = String(stream) === "true"; + req.log.info({ key: key.hash, model }, "Assigned GCP key to request"); // TODO: This should happen in transform-outbound-payload.ts // TODO: Support tools @@ -39,15 +42,15 @@ export const signGcpRequest: RequestPreprocessor = async (req) => { .strip() .parse(req.body); strippedParams.anthropic_version = "vertex-2023-10-16"; - + const [accessToken, credential] = await getAccessToken(req); const host = GCP_HOST.replace("%REGION%", credential.region); // GCP doesn't use the anthropic-version header, but we set it to ensure the // stream adapter selects the correct transformer. - req.headers["anthropic-version"] = "2023-06-01"; + manager.setHeader("anthropic-version", "2023-06-01"); - req.signedRequest = { + manager.setSignedRequest({ method: "POST", protocol: "https:", hostname: host, @@ -58,11 +61,11 @@ export const signGcpRequest: RequestPreprocessor = async (req) => { ["authorization"]: `Bearer ${accessToken}`, }, body: JSON.stringify(strippedParams), - }; + }); }; async function getAccessToken( - req: express.Request + req: Readonly ): Promise<[string, Credential]> { // TODO: access token caching to reduce latency const credential = getCredentialParts(req); @@ -134,19 +137,23 @@ async function exchangeJwtForAccessToken( assertion: signedJwt, }; - const r = await fetch(authUrl, { - method: "POST", - headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: Object.entries(params) - .map(([k, v]) => `${k}=${v}`) - .join("&"), - }).then((res) => res.json()); + try { + const response = await axios.post(authUrl, params, { + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + }); - if (r.access_token) { - return [r.access_token, ""]; + if (response.data.access_token) { + return [response.data.access_token, ""]; + } else { + return [null, JSON.stringify(response.data)]; + } + } catch (error) { + if ("response" in error && "data" in error.response) { + return [null, JSON.stringify(error.response.data)]; + } else { + return [null, "An unexpected error occurred"]; + } } - - return [null, JSON.stringify(r)]; } function str2ab(str: string): ArrayBuffer { @@ -179,7 +186,7 @@ type Credential = { privateKey: string; }; -function getCredentialParts(req: express.Request): Credential { +function getCredentialParts(req: Readonly): Credential { const [projectId, clientEmail, region, rawPrivateKey] = req.key!.key.split(":"); if (!projectId || !clientEmail || !region || !rawPrivateKey) { diff --git a/src/proxy/middleware/request/mutators/strip-headers.ts b/src/proxy/middleware/request/mutators/strip-headers.ts new file mode 100644 index 0000000..ed22ea9 --- /dev/null +++ b/src/proxy/middleware/request/mutators/strip-headers.ts @@ -0,0 +1,21 @@ +import { ProxyReqMutator } from "../index"; + +/** + * Removes origin and referer headers before sending the request to the API for + * privacy reasons. + */ +export const stripHeaders: ProxyReqMutator = (manager) => { + manager.setHeader("origin", ""); + manager.setHeader("referer", ""); + manager.removeHeader("tailscale-user-login"); + manager.removeHeader("tailscale-user-name"); + manager.removeHeader("tailscale-headers-info"); + manager.removeHeader("tailscale-user-profile-pic"); + manager.removeHeader("cf-connecting-ip"); + manager.removeHeader("forwarded"); + manager.removeHeader("true-client-ip"); + manager.removeHeader("x-forwarded-for"); + manager.removeHeader("x-forwarded-host"); + manager.removeHeader("x-forwarded-proto"); + manager.removeHeader("x-real-ip"); +}; diff --git a/src/proxy/middleware/request/onproxyreq-factory.ts b/src/proxy/middleware/request/onproxyreq-factory.ts deleted file mode 100644 index b3dd672..0000000 --- a/src/proxy/middleware/request/onproxyreq-factory.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { - applyQuotaLimits, - blockZoomerOrigins, - checkModelFamily, - HPMRequestCallback, - stripHeaders, -} from "./index"; - -type ProxyReqHandlerFactoryOptions = { pipeline: HPMRequestCallback[] }; - -/** - * Returns an http-proxy-middleware request handler that runs the given set of - * onProxyReq callback functions in sequence. - * - * These will run each time a request is proxied, including on automatic retries - * by the queue after encountering a rate limit. - */ -export const createOnProxyReqHandler = ({ - pipeline, -}: ProxyReqHandlerFactoryOptions): HPMRequestCallback => { - const callbackPipeline = [ - checkModelFamily, - applyQuotaLimits, - blockZoomerOrigins, - stripHeaders, - ...pipeline, - ]; - return (proxyReq, req, res, options) => { - // The streaming flag must be set before any other onProxyReq handler runs, - // as it may influence the behavior of subsequent handlers. - // Image generation requests can't be streamed. - // TODO: this flag is set in too many places - req.isStreaming = - req.isStreaming || req.body.stream === true || req.body.stream === "true"; - req.body.stream = req.isStreaming; - - try { - for (const fn of callbackPipeline) { - fn(proxyReq, req, res, options); - } - } catch (error) { - proxyReq.destroy(error); - } - }; -}; diff --git a/src/proxy/middleware/request/onproxyreq/add-anthropic-preamble.ts b/src/proxy/middleware/request/onproxyreq/add-anthropic-preamble.ts deleted file mode 100644 index b5b5df6..0000000 --- a/src/proxy/middleware/request/onproxyreq/add-anthropic-preamble.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AnthropicKey, Key } from "../../../../shared/key-management"; -import { isTextGenerationRequest } from "../../common"; -import { HPMRequestCallback } from "../index"; - -/** - * Some keys require the prompt to start with `\n\nHuman:`. There is no way to - * know this without trying to send the request and seeing if it fails. If a - * key is marked as requiring a preamble, it will be added here. - */ -export const addAnthropicPreamble: HPMRequestCallback = (_proxyReq, req) => { - if ( - !isTextGenerationRequest(req) || - req.key?.service !== "anthropic" || - req.outboundApi !== "anthropic-text" - ) { - return; - } - - let preamble = ""; - let prompt = req.body.prompt; - assertAnthropicKey(req.key); - if (req.key.requiresPreamble && prompt) { - preamble = prompt.startsWith("\n\nHuman:") ? "" : "\n\nHuman:"; - req.log.debug({ key: req.key.hash, preamble }, "Adding preamble to prompt"); - } - req.body.prompt = preamble + prompt; -}; - -function assertAnthropicKey(key: Key): asserts key is AnthropicKey { - if (key.service !== "anthropic") { - throw new Error(`Expected an Anthropic key, got '${key.service}'`); - } -} diff --git a/src/proxy/middleware/request/onproxyreq/finalize-body.ts b/src/proxy/middleware/request/onproxyreq/finalize-body.ts deleted file mode 100644 index 4686d5e..0000000 --- a/src/proxy/middleware/request/onproxyreq/finalize-body.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { fixRequestBody } from "http-proxy-middleware"; -import type { HPMRequestCallback } from "../index"; - -/** Finalize the rewritten request body. Must be the last rewriter. */ -export const finalizeBody: HPMRequestCallback = (proxyReq, req) => { - if (["POST", "PUT", "PATCH"].includes(req.method ?? "") && req.body) { - // For image generation requests, remove stream flag. - if (req.outboundApi === "openai-image") { - delete req.body.stream; - } - // For anthropic text to chat requests, remove undefined prompt. - if (req.outboundApi === "anthropic-chat") { - delete req.body.prompt; - } - - const updatedBody = JSON.stringify(req.body); - proxyReq.setHeader("Content-Length", Buffer.byteLength(updatedBody)); - (req as any).rawBody = Buffer.from(updatedBody); - - // body-parser and http-proxy-middleware don't play nice together - fixRequestBody(proxyReq, req); - } -}; diff --git a/src/proxy/middleware/request/onproxyreq/finalize-signed-request.ts b/src/proxy/middleware/request/onproxyreq/finalize-signed-request.ts deleted file mode 100644 index 1380d1e..0000000 --- a/src/proxy/middleware/request/onproxyreq/finalize-signed-request.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { HPMRequestCallback } from "../index"; - -/** - * For AWS/GCP/Azure/Google requests, the body is signed earlier in the request - * pipeline, before the proxy middleware. This function just assigns the path - * and headers to the proxy request. - */ -export const finalizeSignedRequest: HPMRequestCallback = (proxyReq, req) => { - if (!req.signedRequest) { - throw new Error("Expected req.signedRequest to be set"); - } - - // The path depends on the selected model and the assigned key's region. - proxyReq.path = req.signedRequest.path; - - // Amazon doesn't want extra headers, so we need to remove all of them and - // reassign only the ones specified in the signed request. - proxyReq.getRawHeaderNames().forEach(proxyReq.removeHeader.bind(proxyReq)); - Object.entries(req.signedRequest.headers).forEach(([key, value]) => { - proxyReq.setHeader(key, value); - }); - - // Don't use fixRequestBody here because it adds a content-length header. - // Amazon doesn't want that and it breaks the signature. - proxyReq.write(req.signedRequest.body); -}; diff --git a/src/proxy/middleware/request/onproxyreq/strip-headers.ts b/src/proxy/middleware/request/onproxyreq/strip-headers.ts deleted file mode 100644 index 30083f6..0000000 --- a/src/proxy/middleware/request/onproxyreq/strip-headers.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { HPMRequestCallback } from "../index"; - -/** - * Removes origin and referer headers before sending the request to the API for - * privacy reasons. - **/ -export const stripHeaders: HPMRequestCallback = (proxyReq) => { - proxyReq.setHeader("origin", ""); - proxyReq.setHeader("referer", ""); - proxyReq.removeHeader("tailscale-user-login"); - proxyReq.removeHeader("tailscale-user-name"); - proxyReq.removeHeader("tailscale-headers-info"); - proxyReq.removeHeader("tailscale-user-profile-pic") - proxyReq.removeHeader("cf-connecting-ip"); - proxyReq.removeHeader("forwarded"); - proxyReq.removeHeader("true-client-ip"); - proxyReq.removeHeader("x-forwarded-for"); - proxyReq.removeHeader("x-forwarded-host"); - proxyReq.removeHeader("x-forwarded-proto"); - proxyReq.removeHeader("x-real-ip"); -}; diff --git a/src/proxy/middleware/request/preprocessor-factory.ts b/src/proxy/middleware/request/preprocessor-factory.ts index 3c5ab43..7358a57 100644 --- a/src/proxy/middleware/request/preprocessor-factory.ts +++ b/src/proxy/middleware/request/preprocessor-factory.ts @@ -4,12 +4,15 @@ import { initializeSseStream } from "../../../shared/streaming"; import { classifyErrorAndSend } from "../common"; import { RequestPreprocessor, + blockZoomerOrigins, countPromptTokens, languageFilter, setApiFormat, transformOutboundPayload, validateContextSize, + validateModelFamily, validateVision, + applyQuotaLimits, } from "."; type RequestPreprocessorOptions = { @@ -30,14 +33,15 @@ type RequestPreprocessorOptions = { /** * Returns a middleware function that processes the request body into the given * API format, and then sequentially runs the given additional preprocessors. + * These should be used for validation and transformations that only need to + * happen once per request. * * These run first in the request lifecycle, a single time per request before it * is added to the request queue. They aren't run again if the request is * re-attempted after a rate limit. * - * To run a preprocessor on every re-attempt, pass it to createQueueMiddleware. - * It will run after these preprocessors, but before the request is sent to - * http-proxy-middleware. + * To run functions against requests every time they are re-attempted, write a + * ProxyReqMutator and pass it to createQueuedProxyMiddleware instead. */ export const createPreprocessorMiddleware = ( apiFormat: Parameters[0], @@ -45,6 +49,7 @@ export const createPreprocessorMiddleware = ( ): RequestHandler => { const preprocessors: RequestPreprocessor[] = [ setApiFormat(apiFormat), + blockZoomerOrigins, ...(beforeTransform ?? []), transformOutboundPayload, countPromptTokens, @@ -52,6 +57,8 @@ export const createPreprocessorMiddleware = ( ...(afterTransform ?? []), validateContextSize, validateVision, + validateModelFamily, + applyQuotaLimits, ]; return async (...args) => executePreprocessors(preprocessors, args); }; @@ -83,10 +90,10 @@ async function executePreprocessors( next(); } catch (error) { if (error.constructor.name === "ZodError") { - const msg = error?.issues + const issues = error?.issues ?.map((issue: ZodIssue) => `${issue.path.join(".")}: ${issue.message}`) .join("; "); - req.log.warn({ issues: msg }, "Prompt validation failed."); + req.log.warn({ issues }, "Prompt failed preprocessor validation."); } else { req.log.error(error, "Error while executing request preprocessor"); } @@ -152,10 +159,7 @@ function isTestMessage(body: any) { messages[0].content === "Hi" ); } else if (contents) { - return ( - contents.length === 1 && - contents[0].parts[0]?.text === "Hi" - ); + return contents.length === 1 && contents[0].parts[0]?.text === "Hi"; } else { return ( prompt?.trim() === "Human: Hi\n\nAssistant:" || diff --git a/src/proxy/middleware/request/preprocessors/apply-quota-limits.ts b/src/proxy/middleware/request/preprocessors/apply-quota-limits.ts index a0f163e..8c83302 100644 --- a/src/proxy/middleware/request/preprocessors/apply-quota-limits.ts +++ b/src/proxy/middleware/request/preprocessors/apply-quota-limits.ts @@ -1,6 +1,6 @@ import { hasAvailableQuota } from "../../../../shared/users/user-store"; import { isImageGenerationRequest, isTextGenerationRequest } from "../../common"; -import { HPMRequestCallback } from "../index"; +import { RequestPreprocessor } from "../index"; export class QuotaExceededError extends Error { public quotaInfo: any; @@ -11,7 +11,7 @@ export class QuotaExceededError extends Error { } } -export const applyQuotaLimits: HPMRequestCallback = (_proxyReq, req) => { +export const applyQuotaLimits: RequestPreprocessor = (req) => { const subjectToQuota = isTextGenerationRequest(req) || isImageGenerationRequest(req); if (!subjectToQuota || !req.user) return; diff --git a/src/proxy/middleware/request/onproxyreq/block-zoomer-origins.ts b/src/proxy/middleware/request/preprocessors/block-zoomer-origins.ts similarity index 88% rename from src/proxy/middleware/request/onproxyreq/block-zoomer-origins.ts rename to src/proxy/middleware/request/preprocessors/block-zoomer-origins.ts index 25ff37c..aaca1bb 100644 --- a/src/proxy/middleware/request/onproxyreq/block-zoomer-origins.ts +++ b/src/proxy/middleware/request/preprocessors/block-zoomer-origins.ts @@ -1,4 +1,4 @@ -import { HPMRequestCallback } from "../index"; +import { RequestPreprocessor } from "../index"; const DISALLOWED_ORIGIN_SUBSTRINGS = "janitorai.com,janitor.ai".split(","); @@ -13,7 +13,7 @@ class ZoomerForbiddenError extends Error { * Blocks requests from Janitor AI users with a fake, scary error message so I * stop getting emails asking for tech support. */ -export const blockZoomerOrigins: HPMRequestCallback = (_proxyReq, req) => { +export const blockZoomerOrigins: RequestPreprocessor = (req) => { const origin = req.headers.origin || req.headers.referer; if (origin && DISALLOWED_ORIGIN_SUBSTRINGS.some((s) => origin.includes(s))) { // Venus-derivatives send a test prompt to check if the proxy is working. diff --git a/src/proxy/middleware/request/preprocessors/count-prompt-tokens.ts b/src/proxy/middleware/request/preprocessors/count-prompt-tokens.ts index fadc0e0..3ccc5a1 100644 --- a/src/proxy/middleware/request/preprocessors/count-prompt-tokens.ts +++ b/src/proxy/middleware/request/preprocessors/count-prompt-tokens.ts @@ -17,7 +17,7 @@ export const countPromptTokens: RequestPreprocessor = async (req) => { switch (service) { case "openai": { - req.outputTokens = req.body.max_tokens; + req.outputTokens = req.body.max_completion_tokens || req.body.max_tokens; const prompt: OpenAIChatMessage[] = req.body.messages; result = await countTokens({ req, prompt, service }); break; diff --git a/src/proxy/middleware/request/preprocessors/set-api-format.ts b/src/proxy/middleware/request/preprocessors/set-api-format.ts index 11a0298..33c1d85 100644 --- a/src/proxy/middleware/request/preprocessors/set-api-format.ts +++ b/src/proxy/middleware/request/preprocessors/set-api-format.ts @@ -4,8 +4,22 @@ import { LLMService } from "../../../../shared/models"; import { RequestPreprocessor } from "../index"; export const setApiFormat = (api: { + /** + * The API format the user made the request in and expects the response to be + * in. + */ inApi: Request["inboundApi"]; + /** + * The API format the proxy will make the request in and expects the response + * to be in. If different from `inApi`, the proxy will transform the user's + * request body to this format, and will transform the response body or stream + * events from this format. + */ outApi: APIFormat; + /** + * The service the request will be sent to, which determines authentication + * and possibly the streaming transport. + */ service: LLMService; }): RequestPreprocessor => { return function configureRequestApiFormat(req) { diff --git a/src/proxy/middleware/request/preprocessors/transform-outbound-payload.ts b/src/proxy/middleware/request/preprocessors/transform-outbound-payload.ts index 406151b..51035c6 100644 --- a/src/proxy/middleware/request/preprocessors/transform-outbound-payload.ts +++ b/src/proxy/middleware/request/preprocessors/transform-outbound-payload.ts @@ -35,15 +35,8 @@ export const transformOutboundPayload: RequestPreprocessor = async (req) => { // target API format. We don't need to transform them. const isNativePrompt = req.inboundApi === req.outboundApi; if (isNativePrompt) { - const result = API_REQUEST_VALIDATORS[req.inboundApi].safeParse(req.body); - if (!result.success) { - req.log.warn( - { issues: result.error.issues, body: req.body }, - "Native prompt request validation failed." - ); - throw result.error; - } - req.body = result.data; + const result = API_REQUEST_VALIDATORS[req.inboundApi].parse(req.body); + req.body = result; return; } diff --git a/src/proxy/middleware/request/onproxyreq/check-model-family.ts b/src/proxy/middleware/request/preprocessors/validate-model-family.ts similarity index 79% rename from src/proxy/middleware/request/onproxyreq/check-model-family.ts rename to src/proxy/middleware/request/preprocessors/validate-model-family.ts index 138b1ee..bb1ea9e 100644 --- a/src/proxy/middleware/request/onproxyreq/check-model-family.ts +++ b/src/proxy/middleware/request/preprocessors/validate-model-family.ts @@ -1,12 +1,12 @@ import { config } from "../../../../config"; import { ForbiddenError } from "../../../../shared/errors"; import { getModelFamilyForRequest } from "../../../../shared/models"; -import { HPMRequestCallback } from "../index"; +import { RequestPreprocessor } from "../index"; /** * Ensures the selected model family is enabled by the proxy configuration. */ -export const checkModelFamily: HPMRequestCallback = (_proxyReq, req) => { +export const validateModelFamily: RequestPreprocessor = (req) => { const family = getModelFamilyForRequest(req); if (!config.allowedModelFamilies.includes(family)) { throw new ForbiddenError( diff --git a/src/proxy/middleware/request/proxy-middleware-factory.ts b/src/proxy/middleware/request/proxy-middleware-factory.ts new file mode 100644 index 0000000..9853547 --- /dev/null +++ b/src/proxy/middleware/request/proxy-middleware-factory.ts @@ -0,0 +1,129 @@ +import { Request, Response } from "express"; +import http from "http"; +import ProxyServer from "http-proxy"; +import { Readable } from "stream"; +import { + createProxyMiddleware, + Options, + debugProxyErrorsPlugin, + proxyEventsPlugin, +} from "http-proxy-middleware"; +import { ProxyReqMutator, RequestPreprocessor } from "./index"; +import { createOnProxyResHandler, ProxyResHandlerWithBody } from "../response"; +import { createQueueMiddleware } from "../../queue"; +import { getHttpAgents } from "../../../shared/network"; +import { classifyErrorAndSend } from "../common"; + +/** + * Options for the `createQueuedProxyMiddleware` factory function. + */ +type ProxyMiddlewareFactoryOptions = { + /** + * Functions which receive a ProxyReqManager and can modify the request before + * it is proxied. The modifications will be automatically reverted if the + * request needs to be returned to the queue. + */ + mutations?: ProxyReqMutator[]; + /** + * The target URL to proxy requests to. This can be a string or a function + * which accepts the request and returns a string. + */ + target: string | Options["router"]; + /** + * A function which receives the proxy response and the JSON-decoded request + * body. Only fired for non-streaming responses; streaming responses are + * handled in `handle-streaming-response.ts`. + */ + blockingResponseHandler?: ProxyResHandlerWithBody; +}; + +/** + * Returns a middleware function that accepts incoming requests and places them + * into the request queue. When the request is dequeued, it is proxied to the + * target URL using the given options and middleware. Non-streaming responses + * are handled by the given `blockingResponseHandler`. + */ +export function createQueuedProxyMiddleware({ + target, + mutations, + blockingResponseHandler, +}: ProxyMiddlewareFactoryOptions) { + const hpmTarget = typeof target === "string" ? target : "https://setbyrouter"; + const hpmRouter = typeof target === "function" ? target : undefined; + + const [httpAgent, httpsAgent] = getHttpAgents(); + const agent = hpmTarget.startsWith("http:") ? httpAgent : httpsAgent; + + const proxyMiddleware = createProxyMiddleware({ + target: hpmTarget, + router: hpmRouter, + agent, + changeOrigin: true, + toProxy: true, + selfHandleResponse: typeof blockingResponseHandler === "function", + // Disable HPM logger plugin (requires re-adding the other default plugins). + // Contrary to name, debugProxyErrorsPlugin is not just for debugging and + // fixes several error handling/connection close issues in http-proxy core. + ejectPlugins: true, + // Inferred (via Options) as Plugin, but + // the default plugins only allow http.IncomingMessage for TReq. They are + // compatible with express.Request, so we can use them. `Plugin` type is not + // exported for some reason. + plugins: [ + debugProxyErrorsPlugin, + pinoLoggerPlugin, + proxyEventsPlugin, + ] as any, + on: { + proxyRes: createOnProxyResHandler( + blockingResponseHandler ? [blockingResponseHandler] : [] + ), + error: classifyErrorAndSend, + }, + buffer: ((req: Request) => { + // This is a hack/monkey patch and is not part of the official + // http-proxy-middleware package. See patches/http-proxy+1.18.1.patch. + let payload = req.body; + if (typeof payload === "string") { + payload = Buffer.from(payload); + } + const stream = new Readable(); + stream.push(payload); + stream.push(null); + return stream; + }) as any, + }); + + return createQueueMiddleware({ mutations, proxyMiddleware }); +} + +type ProxiedResponse = http.IncomingMessage & Response & any; +function pinoLoggerPlugin(proxyServer: ProxyServer) { + proxyServer.on("error", (err, req, res, target) => { + const originalUrl = req.originalUrl; + const targetUrl = target?.toString(); + req.log.error( + { originalUrl, targetUrl, err }, + "Error occurred while proxying request to target" + ); + }); + proxyServer.on("proxyReq", (proxyReq, req, res) => { + const originalUrl = req.originalUrl; + const targetHost = `${proxyReq.protocol}//${proxyReq.host}`; + const targetPath = res.req.url; + req.log.info( + { originalUrl, targetHost, targetPath }, + "Sending request to upstream API..." + ); + }); + proxyServer.on("proxyRes", (proxyRes: ProxiedResponse, req, _res) => { + const originalUrl = req.originalUrl; + const targetHost = `${proxyRes.req.protocol}//${proxyRes.req.hostname}`; + const targetPath = proxyRes.req.path; + const statusCode = proxyRes.statusCode; + req.log.info( + { originalUrl, targetHost, targetPath, statusCode }, + "Got response from upstream API." + ); + }); +} diff --git a/src/proxy/middleware/request/proxy-req-manager.ts b/src/proxy/middleware/request/proxy-req-manager.ts new file mode 100644 index 0000000..eceedee --- /dev/null +++ b/src/proxy/middleware/request/proxy-req-manager.ts @@ -0,0 +1,112 @@ +import { Request } from "express"; +import { Key } from "../../../shared/key-management"; +import { assertNever } from "../../../shared/utils"; + +/** + * Represents a change to the request that will be reverted if the request + * fails. + */ +interface ProxyReqMutation { + target: "header" | "path" | "body" | "api-key" | "signed-request"; + key?: string; + originalValue: any | undefined; +} + +/** + * Manages a request's headers, body, and path, allowing them to be modified + * before the request is proxied and automatically reverted if the request + * needs to be retried. + */ +export class ProxyReqManager { + private req: Request; + private mutations: ProxyReqMutation[] = []; + + /** + * A read-only proxy of the request object. Avoid changing any properties + * here as they will persist across retries. + */ + public readonly request: Readonly; + + constructor(req: Request) { + this.req = req; + + this.request = new Proxy(req, { + get: (target, prop) => { + if (typeof prop === "string") return target[prop as keyof Request]; + return undefined; + }, + }); + } + + setHeader(name: string, newValue: string): void { + const originalValue = this.req.get(name); + this.mutations.push({ target: "header", key: name, originalValue }); + this.req.headers[name.toLowerCase()] = newValue; + } + + removeHeader(name: string): void { + const originalValue = this.req.get(name); + this.mutations.push({ target: "header", key: name, originalValue }); + delete this.req.headers[name.toLowerCase()]; + } + + setBody(newBody: any): void { + const originalValue = this.req.body; + this.mutations.push({ target: "body", key: "body", originalValue }); + this.req.body = newBody; + } + + setKey(newKey: Key): void { + const originalValue = this.req.key; + this.mutations.push({ target: "api-key", key: "key", originalValue }); + this.req.key = newKey; + } + + setPath(newPath: string): void { + const originalValue = this.req.path; + this.mutations.push({ target: "path", key: "path", originalValue }); + this.req.url = newPath; + } + + setSignedRequest(newSignedRequest: typeof this.req.signedRequest): void { + const originalValue = this.req.signedRequest; + this.mutations.push({ target: "signed-request", key: "signedRequest", originalValue }); + this.req.signedRequest = newSignedRequest; + } + + hasChanged(): boolean { + return this.mutations.length > 0; + } + + revert(): void { + for (const mutation of this.mutations.reverse()) { + switch (mutation.target) { + case "header": + if (mutation.originalValue === undefined) { + delete this.req.headers[mutation.key!.toLowerCase()]; + continue; + } else { + this.req.headers[mutation.key!.toLowerCase()] = + mutation.originalValue; + } + break; + case "path": + this.req.url = mutation.originalValue; + break; + case "body": + this.req.body = mutation.originalValue; + break; + case "api-key": + // We don't reset the key here because it's not a property of the + // inbound request, so we'd only ever be reverting it to null. + break; + case "signed-request": + this.req.signedRequest = mutation.originalValue; + break; + default: + assertNever(mutation.target); + } + } + this.mutations = []; + } +} diff --git a/src/proxy/middleware/response/error-generator.ts b/src/proxy/middleware/response/error-generator.ts index f2bfdad..e8ba40a 100644 --- a/src/proxy/middleware/response/error-generator.ts +++ b/src/proxy/middleware/response/error-generator.ts @@ -2,36 +2,33 @@ import express from "express"; import { APIFormat } from "../../../shared/key-management"; import { assertNever } from "../../../shared/utils"; import { initializeSseStream } from "../../../shared/streaming"; +import http from "http"; -function getMessageContent({ - title, - message, - obj, -}: { +/** + * Returns a Markdown-formatted message that renders semi-nicely in most chat + * frontends. For example: + * + * **Proxy error (HTTP 404 Not Found)** + * The proxy encountered an error while trying to send your prompt to the upstream service. Further technical details are provided below. + * *** + * *The requested Claude model might not exist, or the key might not be provisioned for it.* + * ``` + * { + * "type": "error", + * "error": { + * "type": "not_found_error", + * "message": "model: some-invalid-model-id", + * }, + * "proxy_note": "The requested Claude model might not exist, or the key might not be provisioned for it." + * } + * ``` + */ +function getMessageContent(params: { title: string; message: string; obj?: Record; }) { - /* - Constructs a Markdown-formatted message that renders semi-nicely in most chat - frontends. For example: - - **Proxy error (HTTP 404 Not Found)** - The proxy encountered an error while trying to send your prompt to the upstream service. Further technical details are provided below. - *** - *The requested Claude model might not exist, or the key might not be provisioned for it.* - ``` - { - "type": "error", - "error": { - "type": "not_found_error", - "message": "model: some-invalid-model-id", - }, - "proxy_note": "The requested Claude model might not exist, or the key might not be provisioned for it." - } - ``` - */ - + const { title, message, obj } = params; const note = obj?.proxy_note || obj?.error?.message || ""; const header = `### **${title}**`; const friendlyMessage = note ? `${message}\n\n----\n\n*${note}*` : message; @@ -71,7 +68,11 @@ type ErrorGeneratorOptions = { statusCode?: number; }; -export function tryInferFormat(body: any): APIFormat | "unknown" { +/** + * Very crude inference of the request format based on the request body. Don't + * rely on this to be very accurate. + */ +function tryInferFormat(body: any): APIFormat | "unknown" { if (typeof body !== "object" || !body.model) { return "unknown"; } @@ -95,7 +96,11 @@ export function tryInferFormat(body: any): APIFormat | "unknown" { return "unknown"; } -// avoid leaking upstream hostname on dns resolution error +/** + * Redacts the hostname from the error message if it contains a DNS resolution + * error. This is to avoid leaking upstream hostnames on DNS resolution errors, + * as those may contain sensitive information about the proxy's configuration. + */ function redactHostname(options: ErrorGeneratorOptions): ErrorGeneratorOptions { if (!options.message.includes("getaddrinfo")) return options; @@ -112,46 +117,61 @@ function redactHostname(options: ErrorGeneratorOptions): ErrorGeneratorOptions { return redacted; } -export function sendErrorToClient({ - options, - req, - res, -}: { +/** + * Generates an appropriately-formatted error response and sends it to the + * client over their requested transport (blocking or SSE stream). + */ +export function sendErrorToClient(params: { options: ErrorGeneratorOptions; req: express.Request; res: express.Response; }) { - const redactedOpts = redactHostname(options); - const { format: inputFormat } = redactedOpts; + const { req, res } = params; + const options = redactHostname(params.options); + const { statusCode, message, title, obj: details } = options; + // Since we want to send the error in a format the client understands, we + // need to know the request format. `setApiFormat` might not have been called + // yet, so we'll try to infer it from the request body. const format = - inputFormat === "unknown" ? tryInferFormat(req.body) : inputFormat; + options.format === "unknown" ? tryInferFormat(req.body) : options.format; if (format === "unknown") { - return res.status(redactedOpts.statusCode || 400).json({ - error: redactedOpts.message, - details: redactedOpts.obj, + // Early middleware error (auth, rate limit) so we can only send something + // generic. + const code = statusCode || 400; + const hasDetails = details && Object.keys(details).length > 0; + return res.status(code).json({ + error: { + message, + type: http.STATUS_CODES[code]!.replace(/\s+/g, "_").toLowerCase(), + }, + ...(hasDetails ? { details } : {}), }); } - const completion = buildSpoofedCompletion({ ...redactedOpts, format }); - const event = buildSpoofedSSE({ ...redactedOpts, format }); - const isStreaming = - req.isStreaming || req.body.stream === true || req.body.stream === "true"; - + // Cannot modify headers if client opted into streaming and made it into the + // proxy request queue, because that immediately starts an SSE stream. if (!res.headersSent) { - res.setHeader("x-oai-proxy-error", redactedOpts.title); - res.setHeader("x-oai-proxy-error-status", redactedOpts.statusCode || 500); + res.setHeader("x-oai-proxy-error", title); + res.setHeader("x-oai-proxy-error-status", statusCode || 500); } + // By this point, we know the request format. To get the error to display in + // chat clients' UIs, we'll send it as a 200 response as a spoofed completion + // from the language model. Depending on whether the client is streaming, we + // will either send an SSE event or a JSON response. + const isStreaming = req.isStreaming || String(req.body.stream) === "true"; if (isStreaming) { + // User can have opted into streaming but not made it into the queue yet, + // in which case the stream must be started first. if (!res.headersSent) { initializeSseStream(res); } - res.write(event); + res.write(buildSpoofedSSE({ ...options, format })); res.write(`data: [DONE]\n\n`); res.end(); } else { - res.status(200).json(completion); + res.status(200).json(buildSpoofedCompletion({ ...options, format })); } } @@ -193,7 +213,7 @@ export function buildSpoofedCompletion({ return { outputs: [{ text: content, stop_reason: title }], model, - } + }; case "openai-text": return { id: "error-" + id, diff --git a/src/proxy/middleware/response/index.ts b/src/proxy/middleware/response/index.ts index c6a6144..ed6a902 100644 --- a/src/proxy/middleware/response/index.ts +++ b/src/proxy/middleware/response/index.ts @@ -47,7 +47,7 @@ export type ProxyResHandlerWithBody = ( */ body: string | Record ) => Promise; -export type ProxyResMiddleware = ProxyResHandlerWithBody[]; +export type ProxyResMiddleware = ProxyResHandlerWithBody[] | undefined; /** * Returns a on.proxyRes handler that executes the given middleware stack after @@ -71,11 +71,22 @@ export const createOnProxyResHandler = (apiMiddleware: ProxyResMiddleware) => { req: Request, res: Response ) => { - const initialHandler: RawResponseBodyHandler = req.isStreaming + // Proxied request has by now been sent to the upstream API, so we revert + // tracked mutations that were only needed to send the request. + // This generally means path adjustment, headers, and body serialization. + if (req.changeManager) { + req.changeManager.revert(); + } + + const initialHandler = req.isStreaming ? handleStreamedResponse : handleBlockingResponse; let lastMiddleware = initialHandler.name; + if (Buffer.isBuffer(req.body)) { + req.body = JSON.parse(req.body.toString()); + } + try { const body = await initialHandler(proxyRes, req, res); const middlewareStack: ProxyResMiddleware = []; @@ -100,7 +111,7 @@ export const createOnProxyResHandler = (apiMiddleware: ProxyResMiddleware) => { saveImage, logPrompt, logEvent, - ...apiMiddleware + ...(apiMiddleware ?? []) ); } @@ -723,22 +734,26 @@ const trackKeyRateLimit: ProxyResHandlerWithBody = async (proxyRes, req) => { keyPool.updateRateLimits(req.key!, proxyRes.headers); }; + +const omittedHeaders = new Set([ + // Omit content-encoding because we will always decode the response body + "content-encoding", + // Omit transfer-encoding because we are using response.json which will + // set a content-length header, which is not valid for chunked responses. + "transfer-encoding", + // Don't set cookies from upstream APIs because proxied requests are stateless + "set-cookie", + "openai-organization", + "x-request-id", + "cf-ray", +]); const copyHttpHeaders: ProxyResHandlerWithBody = async ( proxyRes, _req, res ) => { Object.keys(proxyRes.headers).forEach((key) => { - // Omit content-encoding because we will always decode the response body - if (key === "content-encoding") { - return; - } - // We're usually using res.json() to send the response, which causes express - // to set content-length. That's not valid for chunked responses and some - // clients will reject it so we need to omit it. - if (key === "transfer-encoding") { - return; - } + if (omittedHeaders.has(key)) return; res.setHeader(key, proxyRes.headers[key] as string); }); }; @@ -782,6 +797,6 @@ function getAwsErrorType(header: string | string[] | undefined) { function assertJsonResponse(body: any): asserts body is Record { if (typeof body !== "object") { - throw new Error("Expected response to be an object"); + throw new Error(`Expected response to be an object, got ${typeof body}`); } } diff --git a/src/proxy/mistral-ai.ts b/src/proxy/mistral-ai.ts index c971e8d..5a69cd8 100644 --- a/src/proxy/mistral-ai.ts +++ b/src/proxy/mistral-ai.ts @@ -1,27 +1,20 @@ -import express, { Request, RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; -import { config } from "../config"; +import { Request, RequestHandler, Router } from "express"; +import { BadRequestError } from "../shared/errors"; import { keyPool } from "../shared/key-management"; import { getMistralAIModelFamily, MistralAIModelFamily, ModelFamily, } from "../shared/models"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; +import { config } from "../config"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { addKey, - createOnProxyReqHandler, createPreprocessorMiddleware, finalizeBody, } from "./middleware/request"; -import { - createOnProxyResHandler, - ProxyResHandlerWithBody, -} from "./middleware/response"; -import { BadRequestError } from "../shared/errors"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; // Mistral can't settle on a single naming scheme and deprecates models within // months of releasing them so this list is hard to keep up to date. 2024-07-28 @@ -127,20 +120,10 @@ export function transformMistralTextToMistralChat(textBody: any) { }; } -const mistralAIProxy = createQueueMiddleware({ - proxyMiddleware: createProxyMiddleware({ - target: "https://api.mistral.ai", - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ - pipeline: [addKey, finalizeBody], - }), - proxyRes: createOnProxyResHandler([mistralAIResponseHandler]), - error: handleProxyError, - }, - }), +const mistralAIProxy = createQueuedProxyMiddleware({ + target: "https://api.mistral.ai", + mutations: [addKey, finalizeBody], + blockingResponseHandler: mistralAIResponseHandler, }); const mistralAIRouter = Router(); diff --git a/src/proxy/openai-image.ts b/src/proxy/openai-image.ts index aed15cc..0ec7391 100644 --- a/src/proxy/openai-image.ts +++ b/src/proxy/openai-image.ts @@ -1,22 +1,15 @@ -import { RequestHandler, Router, Request } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; -import { config } from "../config"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; +import { Request, RequestHandler, Router } from "express"; +import { OpenAIImageGenerationResult } from "../shared/file-storage/mirror-generated-image"; +import { generateModelList } from "./openai"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { addKey, createPreprocessorMiddleware, finalizeBody, - createOnProxyReqHandler, } from "./middleware/request"; -import { - createOnProxyResHandler, - ProxyResHandlerWithBody, -} from "./middleware/response"; -import { generateModelList } from "./openai"; -import { OpenAIImageGenerationResult } from "../shared/file-storage/mirror-generated-image"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { ProxyReqManager } from "./middleware/request/proxy-req-manager"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; const KNOWN_MODELS = ["dall-e-2", "dall-e-3"]; @@ -96,21 +89,19 @@ function transformResponseForChat( }; } -const openaiImagesProxy = createQueueMiddleware({ - proxyMiddleware: createProxyMiddleware({ - target: "https://api.openai.com", - changeOrigin: true, - selfHandleResponse: true, - logger, - pathRewrite: { - "^/v1/chat/completions": "/v1/images/generations", - }, - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [addKey, finalizeBody] }), - proxyRes: createOnProxyResHandler([openaiImagesResponseHandler]), - error: handleProxyError, - }, - }), +function replacePath(manager: ProxyReqManager) { + const req = manager.request; + const pathname = req.url.split("?")[0]; + req.log.debug({ pathname }, "OpenAI image path filter"); + if (req.path.startsWith("/v1/chat/completions")) { + manager.setPath("/v1/images/generations"); + } +} + +const openaiImagesProxy = createQueuedProxyMiddleware({ + target: "https://api.openai.com", + mutations: [replacePath, addKey, finalizeBody], + blockingResponseHandler: openaiImagesResponseHandler, }); const openaiImagesRouter = Router(); diff --git a/src/proxy/openai.ts b/src/proxy/openai.ts index b843651..c9afbc3 100644 --- a/src/proxy/openai.ts +++ b/src/proxy/openai.ts @@ -1,26 +1,18 @@ import { Request, RequestHandler, Router } from "express"; -import { createProxyMiddleware } from "http-proxy-middleware"; import { config } from "../config"; import { AzureOpenAIKey, keyPool, OpenAIKey } from "../shared/key-management"; import { getOpenAIModelFamily } from "../shared/models"; -import { logger } from "../logger"; -import { createQueueMiddleware } from "./queue"; import { ipLimiter } from "./rate-limit"; -import { handleProxyError } from "./middleware/common"; import { addKey, addKeyForEmbeddingsRequest, createEmbeddingsPreprocessorMiddleware, - createOnProxyReqHandler, createPreprocessorMiddleware, finalizeBody, - forceModel, RequestPreprocessor, } from "./middleware/request"; -import { - createOnProxyResHandler, - ProxyResHandlerWithBody, -} from "./middleware/response"; +import { ProxyResHandlerWithBody } from "./middleware/response"; +import { createQueuedProxyMiddleware } from "./middleware/request/proxy-middleware-factory"; // https://platform.openai.com/docs/models/overview let modelsCache: any = null; @@ -126,7 +118,6 @@ const openaiResponseHandler: ProxyResHandlerWithBody = async ( res.status(200).json({ ...newBody, proxy: body.proxy }); }; -/** Only used for non-streaming responses. */ function transformTurboInstructResponse( turboInstructBody: Record ): Record { @@ -144,31 +135,15 @@ function transformTurboInstructResponse( return transformed; } -const openaiProxy = createQueueMiddleware({ - proxyMiddleware: createProxyMiddleware({ - target: "https://api.openai.com", - changeOrigin: true, - selfHandleResponse: true, - logger, - on: { - proxyReq: createOnProxyReqHandler({ pipeline: [addKey, finalizeBody] }), - proxyRes: createOnProxyResHandler([openaiResponseHandler]), - error: handleProxyError, - }, - }), +const openaiProxy = createQueuedProxyMiddleware({ + mutations: [addKey, finalizeBody], + target: "https://api.openai.com", + blockingResponseHandler: openaiResponseHandler, }); -const openaiEmbeddingsProxy = createProxyMiddleware({ +const openaiEmbeddingsProxy = createQueuedProxyMiddleware({ + mutations: [addKeyForEmbeddingsRequest, finalizeBody], target: "https://api.openai.com", - changeOrigin: true, - selfHandleResponse: false, - logger, - on: { - proxyReq: createOnProxyReqHandler({ - pipeline: [addKeyForEmbeddingsRequest, finalizeBody], - }), - error: handleProxyError, - }, }); const openaiRouter = Router(); @@ -215,6 +190,10 @@ openaiRouter.post( openaiEmbeddingsProxy ); +function forceModel(model: string): RequestPreprocessor { + return (req: Request) => void (req.body.model = model); +} + function fixupMaxTokens(req: Request) { if (!req.body.max_completion_tokens) { req.body.max_completion_tokens = req.body.max_tokens; diff --git a/src/proxy/queue.ts b/src/proxy/queue.ts index 127094d..4b4a36c 100644 --- a/src/proxy/queue.ts +++ b/src/proxy/queue.ts @@ -24,9 +24,10 @@ import { import { initializeSseStream } from "../shared/streaming"; import { logger } from "../logger"; import { getUniqueIps } from "./rate-limit"; -import { RequestPreprocessor } from "./middleware/request"; -import { handleProxyError } from "./middleware/common"; +import { ProxyReqMutator, RequestPreprocessor } from "./middleware/request"; import { sendErrorToClient } from "./middleware/response/error-generator"; +import { ProxyReqManager } from "./middleware/request/proxy-req-manager"; +import { classifyErrorAndSend } from "./middleware/common"; const queue: Request[] = []; const log = logger.child({ module: "request-queue" }); @@ -67,6 +68,14 @@ const sharesIdentifierWith = (incoming: Request) => (queued: Request) => getIdentifier(queued) === getIdentifier(incoming); async function enqueue(req: Request) { + if (req.socket.destroyed || req.res?.writableEnded) { + // In rare cases, a request can be disconnected after it is dequeued for a + // retry, but before it is re-enqueued. In this case we may miss the abort + // and the request will loop in the queue forever. + req.log.warn("Attempt to enqueue aborted request."); + throw new Error("Attempt to enqueue aborted request."); + } + const enqueuedRequestCount = queue.filter(sharesIdentifierWith(req)).length; if (enqueuedRequestCount >= USER_CONCURRENCY_LIMIT) { @@ -139,7 +148,14 @@ export function dequeue(partition: ModelFamily): Request | undefined { } const req = modelQueue.reduce((prev, curr) => - prev.startTime + config.tokensPunishmentFactor*((prev.promptTokens ?? 0) + (prev.outputTokens ?? 0)) < curr.startTime + config.tokensPunishmentFactor*((curr.promptTokens ?? 0) + (curr.outputTokens ?? 0)) ? prev : curr + prev.startTime + + config.tokensPunishmentFactor * + ((prev.promptTokens ?? 0) + (prev.outputTokens ?? 0)) < + curr.startTime + + config.tokensPunishmentFactor * + ((curr.promptTokens ?? 0) + (curr.outputTokens ?? 0)) + ? prev + : curr ); queue.splice(queue.indexOf(req), 1); @@ -306,26 +322,35 @@ export function getQueueLength(partition: ModelFamily | "all" = "all") { } export function createQueueMiddleware({ - beforeProxy, + mutations = [], proxyMiddleware, }: { - beforeProxy?: RequestPreprocessor; + mutations?: ProxyReqMutator[]; proxyMiddleware: Handler; }): Handler { return async (req, res, next) => { req.proceed = async () => { - if (beforeProxy) { - try { - // Hack to let us run asynchronous middleware before the - // http-proxy-middleware handler. This is used to sign AWS requests - // before they are proxied, as the signing is asynchronous. - // Unlike RequestPreprocessors, this runs every time the request is - // dequeued, not just the first time. - await beforeProxy(req); - } catch (err) { - return handleProxyError(err, req, res); + // canonicalize the stream field which is set in a few places not always + // consistently + req.isStreaming = req.isStreaming || String(req.body.stream) === "true"; + req.body.stream = req.isStreaming; + + try { + // Just before executing the proxyMiddleware, we will create a + // ProxyReqManager to track modifications to the request. This allows + // us to revert those changes if the proxied request fails with a + // retryable error. That happens in proxyMiddleware's onProxyRes + // handler. + const changeManager = new ProxyReqManager(req); + req.changeManager = changeManager; + for (const mutator of mutations) { + await mutator(changeManager); } + } catch (err) { + // Failure during request preparation is a fatal error. + return classifyErrorAndSend(err, req, res); } + proxyMiddleware(req, res, next); }; diff --git a/src/server.ts b/src/server.ts index 7313575..5b180e8 100644 --- a/src/server.ts +++ b/src/server.ts @@ -23,6 +23,7 @@ import { init as initTokenizers } from "./shared/tokenization"; import { checkOrigin } from "./proxy/check-origin"; import { sendErrorToClient } from "./proxy/middleware/response/error-generator"; import { initializeDatabase, getDatabase } from "./shared/database"; +import { initializeFirebase } from "./shared/firebase"; const PORT = config.port; const BIND_ADDRESS = config.bindAddress; @@ -137,6 +138,12 @@ async function start() { logger.info("Checking configs and external dependencies..."); await assertConfigIsValid(); + if (config.gatekeeperStore.startsWith("firebase")) { + logger.info("Testing Firebase connection..."); + await initializeFirebase(); + logger.info("Firebase connection successful."); + } + keyPool.init(); await initTokenizers(); @@ -166,7 +173,7 @@ async function start() { app.listen(PORT, BIND_ADDRESS, () => { logger.info( { port: PORT, interface: BIND_ADDRESS }, - "Now listening for connections." + "Server ready to accept connections." ); registerUncaughtExceptionHandler(); }); diff --git a/src/shared/custom.d.ts b/src/shared/custom.d.ts index 8f91644..461bc12 100644 --- a/src/shared/custom.d.ts +++ b/src/shared/custom.d.ts @@ -5,6 +5,7 @@ import { Express } from "express-serve-static-core"; import { APIFormat, Key } from "./key-management"; import { User } from "./users/schema"; import { LLMService, ModelFamily } from "./models"; +import { ProxyReqManager } from "../proxy/middleware/request/proxy-req-manager"; declare global { namespace Express { @@ -24,6 +25,7 @@ declare global { queueOutTime?: number; onAborted?: () => void; proceed: () => void; + changeManager?: ProxyReqManager; heartbeatInterval?: NodeJS.Timeout; monitorInterval?: NodeJS.Timeout; promptTokens?: number; diff --git a/src/shared/file-storage/mirror-generated-image.ts b/src/shared/file-storage/mirror-generated-image.ts index c8a1884..978a858 100644 --- a/src/shared/file-storage/mirror-generated-image.ts +++ b/src/shared/file-storage/mirror-generated-image.ts @@ -1,12 +1,14 @@ -import axios from "axios"; import express from "express"; import { promises as fs } from "fs"; import path from "path"; import { v4 } from "uuid"; import { USER_ASSETS_DIR } from "../../config"; +import { getAxiosInstance } from "../network"; import { addToImageHistory } from "./image-history"; import { libSharp } from "./index"; +const axios = getAxiosInstance(); + export type OpenAIImageGenerationResult = { created: number; data: { diff --git a/src/shared/firebase.ts b/src/shared/firebase.ts new file mode 100644 index 0000000..771687d --- /dev/null +++ b/src/shared/firebase.ts @@ -0,0 +1,28 @@ +import type firebase from "firebase-admin"; +import { config } from "../config"; +import { getHttpAgents } from "./network"; + +let firebaseApp: firebase.app.App | undefined; + +export async function initializeFirebase() { + const firebase = await import("firebase-admin"); + const firebaseKey = Buffer.from(config.firebaseKey!, "base64").toString(); + const app = firebase.initializeApp({ + // RTDB doesn't actually seem to use this but respects `WS_PROXY` if set, + // so we do that in the network module. + httpAgent: getHttpAgents()[0], + credential: firebase.credential.cert(JSON.parse(firebaseKey)), + databaseURL: config.firebaseRtdbUrl, + }); + + await app.database().ref("connection-test").set(Date.now()); + + firebaseApp = app; +} + +export function getFirebaseApp(): firebase.app.App { + if (!firebaseApp) { + throw new Error("Firebase app not initialized."); + } + return firebaseApp; +} diff --git a/src/shared/key-management/anthropic/checker.ts b/src/shared/key-management/anthropic/checker.ts index 1727178..122b0ef 100644 --- a/src/shared/key-management/anthropic/checker.ts +++ b/src/shared/key-management/anthropic/checker.ts @@ -1,7 +1,10 @@ -import axios, { AxiosError, AxiosResponse } from "axios"; +import { AxiosError, AxiosResponse } from "axios"; +import { getAxiosInstance } from "../../network"; import { KeyCheckerBase } from "../key-checker-base"; import type { AnthropicKey, AnthropicKeyProvider } from "./provider"; +const axios = getAxiosInstance(); + const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds const KEY_CHECK_PERIOD = 1000 * 60 * 60 * 6; // 6 hours const POST_MESSAGES_URL = "https://api.anthropic.com/v1/messages"; diff --git a/src/shared/key-management/aws/checker.ts b/src/shared/key-management/aws/checker.ts index eee1168..395709a 100644 --- a/src/shared/key-management/aws/checker.ts +++ b/src/shared/key-management/aws/checker.ts @@ -1,13 +1,16 @@ import { Sha256 } from "@aws-crypto/sha256-js"; import { SignatureV4 } from "@smithy/signature-v4"; import { HttpRequest } from "@smithy/protocol-http"; -import axios, { AxiosError, AxiosHeaders, AxiosRequestConfig } from "axios"; +import { AxiosError, AxiosHeaders, AxiosRequestConfig } from "axios"; import { URL } from "url"; import { config } from "../../../config"; import { getAwsBedrockModelFamily } from "../../models"; +import { getAxiosInstance } from "../../network"; import { KeyCheckerBase } from "../key-checker-base"; import type { AwsBedrockKey, AwsBedrockKeyProvider } from "./provider"; +const axios = getAxiosInstance(); + type ParentModelId = string; type AliasModelId = string; type ModuleAliasTuple = [ParentModelId, ...AliasModelId[]]; diff --git a/src/shared/key-management/azure/checker.ts b/src/shared/key-management/azure/checker.ts index 68b5980..423ee40 100644 --- a/src/shared/key-management/azure/checker.ts +++ b/src/shared/key-management/azure/checker.ts @@ -1,7 +1,10 @@ -import axios, { AxiosError } from "axios"; +import { AxiosError } from "axios"; +import { getAzureOpenAIModelFamily } from "../../models"; +import { getAxiosInstance } from "../../network"; import { KeyCheckerBase } from "../key-checker-base"; import type { AzureOpenAIKey, AzureOpenAIKeyProvider } from "./provider"; -import { getAzureOpenAIModelFamily } from "../../models"; + +const axios = getAxiosInstance(); const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds const KEY_CHECK_PERIOD = 60 * 60 * 1000; // 1 hour diff --git a/src/shared/key-management/gcp/checker.ts b/src/shared/key-management/gcp/checker.ts index de1ecdd..a32cfa9 100644 --- a/src/shared/key-management/gcp/checker.ts +++ b/src/shared/key-management/gcp/checker.ts @@ -1,8 +1,11 @@ -import axios, { AxiosError } from "axios"; +import { AxiosError } from "axios"; import crypto from "crypto"; +import { GcpModelFamily } from "../../models"; +import { getAxiosInstance } from "../../network"; import { KeyCheckerBase } from "../key-checker-base"; import type { GcpKey, GcpKeyProvider } from "./provider"; -import { GcpModelFamily } from "../../models"; + +const axios = getAxiosInstance(); const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds const KEY_CHECK_PERIOD = 90 * 60 * 1000; // 90 minutes diff --git a/src/shared/key-management/google-ai/checker.ts b/src/shared/key-management/google-ai/checker.ts index 0d29bcf..a3e0754 100644 --- a/src/shared/key-management/google-ai/checker.ts +++ b/src/shared/key-management/google-ai/checker.ts @@ -1,8 +1,10 @@ -import axios, { AxiosError } from "axios"; -import type { GoogleAIModelFamily } from "../../models"; +import { AxiosError } from "axios"; +import { GoogleAIModelFamily, getGoogleAIModelFamily } from "../../models"; +import { getAxiosInstance } from "../../network"; import { KeyCheckerBase } from "../key-checker-base"; import type { GoogleAIKey, GoogleAIKeyProvider } from "./provider"; -import { getGoogleAIModelFamily } from "../../models"; + +const axios = getAxiosInstance(); const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds const KEY_CHECK_PERIOD = 3 * 60 * 60 * 1000; // 3 hours diff --git a/src/shared/key-management/mistral-ai/checker.ts b/src/shared/key-management/mistral-ai/checker.ts index 12c981d..3a0b31d 100644 --- a/src/shared/key-management/mistral-ai/checker.ts +++ b/src/shared/key-management/mistral-ai/checker.ts @@ -1,8 +1,10 @@ -import axios, { AxiosError } from "axios"; -import type { MistralAIModelFamily } from "../../models"; +import { AxiosError } from "axios"; +import { MistralAIModelFamily, getMistralAIModelFamily } from "../../models"; +import { getAxiosInstance } from "../../network"; import { KeyCheckerBase } from "../key-checker-base"; import type { MistralAIKey, MistralAIKeyProvider } from "./provider"; -import { getMistralAIModelFamily } from "../../models"; + +const axios = getAxiosInstance(); const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds const KEY_CHECK_PERIOD = 60 * 60 * 1000; // 1 hour diff --git a/src/shared/key-management/openai/checker.ts b/src/shared/key-management/openai/checker.ts index 8dac7f9..eb0fc32 100644 --- a/src/shared/key-management/openai/checker.ts +++ b/src/shared/key-management/openai/checker.ts @@ -1,8 +1,10 @@ -import axios, { AxiosError } from "axios"; -import type { OpenAIModelFamily } from "../../models"; +import { AxiosError } from "axios"; import { KeyCheckerBase } from "../key-checker-base"; import type { OpenAIKey, OpenAIKeyProvider } from "./provider"; -import { getOpenAIModelFamily } from "../../models"; +import { OpenAIModelFamily, getOpenAIModelFamily } from "../../models"; +import { getAxiosInstance } from "../../network"; + +const axios = getAxiosInstance(); const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds const KEY_CHECK_PERIOD = 60 * 60 * 1000; // 1 hour diff --git a/src/shared/network.ts b/src/shared/network.ts new file mode 100644 index 0000000..9a4f142 --- /dev/null +++ b/src/shared/network.ts @@ -0,0 +1,69 @@ +import axios, { AxiosInstance } from "axios"; +import http from "http"; +import https from "https"; +import os from "os"; +import { ProxyAgent } from "proxy-agent"; +import { config } from "../config"; +import { logger } from "../logger"; + +const log = logger.child({ module: "network" }); + +export type HttpAgent = http.Agent | https.Agent; + +/** HTTP agent used by http-proxy-middleware when forwarding requests. */ +let httpAgent: HttpAgent; +/** HTTPS agent used by http-proxy-middleware when forwarding requests. */ +let httpsAgent: HttpAgent; +/** Axios instance used for any non-proxied requests. */ +let axiosInstance: AxiosInstance; + +function getInterfaceAddress(iface: string) { + const ifaces = os.networkInterfaces(); + log.debug({ ifaces, iface }, "Found network interfaces."); + if (!ifaces[iface]) { + throw new Error(`Interface ${iface} not found.`); + } + + const addresses = ifaces[iface]!.filter( + ({ family, internal }) => family === "IPv4" && !internal + ); + if (addresses.length === 0) { + throw new Error(`Interface ${iface} has no external IPv4 addresses.`); + } + + log.debug({ selected: addresses[0] }, "Selected network interface."); + return addresses[0].address; +} + +export function getHttpAgents() { + if (httpAgent) return [httpAgent, httpsAgent]; + const { interface: iface, proxyUrl } = config.httpAgent || {}; + + if (iface) { + const address = getInterfaceAddress(iface); + httpAgent = new http.Agent({ localAddress: address, keepAlive: true }); + httpsAgent = new https.Agent({ localAddress: address, keepAlive: true }); + log.info({ address }, "Using configured interface for outgoing requests."); + } else if (proxyUrl) { + process.env.HTTP_PROXY = proxyUrl; + process.env.HTTPS_PROXY = proxyUrl; + process.env.WS_PROXY = proxyUrl; + httpAgent = new ProxyAgent(); + httpsAgent = httpAgent; // ProxyAgent automatically handles HTTPS + const proxy = proxyUrl.replace(/:.*@/, "@******"); + log.info({ proxy }, "Using proxy server for outgoing requests."); + } else { + httpAgent = new http.Agent(); + httpsAgent = new https.Agent(); + } + + return [httpAgent, httpsAgent]; +} + +export function getAxiosInstance() { + if (axiosInstance) return axiosInstance; + + const [httpAgent, httpsAgent] = getHttpAgents(); + axiosInstance = axios.create({ httpAgent, httpsAgent, proxy: false }); + return axiosInstance; +} diff --git a/src/shared/users/user-store.ts b/src/shared/users/user-store.ts index d0244d8..ab0e59f 100644 --- a/src/shared/users/user-store.ts +++ b/src/shared/users/user-store.ts @@ -10,7 +10,10 @@ import admin from "firebase-admin"; import schedule from "node-schedule"; import { v4 as uuid } from "uuid"; -import { config, getFirebaseApp } from "../../config"; +import { config } from "../../config"; +import { logger } from "../../logger"; +import { getFirebaseApp } from "../firebase"; +import { APIFormat } from "../key-management"; import { getAwsBedrockModelFamily, getGcpModelFamily, @@ -22,10 +25,8 @@ import { MODEL_FAMILIES, ModelFamily, } from "../models"; -import { logger } from "../../logger"; -import { User, UserTokenCounts, UserUpdate } from "./schema"; -import { APIFormat } from "../key-management"; import { assertNever } from "../utils"; +import { User, UserTokenCounts, UserUpdate } from "./schema"; const log = logger.child({ module: "users" });