zack3d/simple-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes openai proxy

Browse Source
This commit is contained in:
nai-degen
2023-04-08 08:24:07 -05:00
committed by nai-degen
parent 66b8b6a5d0
commit 0c133b0a2d
12 changed files with 230 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+31 -17
View File
@@ -12,7 +12,7 @@
"cors": "^2.8.5",
"dotenv": "^16.0.3",
"express": "^4.18.2",
"http-proxy-middleware": "^2.0.6",
"http-proxy-middleware": "^3.0.0-beta.1",
"pino-http": "^8.3.3"
},
"devDependencies": {
@@ -88,7 +88,7 @@
"version": "1.19.2",
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz",
"integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==",
"devOptional": true,
"dev": true,
"dependencies": {
"@types/connect": "*",
"@types/node": "*"
@@ -98,7 +98,7 @@
"version": "3.4.35",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz",
"integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==",
"devOptional": true,
"dev": true,
"dependencies": {
"@types/node": "*"
}
@@ -116,7 +116,7 @@
"version": "4.17.17",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.17.tgz",
"integrity": "sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==",
"devOptional": true,
"dev": true,
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.33",
@@ -128,7 +128,7 @@
"version": "4.17.33",
"resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.33.tgz",
"integrity": "sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==",
"devOptional": true,
"dev": true,
"dependencies": {
"@types/node": "*",
"@types/qs": "*",
@@ -147,7 +147,7 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.1.tgz",
"integrity": "sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==",
"devOptional": true
"dev": true
},
"node_modules/@types/node": {
"version": "18.15.11",
@@ -158,19 +158,19 @@
"version": "6.9.7",
"resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz",
"integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==",
"devOptional": true
"dev": true
},
"node_modules/@types/range-parser": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz",
"integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==",
"devOptional": true
"dev": true
},
"node_modules/@types/serve-static": {
"version": "1.15.1",
"resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.1.tgz",
"integrity": "sha512-NUo5XNiAdULrJENtJXZZ3fHtfMolzZwczzBbnAeBbqBwG+LaG6YaJtuwzwGSQZ2wsCrxjEhNNjAkKigy3n8teQ==",
"devOptional": true,
"dev": true,
"dependencies": {
"@types/mime": "*",
"@types/node": "*"
@@ -775,28 +775,42 @@
}
},
"node_modules/http-proxy-middleware": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz",
"integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==",
"version": "3.0.0-beta.1",
"resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-3.0.0-beta.1.tgz",
"integrity": "sha512-hdiTlVVoaxncf239csnEpG5ew2lRWnoNR1PMWOO6kYulSphlrfLs5JFZtFVH3R5EUWSZNMkeUqvkvfctuWaK8A==",
"dependencies": {
"@types/http-proxy": "^1.17.8",
"@types/http-proxy": "^1.17.10",
"debug": "^4.3.4",
"http-proxy": "^1.18.1",
"is-glob": "^4.0.1",
"is-plain-obj": "^3.0.0",
"micromatch": "^4.0.2"
"micromatch": "^4.0.5"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/http-proxy-middleware/node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
"dependencies": {
"ms": "2.1.2"
},
"peerDependencies": {
"@types/express": "^4.17.13"
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"@types/express": {
"supports-color": {
"optional": true
}
}
},
"node_modules/http-proxy-middleware/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
package.json
+1 -1
View File
@@ -13,7 +13,7 @@
"cors": "^2.8.5",
"dotenv": "^16.0.3",
"express": "^4.18.2",
"http-proxy-middleware": "^2.0.6",
"http-proxy-middleware": "^3.0.0-beta.1",
"pino-http": "^8.3.3"
},
"devDependencies": {
src/info-page.ts
+2 -2
View File
@@ -11,8 +11,8 @@ function getInfoPageHtml(host: string) {
message: "OpenAI Reverse Proxy",
uptime: process.uptime(),
timestamp: Date.now(),
kobold: host + "/kobold",
openai: host + "/openai",
kobold: host + "/proxy/kobold",
openai: host + "/proxy/openai",
keys: {
all: keylist.length,
active: keylist.filter((k) => !k.isDisabled).length,
src/keys.ts
+1 -1
View File
@@ -44,7 +44,7 @@ function init() {
const decoded = Buffer.from(keyString, "base64").toString();
keyList = JSON.parse(decoded) as KeySchema[];
} catch (err) {
console.log("Key is not base64-encoded JSON, assuming it's a bare key");
logger.info("OPENAI_KEY is not base64-encoded JSON, assuming bare key");
// We don't actually know if bare keys are paid/GPT-4 so we assume they are
keyList = [{ key: keyString, isTrial: false, isGpt4: true }];
}
src/openai.ts
-7
View File
@@ -60,10 +60,6 @@ const handleResponse = (
errorPayload.proxy_note = message;
} else if (statusCode === 429) {
// Rate limit exceeded
// Annoyingly they send this for:
// - Quota exceeded, key is totally dead
// - Rate limit exceeded, key is still good but backoff needed
// - Model overloaded, their server is fucked
if (errorPayload.error?.type === "insufficient_quota") {
logger.warn(`OpenAI key is exhausted. Keyhash ${req.key?.hash}`);
keys.disable(req.key!);
@@ -104,9 +100,6 @@ const openaiProxy = createProxyMiddleware({
const openaiRouter = Router();
openaiRouter.post("/v1/chat/completions", openaiProxy);
// openaiRouter.post("/v1/completions", openaiProxy);
// openaiRouter.get("/v1/models", handleModels);
// openaiRouter.get("/dashboard/billing/usage, handleUsage);
openaiRouter.use((req, res) => {
logger.warn(`Blocked openai proxy request: ${req.method} ${req.path}`);
res.status(404).json({ error: "Not found" });
src/proxy.ts
-20
View File
@@ -1,20 +0,0 @@
/* Accepts incoming requests at either the /kobold or /openai routes and then
routes them to the appropriate handler to be forwarded to the OpenAI API.
Incoming openai requests are more or less 1:1 with the OpenAI API, but only a
subset of the API is supported. Kobold requests are more complex and are
translated into OpenAI requests. */
import * as express from "express";
import { auth } from "./auth";
import { handleInfoPage } from "./info-page";
import { kobold } from "./kobold";
import { openai } from "./openai";
const router = express.Router();
router.get("/", handleInfoPage);
router.use(auth);
router.use("/kobold", kobold);
router.use("/openai", openai);
export { router as proxy };
src/proxy/auth.ts
+16
View File
@@ -0,0 +1,16 @@
import type { Request, Response, NextFunction } from "express";
const PROXY_KEY = process.env.PROXY_KEY;
export const auth = (req: Request, res: Response, next: NextFunction) => {
if (!PROXY_KEY) {
next();
return;
}
if (req.headers.authorization === `Bearer ${PROXY_KEY}`) {
delete req.headers.authorization;
next();
} else {
res.status(401).json({ error: "Unauthorized" });
}
};
src/proxy/common.ts
+85
View File
@@ -0,0 +1,85 @@
import { Request, Response } from "express";
import * as http from "http";
import * as httpProxy from "http-proxy";
import { logger } from "../logger";
import { keys } from "../keys";
/** Handle and rewrite response to proxied requests to OpenAI */
export const handleResponse = (
proxyRes: http.IncomingMessage,
req: Request,
res: Response
) => {
const statusCode = proxyRes.statusCode || 500;
if (statusCode >= 400) {
let body = "";
proxyRes.on("data", (chunk) => (body += chunk));
proxyRes.on("end", () => {
let errorPayload: any = {
error: "Proxy couldn't parse error from OpenAI",
};
const canTryAgain = keys.anyAvailable()
? "You can try again to get a different key."
: "There are no more keys available.";
try {
errorPayload = JSON.parse(body);
} catch (err) {
logger.error({ error: err }, errorPayload.error);
res.json(errorPayload);
return;
}
if (statusCode === 401) {
// Key is invalid or was revoked
logger.warn(
`OpenAI key is invalid or revoked. Keyhash ${req.key?.hash}`
);
keys.disable(req.key!);
const message = `The OpenAI key is invalid or revoked. ${canTryAgain}`;
errorPayload.proxy_note = message;
} else if (statusCode === 429) {
// Rate limit exceeded
// Annoyingly they send this for:
// - Quota exceeded, key is totally dead
// - Rate limit exceeded, key is still good but backoff needed
// - Model overloaded, their server is overloaded
if (errorPayload.error?.type === "insufficient_quota") {
logger.warn(`OpenAI key is exhausted. Keyhash ${req.key?.hash}`);
keys.disable(req.key!);
const message = `The OpenAI key is exhausted. ${canTryAgain}`;
errorPayload.proxy_note = message;
} else {
logger.warn(
{ errorCode: errorPayload.error?.type },
`OpenAI rate limit exceeded or model overloaded. Keyhash ${req.key?.hash}`
);
}
}
res.status(statusCode).json(errorPayload);
});
} else {
Object.keys(proxyRes.headers).forEach((key) => {
res.setHeader(key, proxyRes.headers[key] as string);
});
proxyRes.pipe(res);
}
};
export const onError: httpProxy.ErrorCallback = (err, _req, res) => {
logger.error({ error: err }, "Error proxying to OpenAI");
(res as http.ServerResponse).writeHead(500, {
"Content-Type": "application/json",
});
res.end(
JSON.stringify({
error: {
type: "proxy_error",
message: err.message,
proxy_note:
"Reverse proxy encountered an error before it could reach OpenAI.",
},
})
);
};
src/proxy/kobold.ts
+6
View File
@@ -0,0 +1,6 @@
import { Request, Response, NextFunction } from "express";
export const kobold = (req: Request, res: Response, next: NextFunction) => {
// TODO: Implement kobold
res.status(501).json({ error: "Not implemented" });
};
src/proxy/openai.ts
+59
View File
@@ -0,0 +1,59 @@
import { Request, Router } from "express";
import * as http from "http";
import { createProxyMiddleware, fixRequestBody } from "http-proxy-middleware";
import { logger } from "../logger";
import { Key, keys } from "../keys";
import { handleResponse, onError } from "./common";
/**
* Modifies the request body to add a randomly selected API key.
*/
const rewriteRequest = (proxyReq: http.ClientRequest, req: Request) => {
let key: Key;
try {
key = keys.get(req.body?.model || "gpt-3.5")!;
} catch (err) {
proxyReq.destroy(err as any);
return;
}
req.key = key;
proxyReq.setHeader("Authorization", `Bearer ${key.key}`);
if (req.method === "POST" && req.body) {
// body-parser and http-proxy-middleware don't play nice together
fixRequestBody(proxyReq, req);
}
if (req.body?.stream) {
req.body.stream = false;
const updatedBody = JSON.stringify(req.body);
proxyReq.setHeader("Content-Length", Buffer.byteLength(updatedBody));
proxyReq.write(updatedBody);
proxyReq.end();
}
};
const openaiProxy = createProxyMiddleware({
target: "https://api.openai.com",
changeOrigin: true,
on: {
proxyReq: rewriteRequest,
proxyRes: handleResponse,
error: onError,
},
selfHandleResponse: true,
logger,
});
const openaiRouter = Router();
openaiRouter.post("/v1/chat/completions", openaiProxy);
// openaiRouter.post("/v1/completions", openaiProxy); // TODO: Implement Davinci
openaiRouter.get("/v1/models", openaiProxy);
openaiRouter.use((req, res) => {
logger.warn(`Blocked openai proxy request: ${req.method} ${req.path}`);
res.status(404).json({ error: "Not found" });
});
export const openai = openaiRouter;
src/proxy/routes.ts
+18
View File
@@ -0,0 +1,18 @@
/* Accepts incoming requests at either the /kobold or /openai routes and then
routes them to the appropriate handler to be forwarded to the OpenAI API.
Incoming OpenAI requests are more or less 1:1 with the OpenAI API, but only a
subset of the API is supported. Kobold requests must be transformed into
equivalent OpenAI requests. */
import * as express from "express";
import { auth } from "./auth";
import { kobold } from "./kobold";
import { openai } from "./openai";
const router = express.Router();
router.use(auth);
router.use("/kobold", kobold);
router.use("/openai", openai);
export { router as proxyRouter };
src/server.ts
+11 -11
View File
@@ -3,28 +3,25 @@ dotenv.config();
import express from "express";
import cors from "cors";
import pinoHttp from "pino-http";
import { logger } from "./logger";
import { keys } from "./keys";
import { proxy } from "./proxy";
import { proxyRouter } from "./proxy/routes";
import { handleInfoPage } from "./info-page";
const PORT = process.env.PORT || 7860;
const app = express();
// middleware
app.use(pinoHttp({ logger }));
app.use(cors());
app.use(
express.json({ limit: "10mb" }),
express.urlencoded({ extended: true, limit: "10mb" })
);
app.use("/", proxy);
app.use((_req: unknown, res: express.Response) => {
res.status(404).json({ error: "Not found" });
});
// routes
app.get("/", handleInfoPage);
app.use("/proxy", proxyRouter);
// 500 and 404
app.use((err: any, _req: unknown, res: express.Response, _next: unknown) => {
if (err.status) {
res.status(err.status).json({ error: err.message });
@@ -33,7 +30,10 @@ app.use((err: any, _req: unknown, res: express.Response, _next: unknown) => {
res.status(500).json({ error: "Internal server error" });
}
});
app.use((_req: unknown, res: express.Response) => {
res.status(404).json({ error: "Not found" });
});
// start server and load keys
app.listen(PORT, () => {
logger.info(`Server listening on port ${PORT}`);
keys.init();