Add docs and support for Render.com deployments (khanon/oai-reverse-proxy!9)

2023-05-15 21:47:30 +00:00
parent 7f202dc9ef
commit ec4e7a3671
13 changed files with 152 additions and 72 deletions
@@ -16,7 +16,8 @@ export const handleInfoPage = (req: Request, res: Response) => {
    return;
  }

-  // Huggingface puts spaces behind some cloudflare ssl proxy, so `req.protocol` is `http` but the correct URL is actually `https`
+  // Some load balancers/reverse proxies don't give us the right protocol in
+  // the host header. Huggingface works this way, Cloudflare does not.
  const host = req.get("host");
  const isHuggingface = host?.includes("hf.space");
  const protocol = isHuggingface ? "https" : req.protocol;
@@ -80,12 +81,10 @@ function cacheInfoPageHtml(host: string) {
    ...getQueueInformation(),
    keys: keyInfo,
    config: listConfig(),
-    commitSha: process.env.COMMIT_SHA || "dev",
+    build: process.env.COMMIT_SHA || "dev",
  };

-  const title = process.env.SPACE_ID
-    ? `${process.env.SPACE_AUTHOR_NAME} / ${process.env.SPACE_TITLE}`
-    : "OAI Reverse Proxy";
+  const title = getServerTitle();
  const headerHtml = buildInfoPageHeader(new showdown.Converter(), title);

  const pageBody = `<!DOCTYPE html>
@@ -160,3 +159,22 @@ function getQueueInformation() {
    estimatedQueueTime: waitMs > 2000 ? waitTime : "no wait",
  };
 }
+
+function getServerTitle() {
+  // Use manually set title if available
+  if (process.env.SERVER_TITLE) {
+    return process.env.SERVER_TITLE;
+  }
+
+  // Huggingface
+  if (process.env.SPACE_ID) {
+    return `${process.env.SPACE_AUTHOR_NAME} / ${process.env.SPACE_TITLE}`;
+  }
+
+  // Render
+  if (process.env.RENDER) {
+    return `Render / ${process.env.RENDER_SERVICE_NAME}`;
+  }
+
+  return "OAI Reverse Proxy";
+}
@@ -22,8 +22,12 @@ app.use(
  pinoHttp({
    quietReqLogger: true,
    logger,
-    // SillyTavern spams the hell out of this endpoint so don't log it
-    autoLogging: { ignore: (req) => req.url === "/proxy/kobold/api/v1/model" },
+    autoLogging: {
+      ignore: (req) => {
+        const ignored = ["/proxy/kobold/api/v1/model", "/health"];
+        return ignored.includes(req.url as string);
+      },
+    },
    redact: {
      paths: [
        "req.headers.cookie",
@@ -36,6 +40,8 @@ app.use(
    },
  })
 );
+
+app.get("/health", (_req, res) => res.sendStatus(200));
 app.use((req, _res, next) => {
  req.startTime = Date.now();
  req.retryCount = 0;
@@ -46,9 +52,10 @@ app.use(
  express.json({ limit: "10mb" }),
  express.urlencoded({ extended: true, limit: "10mb" })
 );
-// TODO: this works if we're always being deployed to Huggingface but if users
-// deploy this somewhere without a load balancer then incoming requests can
-// spoof the X-Forwarded-For header and bypass the rate limiting.
+
+// TODO: Detect (or support manual configuration of) whether the app is behind
+// a load balancer/reverse proxy, which is necessary to determine request IP
+// addresses correctly.
 app.set("trust proxy", true);

 // routes
@@ -126,6 +133,17 @@ function registerUncaughtExceptionHandler() {
 }

 function setGitSha() {
+  // On Render, the .git directory isn't available in the docker build context
+  // so we can't get the SHA directly, but they expose it as an env variable.
+  if (process.env.RENDER) {
+    const shaString = `${process.env.RENDER_GIT_COMMIT?.slice(0, 7)} (${
+      process.env.RENDER_GIT_REPO_SLUG
+    })`;
+    process.env.COMMIT_SHA = shaString;
+    logger.info({ sha: shaString }, "Got commit SHA via Render config.");
+    return;
+  }
+
  try {
    // Huggingface seems to have changed something about how they deploy Spaces
    // and git commands fail because of some ownership issue with the .git
@@ -7,7 +7,7 @@ declare global {
    interface Request {
      key?: Key;
      api: "kobold" | "openai" | "anthropic";
-      user: User;
+      user?: User;
      isStreaming?: boolean;
      startTime: number;
      retryCount: number;