returns static model list instead of proxying

.gitignore

@@ -1,5 +1,6 @@
-.vscode
 .env
+.vscode
 build
-node_modules
 greeting.md
+node_modules
+venv

package.json

@@ -6,7 +6,7 @@
     "build:watch": "esbuild src/server.ts --outfile=build/server.js --platform=node --target=es2020 --format=cjs --bundle --sourcemap --watch",
     "build": "tsc",
     "start:dev": "concurrently \"npm run build:watch\" \"npm run start:watch\"",
-    "start:dev:tsc": "nodemon --watch src --exec ts-node src/server.ts",
+    "start:dev:tsc": "nodemon --watch src --exec ts-node --transpile-only src/server.ts",
     "start:watch": "nodemon --require source-map-support/register build/server.js",
     "start:replit": "tsc && node build/server.js",
     "start": "node build/server.js",

src/proxy/anthropic.ts

@@ -145,7 +145,16 @@ anthropicRouter.get("*", (req, res, next) => {
   }
 });
 
+let modelsCache: any = null;
+let modelsCacheTime = 0;
+
 function buildFakeModelsResponse() {
+  if (new Date().getTime() - modelsCacheTime < 1000 * 60) {
+    return modelsCache;
+  }
+
+  if (!config.anthropicKey) return { object: "list", data: [] };
+
   const claudeVariants = [
     "claude-v1",
     "claude-v1-100k",
@@ -170,7 +179,10 @@ function buildFakeModelsResponse() {
     parent: null,
   }));
 
-  return { models };
+  modelsCache = { object: "list", data: models };
+  modelsCacheTime = new Date().getTime();
+
+  return modelsCache;
 }
 
 export const anthropic = anthropicRouter;

src/proxy/openai.ts

@@ -2,6 +2,7 @@ import { Request, Router } from "express";
 import * as http from "http";
 import { createProxyMiddleware } from "http-proxy-middleware";
 import { config } from "../config";
+import { keyPool } from "../key-management";
 import { logger } from "../logger";
 import { createQueueMiddleware } from "./queue";
 import { ipLimiter } from "./rate-limit";
@@ -86,7 +87,9 @@ openaiRouter.use((req, _res, next) => {
 openaiRouter.get(
   "/v1/models",
   setApiFormat({ in: "openai", out: "openai" }),
-  openaiProxy // TODO: replace with fake model list instead of proxying
+  (_req, res) => {
+    res.json(buildFakeModelsResponse());
+  }
 );
 openaiRouter.post(
   "/v1/chat/completions",
@@ -108,4 +111,57 @@ openaiRouter.use((req, res) => {
   res.status(404).json({ error: "Not found" });
 });
 
+let modelsCache: any = null;
+let modelsCacheTime = 0;
+
+function buildFakeModelsResponse() {
+  if (new Date().getTime() - modelsCacheTime < 1000 * 60) {
+    return modelsCache;
+  }
+
+  const gptVariants = [
+    "gpt-4",
+    "gpt-4-0314",
+    "gpt-4-32k",
+    "gpt-4-32k-0314",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0301",
+  ];
+
+  const gpt4Available = keyPool.list().filter((key) => {
+    return key.service === "openai" && !key.isDisabled && key.isGpt4;
+  }).length;
+
+  const models = gptVariants
+    .map((id) => ({
+      id,
+      object: "model",
+      created: new Date().getTime(),
+      owned_by: "openai",
+      permission: [
+        {
+          id: "modelperm-" + id,
+          object: "model_permission",
+          created: new Date().getTime(),
+          organization: "*",
+          group: null,
+          is_blocking: false,
+        },
+      ],
+      root: id,
+      parent: null,
+    }))
+    .filter((model) => {
+      if (model.id.startsWith("gpt-4")) {
+        return gpt4Available > 0;
+      }
+      return true;
+    });
+
+  modelsCache = { object: "list", data: models };
+  modelsCacheTime = new Date().getTime();
+
+  return modelsCache;
+}
+
 export const openai = openaiRouter;