adds gpt4-turbo model family and support for gpt-4-1106-preview model

src/shared/key-management/openai/checker.ts

@@ -2,6 +2,7 @@ import axios, { AxiosError } from "axios";
 import type { OpenAIModelFamily } from "../../models";
 import { KeyCheckerBase } from "../key-checker-base";
 import type { OpenAIKey, OpenAIKeyProvider } from "./provider";
+import { getOpenAIModelFamily } from "../../models";
 
 const MIN_CHECK_INTERVAL = 3 * 1000; // 3 seconds
 const KEY_CHECK_PERIOD = 60 * 60 * 1000; // 1 hour
@@ -94,29 +95,21 @@ export class OpenAIKeyChecker extends KeyCheckerBase<OpenAIKey> {
     const { data } = await axios.get<GetModelsResponse>(GET_MODELS_URL, opts);
     const models = data.data;
 
-    const families: OpenAIModelFamily[] = [];
-    if (models.some(({ id }) => id.startsWith("gpt-3.5-turbo"))) {
-      families.push("turbo");
-    }
-
-    if (models.some(({ id }) => id.startsWith("gpt-4"))) {
-      families.push("gpt4");
-    }
-
-    if (models.some(({ id }) => id.startsWith("gpt-4-32k"))) {
-      families.push("gpt4-32k");
-    }
+    // const families: OpenAIModelFamily[] = [];
+    const families = new Set<OpenAIModelFamily>();
+    models.forEach(({ id }) => families.add(getOpenAIModelFamily(id, "turbo")));
 
     // We want to update the key's model families here, but we don't want to
     // update its `lastChecked` timestamp because we need to let the liveness
     // check run before we can consider the key checked.
 
+    const familiesArray = [...families];
     const keyFromPool = this.keys.find((k) => k.hash === key.hash)!;
     this.updateKey(key.hash, {
-      modelFamilies: families,
+      modelFamilies: familiesArray,
       lastChecked: keyFromPool.lastChecked,
     });
-    return families;
+    return familiesArray;
   }
 
   private async maybeCreateOrganizationClones(key: OpenAIKey) {

src/shared/key-management/openai/provider.ts

@@ -14,6 +14,7 @@ export type OpenAIModel =
   | "gpt-3.5-turbo-instruct"
   | "gpt-4"
   | "gpt-4-32k"
+  | "gpt-4-1106"
   | "text-embedding-ada-002";
 export const OPENAI_SUPPORTED_MODELS: readonly OpenAIModel[] = [
   "gpt-3.5-turbo",
@@ -98,7 +99,11 @@ export class OpenAIKeyProvider implements KeyProvider<OpenAIKey> {
       const newKey: OpenAIKey = {
         key: k,
         service: "openai" as const,
-        modelFamilies: ["turbo" as const, "gpt4" as const],
+        modelFamilies: [
+          "turbo" as const,
+          "gpt4" as const,
+          "gpt4-turbo" as const,
+        ],
         isTrial: false,
         isDisabled: false,
         isRevoked: false,
@@ -117,6 +122,7 @@ export class OpenAIKeyProvider implements KeyProvider<OpenAIKey> {
         turboTokens: 0,
         gpt4Tokens: 0,
         "gpt4-32kTokens": 0,
+        "gpt4-turboTokens": 0,
       };
       this.keys.push(newKey);
     }
@@ -383,10 +389,9 @@ export class OpenAIKeyProvider implements KeyProvider<OpenAIKey> {
     const now = Date.now();
     const key = this.keys.find((k) => k.hash === hash)!;
 
-    const currentRateLimit = Math.max(
-      key.rateLimitRequestsReset,
-      key.rateLimitTokensReset
-    ) + key.rateLimitedAt;
+    const currentRateLimit =
+      Math.max(key.rateLimitRequestsReset, key.rateLimitTokensReset) +
+      key.rateLimitedAt;
     const nextRateLimit = now + KEY_REUSE_DELAY;
 
     // Don't throttle if the key is already naturally rate limited.

src/shared/models.ts

@@ -1,6 +1,6 @@
 import { logger } from "../logger";
 
-export type OpenAIModelFamily = "turbo" | "gpt4" | "gpt4-32k";
+export type OpenAIModelFamily = "turbo" | "gpt4" | "gpt4-32k" | "gpt4-turbo";
 export type AnthropicModelFamily = "claude";
 export type GooglePalmModelFamily = "bison";
 export type AwsBedrockModelFamily = "aws-claude";
@@ -16,12 +16,14 @@ export const MODEL_FAMILIES = (<A extends readonly ModelFamily[]>(
   "turbo",
   "gpt4",
   "gpt4-32k",
+  "gpt4-turbo",
   "claude",
   "bison",
   "aws-claude",
 ] as const);
 
 export const OPENAI_MODEL_FAMILY_MAP: { [regex: string]: OpenAIModelFamily } = {
+  "^gpt-4-1106(-preview)?$": "gpt4-turbo",
   "^gpt-4-32k-\\d{4}$": "gpt4-32k",
   "^gpt-4-32k$": "gpt4-32k",
   "^gpt-4-\\d{4}$": "gpt4",
@@ -30,13 +32,14 @@ export const OPENAI_MODEL_FAMILY_MAP: { [regex: string]: OpenAIModelFamily } = {
   "^text-embedding-ada-002$": "turbo",
 };
 
-export function getOpenAIModelFamily(model: string): OpenAIModelFamily {
+export function getOpenAIModelFamily(
+  model: string,
+  defaultFamily: OpenAIModelFamily = "gpt4"
+): OpenAIModelFamily {
   for (const [regex, family] of Object.entries(OPENAI_MODEL_FAMILY_MAP)) {
     if (model.match(regex)) return family;
   }
-  const stack = new Error().stack;
-  logger.warn({ model, stack }, "Unmapped model family");
-  return "gpt4";
+  return defaultFamily;
 }
 
 export function getClaudeModelFamily(_model: string): ModelFamily {

src/shared/stats.ts

@@ -5,6 +5,9 @@ import { ModelFamily } from "./models";
 export function getTokenCostUsd(model: ModelFamily, tokens: number) {
   let cost = 0;
   switch (model) {
+    case "gpt4-turbo":
+      cost = 0.00001;
+      break;
     case "gpt4-32k":
       cost = 0.00006;
       break;
@@ -12,7 +15,7 @@ export function getTokenCostUsd(model: ModelFamily, tokens: number) {
       cost = 0.00003;
       break;
     case "turbo":
-      cost = 0.0000015;
+      cost = 0.000001;
       break;
     case "aws-claude":
     case "claude":

src/shared/users/user-store.ts

@@ -21,6 +21,7 @@ const INITIAL_TOKENS: Required<UserTokenCounts> = {
   turbo: 0,
   gpt4: 0,
   "gpt4-32k": 0,
+  "gpt4-turbo": 0,
   claude: 0,
   bison: 0,
   "aws-claude": 0,
@@ -362,6 +363,9 @@ async function flushUsers() {
 
 // TODO: use key-management/models.ts for family mapping
 function getModelFamilyForQuotaUsage(model: string): ModelFamily {
+  if (model.startsWith("gpt-4-1106")) {
+    return "gpt4-turbo";
+  }
   if (model.includes("32k")) {
     return "gpt4-32k";
   }