adds azure rate limit auto-retry

scripts/test-concurrency.js

@@ -0,0 +1,44 @@
+const axios = require("axios");
+
+const concurrentRequests = 5;
+const headers = {
+  Authorization: "Bearer test",
+  "Content-Type": "application/json",
+};
+
+const payload = {
+  model: "gpt-4",
+  max_tokens: 1,
+  stream: false,
+  messages: [{ role: "user", content: "Hi" }],
+};
+
+const makeRequest = async (i) => {
+  try {
+    const response = await axios.post(
+      "http://localhost:7860/proxy/azure/openai/v1/chat/completions",
+      payload,
+      { headers }
+    );
+    console.log(
+      `Req ${i} finished with status code ${response.status} and response:`,
+      response.data
+    );
+  } catch (error) {
+    console.error(`Error in req ${i}:`, error.message);
+  }
+};
+
+const executeRequestsConcurrently = () => {
+  const promises = [];
+  for (let i = 1; i <= concurrentRequests; i++) {
+    console.log(`Starting request ${i}`);
+    promises.push(makeRequest(i));
+  }
+
+  Promise.all(promises).then(() => {
+    console.log("All requests finished");
+  });
+};
+
+executeRequestsConcurrently();

scripts/test_concurrency.ps1

@@ -1,40 +0,0 @@
-$NumThreads = 10
-
-$runspacePool = [runspacefactory]::CreateRunspacePool(1, $NumThreads)
-$runspacePool.Open()
-$runspaces = @()
-
-$headers = @{
-    "Authorization" = "Bearer test"
-    "anthropic-version" = "2023-01-01"
-    "Content-Type" = "application/json"
-}
-
-$payload = @{
-    model = "claude-v2"
-    max_tokens_to_sample = 40
-    temperature = 0
-    stream = $true
-    prompt = "Test prompt, please reply with lorem ipsum`n`n:Assistant:"
-} | ConvertTo-Json
-
-for ($i = 1; $i -le $NumThreads; $i++) {
-    Write-Host "Starting thread $i"
-    $runspace = [powershell]::Create()
-    $runspace.AddScript({
-        param($i, $headers, $payload)
-        $response = Invoke-WebRequest -Uri "http://localhost:7860/proxy/aws/claude/v1/complete" -Method Post -Headers $headers -Body $payload
-        Write-Host "Response from server: $($response.StatusCode)"
-    }).AddArgument($i).AddArgument($headers).AddArgument($payload)
-
-    $runspace.RunspacePool = $runspacePool
-    $runspaces += [PSCustomObject]@{ Pipe = $runspace; Status = $runspace.BeginInvoke() }
-}
-
-$runspaces | ForEach-Object {
-    $_.Pipe.EndInvoke($_.Status)
-    $_.Pipe.Dispose()
-}
-
-$runspacePool.Close()
-$runspacePool.Dispose()