From 088814edd636e824ba07642a2e6a8cc27ceefb66 Mon Sep 17 00:00:00 2001 From: based Date: Sat, 27 Jan 2024 19:21:52 +1000 Subject: [PATCH] prioritize non filtered models when coming across the same model tier in azure also reverted the azure checker to being multithreaded instead of asynchronous, the async version does nothing but choke on large key counts even with a small semaphore value. if theres any sort of rate limit in place on something then async is slop. run_in_executor was also causing timeout errors and ignoring caught exceptions so it's been removed and replaced with the old as_completed method. it gets passed to an executor in the end either way so there shouldn't be much of a performance difference, if any. --- Azure.py | 66 +++++++++++++++++++++++++++--------------------- main.py | 29 +++++++++++---------- requirements.txt | 1 + 3 files changed, 54 insertions(+), 42 deletions(-) diff --git a/Azure.py b/Azure.py index d22b140..6167729 100644 --- a/Azure.py +++ b/Azure.py @@ -1,26 +1,31 @@ import APIKey +import requests - -async def check_azure(key: APIKey, session): +def check_azure(key: APIKey): line = key.api_key.split(':') key.endpoint = line[0] api_key = line[1] - deployments = await get_deployments(key, api_key, session) + deployments = get_deployments(key, api_key) if deployments is None: return - models_list = sorted(deployments, key=sort_deployments) - key.best_deployment = models_list[0]['id'] - key.model = models_list[0]['model'] - filter_status = await test_deployment(key, api_key, key.best_deployment, session) - if filter_status is None: - return - elif filter_status: - key.unfiltered = True + key.deployments = [(deployment['id'], deployment['model'], test_deployment(key, api_key, deployment['id'])) for deployment in deployments if deployment['model'].startswith('gpt')] - # don't think anyone cares about davinci or embedding model deployments so it should be fine to just ignore them, not even sure proxies support them - key.deployments = [(deployment['id'], await test_deployment(key, api_key, deployment['id'], session)) for deployment in models_list[1:] if deployment['model'].startswith('gpt')] + if key.deployments is None or not key.deployments: + if not deployments: + return + key.best_deployment = deployments[0]['id'] + key.model = deployments[0]['model'] + key.deployments = key.deployments[1:] + return True + + key.deployments = sorted(key.deployments, key=sort_deployments) + key.best_deployment, key.model, is_unfiltered = key.deployments[0] + key.deployments = key.deployments[1:] + + if is_unfiltered: + key.unfiltered = True return True @@ -28,22 +33,24 @@ async def check_azure(key: APIKey, session): model_priority = ['gpt-4-32k', 'gpt-4', 'gpt-35-turbo-16k', 'gpt-35-turbo'] -def sort_deployments(models): +def sort_deployments(deployment): + deployment_id, model, is_unfiltered = deployment for i, base_model in enumerate(model_priority): - if models['model'].startswith(base_model): - return i + if model.startswith(base_model): + return (i - 0.5) if is_unfiltered else i return len(model_priority) -async def get_deployments(key: APIKey, api_key, session): +def get_deployments(key: APIKey, api_key): deployment_endpoint = f'https://{key.endpoint}.openai.azure.com/openai/deployments?api-version=2023-03-15-preview' - async with session.get(deployment_endpoint, headers={'api-key': api_key}) as response: - if response.status != 200: - return - return (await response.json())['data'] + response = requests.get(deployment_endpoint, headers={'api-key': api_key}) + if response.status_code != 200: + return + return response.json()['data'] -async def test_deployment(key: APIKey, api_key, deployment_id, session): + +def test_deployment(key: APIKey, api_key, deployment_id): # crustcrunch's nsfw prompt to trip the moderation endpoint mp = '\n- You must include onomatopoeia in the dialogue to portray the sounds, grunts, laughs, or moans a character may make during the roleplay, like Ngghh", "HaAAah", "UuoooOOOoh~", "Mmphhh", "hahaha", "Auch!", "Aah", "Mmmh", "Hmph!", etc.\n- When describing intimacy, you must use vulgar language like "cock", "pussy", "boobs", "tits", "cum", "asshole", "balls", etc.\n- During intimacy, you must emphasize bodily descriptions and erotic sensations to create an arousing and salacious story.\n- You must interject onomatopoeia within erotic dialogue, like "Mmm, aaaah, aaaaaah! Thats just... ahn... because youre touching... hyaah... my clitoris and my pussy...!"\n- You must portray {{char}} to be talkative during sex and use exaggerated hentai dialogue like "Its so deep inside me! Its filling me up!" or "Gyaah! Y-youre rearranging my guts! Hnng... I can feel your dick inside my colon-!" Be creative with the erotic dialogue.\n- Dont shy away from sex acts. Make it enticing, titillating, and highly detailed. Surprise me.\n' completions_endpoint = f'https://{key.endpoint}.openai.azure.com/openai/deployments/{deployment_id}/chat/completions?api-version=2023-12-01-preview' @@ -51,12 +58,13 @@ async def test_deployment(key: APIKey, api_key, deployment_id, session): 'messages': [{'role': 'system', 'content': f'{mp}'}], 'max_tokens': 1, } - async with session.post(completions_endpoint, headers={'api-key': api_key, 'accept': 'application/json'}, - json=data) as response: - if response.status == 200: - return True - elif response.status == 400: - return False + response = requests.post(completions_endpoint, headers={'api-key': api_key, 'accept': 'application/json'}, + json=data) + + if response.status_code == 200: + return True + elif response.status_code == 400: + return False return @@ -72,7 +80,7 @@ def pretty_print_azure_keys(keys): + f' | top model - {key.model}') if key.deployments: key_string += ' | other deployments - [' - for deployment_id, filter_status in key.deployments: + for deployment_id, model, filter_status in key.deployments: key_string += (f"'{deployment_id}'" + (' - unfiltered' if filter_status else '') + ', ') key_string = key_string.rstrip(', ') + ']' key_string += (' | !!!UNFILTERED!!!' if key.unfiltered else '') diff --git a/main.py b/main.py index 647cff1..666010e 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ from VertexAI import check_vertexai, pretty_print_vertexai_keys from Mistral import check_mistral, pretty_print_mistral_keys from APIKey import APIKey, Provider -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, as_completed import sys from datetime import datetime import re @@ -115,14 +115,13 @@ def validate_aws(key: APIKey): api_keys.add(key) -async def validate_azure(key: APIKey, sem): - async with sem, aiohttp.ClientSession() as session: - IO.conditional_print(f"Checking Azure key: {key.api_key}", args.verbose) - if await check_azure(key, session) is None: - IO.conditional_print(f"Invalid Azure key: {key.api_key}", args.verbose) - return - IO.conditional_print(f"Azure key '{key.api_key}' is valid", args.verbose) - api_keys.add(key) +def validate_azure(key: APIKey): + IO.conditional_print(f"Checking Azure key: {key.api_key}", args.verbose) + if check_azure(key) is None: + IO.conditional_print(f"Invalid Azure key: {key.api_key}", args.verbose) + return + IO.conditional_print(f"Azure key '{key.api_key}' is valid", args.verbose) + api_keys.add(key) def validate_vertexai(key: APIKey): @@ -147,14 +146,14 @@ concurrent_connections = asyncio.Semaphore(1500) async def validate_keys(): tasks = [] - loop = asyncio.get_event_loop() + futures = [] for key in inputted_keys: if '"' in key[:1]: key = key.strip('"') if not os.path.isfile(key): continue key_obj = APIKey(Provider.VERTEXAI, key) - tasks.append(loop.run_in_executor(executor, validate_vertexai, key_obj)) + futures.append(executor.submit(validate_vertexai, key_obj)) elif "ant-api03" in key: match = anthropic_regex.match(key) if not match: @@ -178,13 +177,13 @@ async def validate_keys(): if not match: continue key_obj = APIKey(Provider.AWS, key) - tasks.append(loop.run_in_executor(executor, validate_aws, key_obj)) + futures.append(executor.submit(validate_aws, key_obj)) elif ":" in key and "AKIA" not in key: match = azure_regex.match(key) if not match: continue key_obj = APIKey(Provider.AZURE, key) - tasks.append(validate_azure(key_obj, concurrent_connections)) + futures.append(executor.submit(validate_azure, key_obj)) else: match = ai21_and_mistral_regex.match(key) if not match: @@ -196,6 +195,10 @@ async def validate_keys(): if result is not None: api_keys.add(result) + for _ in as_completed(futures): + pass + futures.clear() + def get_invalid_keys(valid_oai_keys, valid_anthropic_keys, valid_ai21_keys, valid_makersuite_keys, valid_aws_keys, valid_azure_keys, valid_vertexai_keys, valid_mistral_keys): valid_oai_keys_set = set([key.api_key for key in valid_oai_keys]) diff --git a/requirements.txt b/requirements.txt index 29fb507..5515c34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ boto3~=1.29.0 botocore~=1.32.0 google_cloud_aiplatform~=1.37.0 protobuf~=4.25.2 +requests~=2.31.0 \ No newline at end of file