diff --git a/api-reference/benchmarks/filter_benchmark.mdx b/api-reference/benchmarks/filter_benchmark.mdx deleted file mode 100644 index 22c17a5c9..000000000 --- a/api-reference/benchmarks/filter_benchmark.mdx +++ /dev/null @@ -1,84 +0,0 @@ ---- -title: 'Filter Benchmark' -api: 'POST /v0/benchmarks/filter' ---- - - -#### Authorizations - - - Bearer authentication header of the form `Bearer `, where `` is your auth token. - - -#### Query Parameters - - -Model name - - - -Provider name - - - -Window start time - - - -Window end time - - - - - - - -Region where the benchmark is run. Options are: "Belgium", "Hong Kong", "Iowa". - - - -Length of the sequence used for benchmarking. Options are: "short", "long". - - - - -```bash cURL -curl --request POST \ - --url 'https://api.unify.ai/v0/benchmarks/filter?model=gpt-4o-mini&provider=openai&start_time=2024-07-12T04:20:32.808410&end_time=2024-08-12T04:20:32.808410®ime=concurrent-1®ion=Belgium&seq_len=short' \ - --header 'Authorization: Bearer ' -``` - -```python Python -import requests - -url = "https://api.unify.ai/v0/benchmarks/filter?model=gpt-4o-mini&provider=openai&start_time=2024-07-12T04:20:32.808410&end_time=2024-08-12T04:20:32.808410®ime=concurrent-1®ion=Belgium&seq_len=short" - -headers = {"Authorization": "Bearer "} - -response = requests.request("POST", url, headers=headers) - -print(response.text) -``` - - - - -```json 200 -{} -``` - -```json 422 -{ - "detail": [ - { - "loc": [ - "string" - ], - "msg": "string", - "type": "string" - } - ] -} -``` - - diff --git a/api-reference/benchmarks/get_latest_benchmark.mdx b/api-reference/benchmarks/get_latest_benchmark.mdx deleted file mode 100644 index 049f83641..000000000 --- a/api-reference/benchmarks/get_latest_benchmark.mdx +++ /dev/null @@ -1,76 +0,0 @@ ---- -title: 'Get Latest Benchmark' -api: 'GET /v0/benchmarks' ---- - - -#### Authorizations - - - Bearer authentication header of the form `Bearer `, where `` is your auth token. - - -#### Query Parameters - - -Model name - - - -Provider name - - - - - - - -Region where the benchmark is run. Options are: "Belgium", "Hong Kong", "Iowa". - - - -Length of the sequence used for benchmarking, can be short or long - - - - -```bash cURL -curl --request GET \ - --url 'https://api.unify.ai/v0/benchmarks?model=gpt-4o-mini&provider=openai®ime=concurrent-1®ion=Belgium&seq_len=short' \ - --header 'Authorization: Bearer ' -``` - -```python Python -import requests - -url = "https://api.unify.ai/v0/benchmarks?model=gpt-4o-mini&provider=openai®ime=concurrent-1®ion=Belgium&seq_len=short" - -headers = {"Authorization": "Bearer "} - -response = requests.request("GET", url, headers=headers) - -print(response.text) -``` - - - - -```json 200 -{} -``` - -```json 422 -{ - "detail": [ - { - "loc": [ - "string" - ], - "msg": "string", - "type": "string" - } - ] -} -``` - - diff --git a/api-reference/user/get_credits.mdx b/api-reference/credits/get_credits.mdx similarity index 84% rename from api-reference/user/get_credits.mdx rename to api-reference/credits/get_credits.mdx index 8121e45c9..a78a44338 100644 --- a/api-reference/user/get_credits.mdx +++ b/api-reference/credits/get_credits.mdx @@ -1,6 +1,6 @@ --- title: 'Get Credits' -api: 'GET /v0/get_credits' +api: 'GET /v0/credits' --- Returns the number of available credits. @@ -14,14 +14,14 @@ Returns the number of available credits. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/get_credits' \ + --url 'https://api.unify.ai/v0/credits' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/get_credits" +url = "https://api.unify.ai/v0/credits" headers = {"Authorization": "Bearer "} diff --git a/api-reference/user/credits_code.mdx b/api-reference/credits/promo_code.mdx similarity index 91% rename from api-reference/user/credits_code.mdx rename to api-reference/credits/promo_code.mdx index ab425115f..39d189843 100644 --- a/api-reference/user/credits_code.mdx +++ b/api-reference/credits/promo_code.mdx @@ -1,5 +1,5 @@ --- -title: 'Credits Code' +title: 'Promo Code' api: 'POST /v0/promo' --- Activates a promotional code. @@ -16,7 +16,7 @@ Activates a promotional code. Promo code to be activated. - + ID of the user that receives the credits,defaults to the user making the request. @@ -45,7 +45,7 @@ print(response.text) ```json 200 { - "info": "Code {code} activated succesfully!" + "info": "Code {code} activated successfully!" } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/create_custom_api_key.mdx b/api-reference/custom_api_keys/create_custom_api_key.mdx similarity index 52% rename from api-reference/custom_endpoints_and_api_keys/create_custom_api_key.mdx rename to api-reference/custom_api_keys/create_custom_api_key.mdx index 84f86f367..b5f034cea 100644 --- a/api-reference/custom_endpoints_and_api_keys/create_custom_api_key.mdx +++ b/api-reference/custom_api_keys/create_custom_api_key.mdx @@ -1,13 +1,17 @@ --- title: 'Create Custom API Key' -api: 'PUT /v0/custom_api_key' +api: 'POST /v0/custom_api_key' --- -Stores a custom API key from a LLM provider in your account. This can be used in two ways: -1. As part of a custom endpoint. If you define a custom endpoint, you can reference a custom API -key. This will be sent to the endpoint as part of the request. -2. To use your own API keys in standard providers. If any of your custom API keys matches a provider -name and you pass `use_custom_keys=True` to the `/chat/completions` endpoint, this API key will -be used, charging your account directly instead of consuming Unify credits. +Stores a custom API key from an LLM provider in your account. This can be done in +one of two ways: + +1. As part of a custom endpoint. If you define a custom endpoint, you can reference +a custom API key. This will be sent to the endpoint as part of the request. + +2. To use your own API keys with the standard providers. If any of your custom API +keys match a provider name and you pass `use_custom_keys=True` to the +`/chat/completions` endpoint, then this API key will be used, using your own +account with the provider directly. #### Authorizations @@ -17,7 +21,7 @@ be used, charging your account directly instead of consuming Unify credits. #### Query Parameters - + Name of the API key. @@ -28,19 +32,19 @@ Value of the API key. ```bash cURL -curl --request PUT \ - --url 'https://api.unify.ai/v0/custom_api_key?key=key1&value=value1' \ +curl --request POST \ + --url 'https://api.unify.ai/v0/custom_api_key?name=key1&value=value1' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_api_key?key=key1&value=value1" +url = "https://api.unify.ai/v0/custom_api_key?name=key1&value=value1" headers = {"Authorization": "Bearer "} -response = requests.request("PUT", url, headers=headers) +response = requests.request("POST", url, headers=headers) print(response.text) ``` @@ -50,7 +54,7 @@ print(response.text) ```json 200 { - "info": "API key created succesfully!" + "info": "API key created successfully!" } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/delete_custom_api_key.mdx b/api-reference/custom_api_keys/delete_custom_api_key.mdx similarity index 78% rename from api-reference/custom_endpoints_and_api_keys/delete_custom_api_key.mdx rename to api-reference/custom_api_keys/delete_custom_api_key.mdx index 79d9cf9be..15e2dc3cc 100644 --- a/api-reference/custom_endpoints_and_api_keys/delete_custom_api_key.mdx +++ b/api-reference/custom_api_keys/delete_custom_api_key.mdx @@ -2,7 +2,7 @@ title: 'Delete Custom API Key' api: 'DELETE /v0/custom_api_key' --- -Deletes a custom API key in your account. +Deletes the custom API key from your account. #### Authorizations @@ -12,7 +12,7 @@ Deletes a custom API key in your account. #### Query Parameters - + Name of the custom API key to delete. @@ -20,14 +20,14 @@ Name of the custom API key to delete. ```bash cURL curl --request DELETE \ - --url 'https://api.unify.ai/v0/custom_api_key?key=key1' \ + --url 'https://api.unify.ai/v0/custom_api_key?name=key1' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_api_key?key=key1" +url = "https://api.unify.ai/v0/custom_api_key?name=key1" headers = {"Authorization": "Bearer "} @@ -41,7 +41,7 @@ print(response.text) ```json 200 { - "info": "API key deleted succesfully!" + "info": "API key deleted successfully!" } ``` diff --git a/api-reference/dataset_evaluation/return_eval_config.mdx b/api-reference/custom_api_keys/get_custom_api_key.mdx similarity index 66% rename from api-reference/dataset_evaluation/return_eval_config.mdx rename to api-reference/custom_api_keys/get_custom_api_key.mdx index 59596525f..7c9c3119c 100644 --- a/api-reference/dataset_evaluation/return_eval_config.mdx +++ b/api-reference/custom_api_keys/get_custom_api_key.mdx @@ -1,8 +1,8 @@ --- -title: 'Return Eval Config' -api: 'GET /v0/evals/get_config' +title: 'Get Custom API Key' +api: 'GET /v0/custom_api_key' --- -Returns the configuration JSON for a named eval. +Returns the value of the key for the specified custom API key name. #### Authorizations @@ -12,22 +12,22 @@ Returns the configuration JSON for a named eval. #### Query Parameters - -Name of the eval to return the configuration of + +Name of the API key to get the value for. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/evals/get_config?eval_name=eval1' \ + --url 'https://api.unify.ai/v0/custom_api_key?name=key1' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/evals/get_config?eval_name=eval1" +url = "https://api.unify.ai/v0/custom_api_key?name=key1" headers = {"Authorization": "Bearer "} @@ -40,7 +40,10 @@ print(response.text) ```json 200 -{} +{ + "key": "string", + "value": "string" +} ``` ```json 422 diff --git a/api-reference/custom_endpoints_and_api_keys/get_custom_api_keys.mdx b/api-reference/custom_api_keys/list_custom_api_keys.mdx similarity index 71% rename from api-reference/custom_endpoints_and_api_keys/get_custom_api_keys.mdx rename to api-reference/custom_api_keys/list_custom_api_keys.mdx index b05e0ecb3..74db319d9 100644 --- a/api-reference/custom_endpoints_and_api_keys/get_custom_api_keys.mdx +++ b/api-reference/custom_api_keys/list_custom_api_keys.mdx @@ -1,8 +1,8 @@ --- -title: 'Get Custom API Keys' -api: 'GET /v0/custom_api_key' +title: 'List Custom API Keys' +api: 'GET /v0/custom_api_key/list' --- -Returns a list of the available custom API keys. +Returns a list of the names for all custom API keys in your account. #### Authorizations @@ -14,14 +14,14 @@ Returns a list of the available custom API keys. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/custom_api_key' \ + --url 'https://api.unify.ai/v0/custom_api_key/list' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_api_key" +url = "https://api.unify.ai/v0/custom_api_key/list" headers = {"Authorization": "Bearer "} diff --git a/api-reference/custom_endpoints_and_api_keys/rename_custom_api_key.mdx b/api-reference/custom_api_keys/rename_custom_api_key.mdx similarity index 72% rename from api-reference/custom_endpoints_and_api_keys/rename_custom_api_key.mdx rename to api-reference/custom_api_keys/rename_custom_api_key.mdx index 5510967bf..919bf6d0f 100644 --- a/api-reference/custom_endpoints_and_api_keys/rename_custom_api_key.mdx +++ b/api-reference/custom_api_keys/rename_custom_api_key.mdx @@ -2,7 +2,7 @@ title: 'Rename Custom API Key' api: 'POST /v0/custom_api_key/rename' --- -Renames a custom API key from a LLM provider in your account. +Renames the custom API key in your account. #### Authorizations @@ -12,11 +12,11 @@ Renames a custom API key from a LLM provider in your account. #### Query Parameters - + Name of the custom API key to be updated. - + New name for the custom API key. @@ -24,14 +24,14 @@ New name for the custom API key. ```bash cURL curl --request POST \ - --url 'https://api.unify.ai/v0/custom_api_key/rename?key=key1&new_key=key2' \ + --url 'https://api.unify.ai/v0/custom_api_key/rename?name=key1&new_name=key2' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_api_key/rename?key=key1&new_key=key2" +url = "https://api.unify.ai/v0/custom_api_key/rename?name=key1&new_name=key2" headers = {"Authorization": "Bearer "} @@ -45,7 +45,7 @@ print(response.text) ```json 200 { - "info": "API key renamed succesfully!" + "info": "API key renamed successfully!" } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/create_custom_endpoint.mdx b/api-reference/custom_endpoints/create_custom_endpoint.mdx similarity index 52% rename from api-reference/custom_endpoints_and_api_keys/create_custom_endpoint.mdx rename to api-reference/custom_endpoints/create_custom_endpoint.mdx index c3f13fb0e..7390e9bdd 100644 --- a/api-reference/custom_endpoints_and_api_keys/create_custom_endpoint.mdx +++ b/api-reference/custom_endpoints/create_custom_endpoint.mdx @@ -1,10 +1,14 @@ --- title: 'Create Custom Endpoint' -api: 'PUT /v0/custom_endpoint' +api: 'POST /v0/custom_endpoint' --- -Creates a custom endpoint. This endpoint must support the OpenAI `/chat/completions` -format. To query your custom endpoint, replace your endpoint string with `@custom` -when querying the unified API. +Creates a custom endpoint. This endpoint must either be a fine-tuned model from one +of the supported providers (`/v0/providers`), in which case the "provider" argument +must be set accordingly. Otherwise, the endpoint must support the OpenAI +`/chat/completions` format. To query your custom endpoint, replace your endpoint +string with `@custom` when querying any general custom endpoint. You can show +all *custom* endpoints by querying `/v0/endpoints` and passing `custom` as the +`provider` argument. #### Authorizations @@ -26,26 +30,30 @@ Base URL of the endpoint being called. Must support the OpenAI format. Name of the API key that will be passed as part of the query. - + Named passed to the custom endpoint as model name. If not specified, it will default to the endpoint alias. + +If the custom endpoint is for a fine-tuned model which is hosted directly via one of the supported providers, then this argument should be specified as the provider used. The unification logic will be used for this custom fine-tuned model behind the scenes, in the same manner as used for the foundation models with the same provider. + + ```bash cURL -curl --request PUT \ - --url 'https://api.unify.ai/v0/custom_endpoint?name=endpoint1&url=https://api.url1.com&key_name=key1&mdl_name=llama-3.1-8b-finetuned' \ +curl --request POST \ + --url 'https://api.unify.ai/v0/custom_endpoint?name=endpoint1&url=https://api.url1.com&key_name=key1&model_name=llama-3.1-8b-finetuned&provider=together-ai' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_endpoint?name=endpoint1&url=https://api.url1.com&key_name=key1&mdl_name=llama-3.1-8b-finetuned" +url = "https://api.unify.ai/v0/custom_endpoint?name=endpoint1&url=https://api.url1.com&key_name=key1&model_name=llama-3.1-8b-finetuned&provider=together-ai" headers = {"Authorization": "Bearer "} -response = requests.request("PUT", url, headers=headers) +response = requests.request("POST", url, headers=headers) print(response.text) ``` @@ -55,7 +63,7 @@ print(response.text) ```json 200 { - "info": "Custom endpoint created succesfully!" + "info": "Custom endpoint created successfully!" } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/delete_custom_endpoint.mdx b/api-reference/custom_endpoints/delete_custom_endpoint.mdx similarity index 91% rename from api-reference/custom_endpoints_and_api_keys/delete_custom_endpoint.mdx rename to api-reference/custom_endpoints/delete_custom_endpoint.mdx index 91fb2abfd..7d34dfc03 100644 --- a/api-reference/custom_endpoints_and_api_keys/delete_custom_endpoint.mdx +++ b/api-reference/custom_endpoints/delete_custom_endpoint.mdx @@ -2,7 +2,7 @@ title: 'Delete Custom Endpoint' api: 'DELETE /v0/custom_endpoint' --- -Deletes a custom endpoint in your account. +Deletes a custom endpoint from your account. #### Authorizations @@ -41,7 +41,7 @@ print(response.text) ```json 200 { - "info": "Custom endpoint deleted succesfully!" + "info": "Custom endpoint deleted successfully!" } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/rename_custom_endpoint.mdx b/api-reference/custom_endpoints/rename_custom_endpoint.mdx similarity index 96% rename from api-reference/custom_endpoints_and_api_keys/rename_custom_endpoint.mdx rename to api-reference/custom_endpoints/rename_custom_endpoint.mdx index a57ca88eb..d251ffbe0 100644 --- a/api-reference/custom_endpoints_and_api_keys/rename_custom_endpoint.mdx +++ b/api-reference/custom_endpoints/rename_custom_endpoint.mdx @@ -45,7 +45,7 @@ print(response.text) ```json 200 { - "info": "Custom endpoint renamed succesfully!" + "info": "Custom endpoint renamed successfully!" } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/get_custom_benchmarks.mdx b/api-reference/custom_endpoints_and_api_keys/get_custom_benchmarks.mdx deleted file mode 100644 index 7a2248771..000000000 --- a/api-reference/custom_endpoints_and_api_keys/get_custom_benchmarks.mdx +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: 'Get Custom Benchmarks' -api: 'GET /v0/custom_endpoint/get_benchmark' ---- - - -#### Authorizations - - - Bearer authentication header of the form `Bearer `, where `` is your auth token. - - -#### Query Parameters - - -Name of the custom endpoint to get a benchmark for. - - - -Name of the metric to get the benchmark of. - - - -Start time of window to get benchmarks between. Format YYYY-MM-DD - - - -End time of window to get benchmarks between. Format YYYY-MM-DD - - - - -```bash cURL -curl --request GET \ - --url 'https://api.unify.ai/v0/custom_endpoint/get_benchmark?endpoint_name=endpoint1&metric_name=tokens-per-second&start_time=2024-01-01&end_time=2024-12-12' \ - --header 'Authorization: Bearer ' -``` - -```python Python -import requests - -url = "https://api.unify.ai/v0/custom_endpoint/get_benchmark?endpoint_name=endpoint1&metric_name=tokens-per-second&start_time=2024-01-01&end_time=2024-12-12" - -headers = {"Authorization": "Bearer "} - -response = requests.request("GET", url, headers=headers) - -print(response.text) -``` - - - - -```json 200 -{} -``` - -```json 422 -{ - "detail": [ - { - "loc": [ - "string" - ], - "msg": "string", - "type": "string" - } - ] -} -``` - - diff --git a/api-reference/dataset/delete_dataset.mdx b/api-reference/datasets/delete_dataset.mdx similarity index 88% rename from api-reference/dataset/delete_dataset.mdx rename to api-reference/datasets/delete_dataset.mdx index 4b39ba8ce..a908eb51d 100644 --- a/api-reference/dataset/delete_dataset.mdx +++ b/api-reference/datasets/delete_dataset.mdx @@ -2,7 +2,7 @@ title: 'Delete Dataset' api: 'DELETE /v0/dataset' --- -Deletes a previously updated dataset and any relevant artifacts from the platform. +Deletes a previously updated dataset and any relevant artifacts from your account. #### Authorizations @@ -41,13 +41,13 @@ print(response.text) ```json 200 { - "info": "Dataset deleted succesfully!" + "info": "Dataset deleted successfully!" } ``` ```json 400 { - "detail": "Invalid name for a dataset. Please, choose a different one." + "detail": "Invalid name for a dataset.Please, choose a different one." } ``` diff --git a/api-reference/dataset/download_dataset.mdx b/api-reference/datasets/download_dataset.mdx similarity index 96% rename from api-reference/dataset/download_dataset.mdx rename to api-reference/datasets/download_dataset.mdx index 6fa075f0a..3258d6f18 100644 --- a/api-reference/dataset/download_dataset.mdx +++ b/api-reference/datasets/download_dataset.mdx @@ -2,7 +2,7 @@ title: 'Download Dataset' api: 'GET /v0/dataset' --- -Downloads a specific dataset from the platform. +Downloads a specific dataset from your account. #### Authorizations diff --git a/api-reference/dataset/list_datasets.mdx b/api-reference/datasets/list_datasets.mdx similarity index 91% rename from api-reference/dataset/list_datasets.mdx rename to api-reference/datasets/list_datasets.mdx index b3344b058..e285df913 100644 --- a/api-reference/dataset/list_datasets.mdx +++ b/api-reference/datasets/list_datasets.mdx @@ -2,7 +2,7 @@ title: 'List Datasets' api: 'GET /v0/dataset/list' --- -Lists all the custom datasets uploaded by the user to the platform. +Lists all the datasets stored in the user account by name. #### Authorizations diff --git a/api-reference/dataset/rename_dataset.mdx b/api-reference/datasets/rename_dataset.mdx similarity index 90% rename from api-reference/dataset/rename_dataset.mdx rename to api-reference/datasets/rename_dataset.mdx index 4c1df91fe..151156282 100644 --- a/api-reference/dataset/rename_dataset.mdx +++ b/api-reference/datasets/rename_dataset.mdx @@ -2,7 +2,7 @@ title: 'Rename Dataset' api: 'POST /v0/dataset/rename' --- -Renames a previously updated dataset. +Renames a previously uploaded dataset. #### Authorizations @@ -51,7 +51,7 @@ print(response.text) ```json 400 { - "detail": "A dataset with this name already exists. Please, choose a different one." + "detail": "A dataset with this name already exists.Please, choose a different one." } ``` diff --git a/api-reference/dataset/upload_dataset.mdx b/api-reference/datasets/upload_dataset.mdx similarity index 78% rename from api-reference/dataset/upload_dataset.mdx rename to api-reference/datasets/upload_dataset.mdx index e40346c3a..0de838442 100644 --- a/api-reference/dataset/upload_dataset.mdx +++ b/api-reference/datasets/upload_dataset.mdx @@ -2,9 +2,10 @@ title: 'Upload Dataset' api: 'POST /v0/dataset' --- -Uploads a custom dataset to the platform. +Uploads a custom dataset to your account. -The uploaded file must be a JSONL file with **at least** a `prompt` key: +The uploaded file must be a JSONL file with **at least** a `prompt` key for +each prompt each: ``` {"prompt": "This is the first prompt"} @@ -12,8 +13,9 @@ The uploaded file must be a JSONL file with **at least** a `prompt` key: {"prompt": "This is the third prompt"} ``` -Additionally, you can include a `ref_answer` key, which will be accounted -during the evaluations. +Additionally, you can include any extra keys as desired, depending on the use case +for the dataset, and how it will be used by the evaluators and/or router training. +For example, you could include a reference answer to each prompt as follows: ``` {"prompt": "This is the first prompt", "ref_answer": "First reference answer"} @@ -30,11 +32,11 @@ during the evaluations. #### Body - +The contents of the `.jsonl` file being uploaded. - +The name to give to this dataset. @@ -76,7 +78,7 @@ print(response.text) ```json 400 { - "detail": "A dataset with this name already exists. Please, choose a different one." + "detail": "A dataset with this name already exists.Please, choose a different one." } ``` diff --git a/api-reference/custom_endpoints_and_api_keys/upload_custom_benchmark.mdx b/api-reference/efficiency_benchmarks/append_to_benchmark.mdx similarity index 54% rename from api-reference/custom_endpoints_and_api_keys/upload_custom_benchmark.mdx rename to api-reference/efficiency_benchmarks/append_to_benchmark.mdx index 84f46396c..eb8cfa07c 100644 --- a/api-reference/custom_endpoints_and_api_keys/upload_custom_benchmark.mdx +++ b/api-reference/efficiency_benchmarks/append_to_benchmark.mdx @@ -1,8 +1,9 @@ --- -title: 'Upload Custom Benchmark' -api: 'POST /v0/custom_endpoint/benchmark' +title: 'Append To Benchmark' +api: 'POST /v0/benchmark' --- - +Append speed or cost data to the standardized time-series benchmarks for a custom +endpoint (only custom endpoints are publishable by end users). #### Authorizations @@ -13,29 +14,34 @@ api: 'POST /v0/custom_endpoint/benchmark' #### Query Parameters -Name of the custom endpoint to submit a benchmark for. +Name of the *custom* endpoint to append benchmark data for. -Name of the metric to submit. Allowed metrics are: "input-cost", "output-cost", "tokens-per-second", "time-to-first-token", "inter-token-latency", "end-2-end-latency", "cold-start". +Name of the metric to submit. Allowed metrics are: + "input-cost", "output-cost", "time-to-first-token", "inter-token-latency". Value of the metric to submit. + +The timestamp to associate with the submission. Defaults to current time if unspecified. + + ```bash cURL curl --request POST \ - --url 'https://api.unify.ai/v0/custom_endpoint/benchmark?endpoint_name=endpoint1&metric_name=tokens-per-second&value=10' \ + --url 'https://api.unify.ai/v0/benchmark?endpoint_name=my_endpoint&metric_name=tokens-per-second&value=10&measured_at=2024-08-12T04:20:32.808410' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_endpoint/benchmark?endpoint_name=endpoint1&metric_name=tokens-per-second&value=10" +url = "https://api.unify.ai/v0/benchmark?endpoint_name=my_endpoint&metric_name=tokens-per-second&value=10&measured_at=2024-08-12T04:20:32.808410" headers = {"Authorization": "Bearer "} @@ -49,7 +55,7 @@ print(response.text) ```json 200 { - "info": "Custom endpoint benchmark uploaded succesfully!" + "info": "Custom endpoint benchmark uploaded successfully!" } ``` diff --git a/api-reference/efficiency_benchmarks/delete_benchmark.mdx b/api-reference/efficiency_benchmarks/delete_benchmark.mdx new file mode 100644 index 000000000..c7bc24f09 --- /dev/null +++ b/api-reference/efficiency_benchmarks/delete_benchmark.mdx @@ -0,0 +1,61 @@ +--- +title: 'Delete Benchmark' +api: 'DELETE /v0/benchmark' +--- +Delete *all* benchmark time-series data for a given *custom* endpoint. +The time-series benchmark data for *public* endpoints are not deletable. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +Name of the *custom* endpoint to submit a benchmark for. + + + + +```bash cURL +curl --request DELETE \ + --url 'https://api.unify.ai/v0/benchmark?endpoint_name=my_endpoint' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/benchmark?endpoint_name=my_endpoint" + +headers = {"Authorization": "Bearer "} + +response = requests.request("DELETE", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/efficiency_benchmarks/get_benchmark.mdx b/api-reference/efficiency_benchmarks/get_benchmark.mdx new file mode 100644 index 000000000..10cb4f312 --- /dev/null +++ b/api-reference/efficiency_benchmarks/get_benchmark.mdx @@ -0,0 +1,98 @@ +--- +title: 'Get Benchmark' +api: 'GET /v0/benchmark' +--- +Extracts cost and speed data for the provided endpoint via our standardized +efficiency benchmarks, in the specified region, with the specified sequence length, +with all benchmark values returned within the specified time window. + +When extracting data for a *custom* endpoint, then `model` is the endpoint name, and +`provider` must be set as `"custom"`. The arguments `region` and `seq_len` are +ignored for custom endpoints (they are not publishable). + +If neither `start_time` nor `end_time` are provided, then only the *latest* +benchmark data is returned. If only `start_time` is provided, then `end_time` is +assumed to be the current time. An exception is raised if only `end_time` is +provided. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +Name of the model. + + + +Name of the provider. + + + +Region where the benchmark is run. + Options are: `"Belgium"`, `"Hong Kong"` or `"Iowa"`. + + + +Length of the sequence used for benchmarking, can be short or long + + + +Window start time. Only returns the latest benchmark if unspecified + + + +Window end time. Assumed to be the current time if this is unspecified *and* start_time *is* specified. Only the latest benchmark is returned if both are unspecified. + + + + +```bash cURL +curl --request GET \ + --url 'https://api.unify.ai/v0/benchmark?model=gpt-4o-mini&provider=openai®ion=Belgium&seq_len=short&start_time=2024-07-12T04:20:32.808410&end_time=2024-08-12T04:20:32.808410' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/benchmark?model=gpt-4o-mini&provider=openai®ion=Belgium&seq_len=short&start_time=2024-07-12T04:20:32.808410&end_time=2024-08-12T04:20:32.808410" + +headers = {"Authorization": "Bearer "} + +response = requests.request("GET", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{ + "ttft": 440.2323939998496, + "itl": 8.797065147959705, + "input_cost": 0.15, + "output_cost": 0.6, + "measured_at": "2024-08-17T19:19:37.289937" +} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/evaluations/delete_evaluations.mdx b/api-reference/evaluations/delete_evaluations.mdx new file mode 100644 index 000000000..be5086a53 --- /dev/null +++ b/api-reference/evaluations/delete_evaluations.mdx @@ -0,0 +1,71 @@ +--- +title: 'Delete Evaluations' +api: 'DELETE /v0/evaluation' +--- +Deletes evaluations on a given dataset, for a specific endpoint (optional) based on +a specific evaluator (optional). If no `evaluator` is provided, then evaluations for +all valid evaluators are deleted. Similarly, if no `endpoint` is provided, then +evaluations for all valid endpoints are deleted. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +Name of the dataset to delete the evaluation for. + + + +The endpoint to delete the evaluation for. If `None`, deletes the evaluations for all endpoints. + + + +Name of the evaluator to delete the evaluation for. If `None`, deletes all available evaluations for the dataset and endpoint pair. + + + + +```bash cURL +curl --request DELETE \ + --url 'https://api.unify.ai/v0/evaluation?dataset=dataset1&endpoint=gpt-4o-mini@openai&evaluator=eval1' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/evaluation?dataset=dataset1&endpoint=gpt-4o-mini@openai&evaluator=eval1" + +headers = {"Authorization": "Bearer "} + +response = requests.request("DELETE", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/dataset_evaluation/eval_status.mdx b/api-reference/evaluations/eval_status.mdx similarity index 79% rename from api-reference/dataset_evaluation/eval_status.mdx rename to api-reference/evaluations/eval_status.mdx index d24da6e6a..23efd1b03 100644 --- a/api-reference/dataset_evaluation/eval_status.mdx +++ b/api-reference/evaluations/eval_status.mdx @@ -1,10 +1,8 @@ --- title: 'Eval Status' -api: 'GET /v0/evals/status' +api: 'GET /v0/evaluation/status' --- Fetches the eval status on a given dataset. Returns object of the form: - -``` { "responses": { "last_updated": "2024-08-19 13:58:20.866092", @@ -21,7 +19,6 @@ Fetches the eval status on a given dataset. Returns object of the form: } }, } -``` #### Authorizations @@ -39,22 +36,22 @@ Name of the dataset to get evaluation status of. Endpoint to get evaluation status of - -Name of the eval to get evaluation status of. + +Name of the evaluator to get status of. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/evals/status?dataset=dataset1&endpoint=llama-3-8b-chat@aws-bedrock&eval_name=eval1' \ + --url 'https://api.unify.ai/v0/evaluation/status?dataset=dataset1&endpoint=llama-3-8b-chat@aws-bedrock&evaluator=eval1' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/evals/status?dataset=dataset1&endpoint=llama-3-8b-chat@aws-bedrock&eval_name=eval1" +url = "https://api.unify.ai/v0/evaluation/status?dataset=dataset1&endpoint=llama-3-8b-chat@aws-bedrock&evaluator=eval1" headers = {"Authorization": "Bearer "} diff --git a/api-reference/dataset_evaluation/get_eval_scores.mdx b/api-reference/evaluations/get_evaluations.mdx similarity index 54% rename from api-reference/dataset_evaluation/get_eval_scores.mdx rename to api-reference/evaluations/get_evaluations.mdx index 8236230a6..efd8def4a 100644 --- a/api-reference/dataset_evaluation/get_eval_scores.mdx +++ b/api-reference/evaluations/get_evaluations.mdx @@ -1,8 +1,11 @@ --- -title: 'Get Eval Scores' -api: 'GET /v0/evals/get_scores' +title: 'Get Evaluations' +api: 'GET /v0/evaluation' --- -Fetches the results of an eval on a given dataset. If no `eval_name` is provided, returns scores for all completed evals on that dataset. +Fetches evaluation results on a given dataset, for a specific endpoint (optional) +based on a specific evaluator (optional). If no `evaluator` is provided, then scores +are returned for all valid evaluators. Similarly, if no `endpoint` is provided, then +scores are returned for all valid endpoints. #### Authorizations @@ -16,12 +19,12 @@ Fetches the results of an eval on a given dataset. If no `eval_name` is provided Name of the dataset to fetch evaluation from. - -Name of the eval to fetch evaluation from. If `None`, returns all available evaluations for the dataset. + +The endpoint to fetch the evaluation for. If `None`, returns evaluations for all endpoints. - -Name of endpoint to fetch eval from. If not set returns evals for all endpoints + +Name of the evaluator to fetch the evaluation for. If `None`, returns all available evaluations for the dataset and endpoint pair. @@ -32,14 +35,14 @@ If `True`, returns the scores on a per-prompt level. By default set to `False`. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/evals/get_scores?dataset=dataset1&eval_name=eval1&endpoint=None&per_prompt=False' \ + --url 'https://api.unify.ai/v0/evaluation?dataset=dataset1&endpoint=gpt-4o-mini@openai&evaluator=eval1&per_prompt=False' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/evals/get_scores?dataset=dataset1&eval_name=eval1&endpoint=None&per_prompt=False" +url = "https://api.unify.ai/v0/evaluation?dataset=dataset1&endpoint=gpt-4o-mini@openai&evaluator=eval1&per_prompt=False" headers = {"Authorization": "Bearer "} diff --git a/api-reference/dataset_evaluation/trigger_eval.mdx b/api-reference/evaluations/trigger_evaluation.mdx similarity index 69% rename from api-reference/dataset_evaluation/trigger_eval.mdx rename to api-reference/evaluations/trigger_evaluation.mdx index f0b57f88f..190f1ce45 100644 --- a/api-reference/dataset_evaluation/trigger_eval.mdx +++ b/api-reference/evaluations/trigger_evaluation.mdx @@ -1,10 +1,12 @@ --- -title: 'Trigger Eval' -api: 'POST /v0/evals/trigger' +title: 'Trigger Evaluation' +api: 'POST /v0/evaluation' --- -Uses the named `eval` to begin an evaluation of quality scores for the selected LLM `endpoint`, on the given `dataset`. -You can upload custom scores (and bypass the LLM judge) by uploading a file via the `client_side_scores` argument. -Once the evaluation has finished, you can access the scores using the `evals/get_scores` endpoint. +Uses the named `evaluator` to trigger an evaluation of quality scores for the +selected LLM `endpoint` on the selected `dataset`. You can upload custom scores (and +bypass the LLM judge entirely) by uploading a file via the `client_side_scores` +argument. Once the evaluation has finished, you can access the scores using the +`/v0/evaluation` endpoint. #### Authorizations @@ -14,8 +16,8 @@ Once the evaluation has finished, you can access the scores using the `evals/get #### Query Parameters - -Name of the eval to use. + +Name of the evaluator to use. @@ -28,15 +30,15 @@ Name of the endpoint to evaluate. Endpoints must be specified using the `model@p #### Body - -An optional file upload for client-side scores. The file must be in JSONL format and the prompts must match the order of the `dataset`. Each entry should include `prompt` and `score` keys, with `score` being a float between 0 and 1. The evaluation corresponding to the `eval_name` must have `client_side=True`. + +An optional file upload for client-side scores. The file must be in JSONL format and the prompts must match the order of the `dataset`. Each entry should include `prompt` and `score` keys, with `score` being a float between 0 and 1. The evaluation corresponding to the `evaluator` must have `client_side=True`. ```bash cURL curl --request POST \ - --url 'https://api.unify.ai/v0/evals/trigger?eval_name=eval1&dataset=dataset1&endpoint=gpt-4o-mini@openai' \ + --url 'https://api.unify.ai/v0/evaluation?evaluator=eval1&dataset=dataset1&endpoint=gpt-4o-mini@openai' \ --header 'Authorization: Bearer ' --header 'Content-Type: multipart/form-data' \ --form 'client_side_scores=@client_scores.jsonl' @@ -45,7 +47,7 @@ curl --request POST \ ```python Python import requests -url = "https://api.unify.ai/v0/evals/trigger?eval_name=eval1&dataset=dataset1&endpoint=gpt-4o-mini@openai" +url = "https://api.unify.ai/v0/evaluation?evaluator=eval1&dataset=dataset1&endpoint=gpt-4o-mini@openai" headers = {"Authorization": "Bearer "} diff --git a/api-reference/dataset_evaluation/create_eval.mdx b/api-reference/evaluators/create_evaluator.mdx similarity index 73% rename from api-reference/dataset_evaluation/create_eval.mdx rename to api-reference/evaluators/create_evaluator.mdx index 7b8bb5003..76deed8ee 100644 --- a/api-reference/dataset_evaluation/create_eval.mdx +++ b/api-reference/evaluators/create_evaluator.mdx @@ -1,8 +1,9 @@ --- -title: 'Create Eval' -api: 'POST /v0/evals/create' +title: 'Create Evaluator' +api: 'POST /v0/evaluator' --- -Create a re-useable, named eval configuration. This can be used to trigger an evaluation via the `/evals/trigger` endpoint. +Create a re-usable, named evaluator, and adds this to your account. This can be used +to trigger an evaluation via `POST` requests to the `/evaluation` endpoint. #### Authorizations @@ -12,8 +13,8 @@ Create a re-useable, named eval configuration. This can be used to trigger an ev #### Body - -A unique, user-defined name used when referencing and triggering the eval. + +A unique, user-defined name used when referencing and triggering the evaluation. @@ -21,17 +22,20 @@ An optional custom system prompt to provide specific instructions to the judge o -If set, describes the list of classifications that the LLM judge uses to score each prompt. For example: +If set, describes the list of classifications that the LLM judge uses to + score each prompt. For example: ``` -[{"label": "Excellent", "score": 1.0, "description": "A perfect answer with no factual mistakes"}, +[{"label": "Excellent", "score": 1.0, "description": "A perfect answer with no factual +mistakes"}, {"label": "Good", "score": 0.5, "description": "An average answer"}, -{"label": "Bad", "score": 0.0, "description": "An incorrect answer, containing a significant factual mistake"}] +{"label": "Bad", "score": 0.0, "description": "An incorrect answer, containing a +significant factual mistake"}] ``` -Specifies the LLM(s) to be used as the judge. This can be a string containining a single model name or a list of model names. +Specifies the LLM(s) to be used as the judge. This can be a string containing a single model name or a list of model names. @@ -42,11 +46,11 @@ Indicates whether evaluations are performed on the client-side. If `True`, the L ```bash cURL curl --request POST \ - --url 'https://api.unify.ai/v0/evals/create' \ + --url 'https://api.unify.ai/v0/evaluator' \ --header 'Authorization: Bearer ' \ --header 'Content-Type: application/json' \ --data '{ - "eval_name": "eval1", + "name": "eval1", "judge_models": "claude-3.5-sonnet@aws-bedrock" }' ``` @@ -54,11 +58,11 @@ curl --request POST \ ```python Python import requests -url = "https://api.unify.ai/v0/evals/create" +url = "https://api.unify.ai/v0/evaluator" headers = {"Authorization": "Bearer "} -json_input = {"eval_name": "eval1", "judge_models": "claude-3.5-sonnet@aws-bedrock"} +json_input = {"name": "eval1", "judge_models": "claude-3.5-sonnet@aws-bedrock"} response = requests.request("POST", url, json=json_input, headers=headers) diff --git a/api-reference/dataset_evaluation/delete_eval.mdx b/api-reference/evaluators/delete_evaluator.mdx similarity index 71% rename from api-reference/dataset_evaluation/delete_eval.mdx rename to api-reference/evaluators/delete_evaluator.mdx index f93c6ae69..b2eb4bcac 100644 --- a/api-reference/dataset_evaluation/delete_eval.mdx +++ b/api-reference/evaluators/delete_evaluator.mdx @@ -1,8 +1,8 @@ --- -title: 'Delete Eval' -api: 'DELETE /v0/evals/delete' +title: 'Delete Evaluator' +api: 'DELETE /v0/evaluator' --- -Deletes a named eval from your account. +Deletes an evaluator from your account. #### Authorizations @@ -12,22 +12,22 @@ Deletes a named eval from your account. #### Query Parameters - -Name of the eval to delete + +Name of the evaluator to delete. ```bash cURL curl --request DELETE \ - --url 'https://api.unify.ai/v0/evals/delete?eval_name=eval1' \ + --url 'https://api.unify.ai/v0/evaluator?name=eval1' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/evals/delete?eval_name=eval1" +url = "https://api.unify.ai/v0/evaluator?name=eval1" headers = {"Authorization": "Bearer "} diff --git a/api-reference/evaluators/get_evaluator.mdx b/api-reference/evaluators/get_evaluator.mdx new file mode 100644 index 000000000..ee68876ee --- /dev/null +++ b/api-reference/evaluators/get_evaluator.mdx @@ -0,0 +1,62 @@ +--- +title: 'Get Evaluator' +api: 'GET /v0/evaluator' +--- +Returns the configuration JSON for an evaluator from your account. The configuration +contains the same information as the arguments passed to the `POST` function for the +same endpoint `/v0/evaluator`. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +Name of the evaluator to return the configuration of. + + + + +```bash cURL +curl --request GET \ + --url 'https://api.unify.ai/v0/evaluator?name=eval1' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/evaluator?name=eval1" + +headers = {"Authorization": "Bearer "} + +response = requests.request("GET", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/dataset_evaluation/list_evals.mdx b/api-reference/evaluators/list_evaluators.mdx similarity index 71% rename from api-reference/dataset_evaluation/list_evals.mdx rename to api-reference/evaluators/list_evaluators.mdx index 4569b395c..97022eb11 100644 --- a/api-reference/dataset_evaluation/list_evals.mdx +++ b/api-reference/evaluators/list_evaluators.mdx @@ -1,8 +1,8 @@ --- -title: 'List Evals' -api: 'GET /v0/evals/list_configs' +title: 'List Evaluators' +api: 'GET /v0/evaluator/list' --- -Returns the names of the eval configurations you have created. +Returns the names of all evaluators stored in your account. #### Authorizations @@ -14,14 +14,14 @@ Returns the names of the eval configurations you have created. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/evals/list_configs' \ + --url 'https://api.unify.ai/v0/evaluator/list' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/evals/list_configs" +url = "https://api.unify.ai/v0/evaluator/list" headers = {"Authorization": "Bearer "} diff --git a/api-reference/dataset_evaluation/rename_eval.mdx b/api-reference/evaluators/rename_evaluator.mdx similarity index 63% rename from api-reference/dataset_evaluation/rename_eval.mdx rename to api-reference/evaluators/rename_evaluator.mdx index 704f244e2..0e261e775 100644 --- a/api-reference/dataset_evaluation/rename_eval.mdx +++ b/api-reference/evaluators/rename_evaluator.mdx @@ -1,8 +1,8 @@ --- -title: 'Rename Eval' -api: 'POST /v0/evals/rename' +title: 'Rename Evaluator' +api: 'POST /v0/evaluator/rename' --- -Renames a named eval from `eval_name` to `new_eval_name`. +Renames an evaluator from `name` to `new_name` in your account. #### Authorizations @@ -12,26 +12,26 @@ Renames a named eval from `eval_name` to `new_eval_name`. #### Query Parameters - -Name of the existing eval to rename + +Name of the evaluator to rename. - -New name for the eval + +New name for the evaluator. ```bash cURL curl --request POST \ - --url 'https://api.unify.ai/v0/evals/rename?eval_name=eval1&new_eval_name=eval2' \ + --url 'https://api.unify.ai/v0/evaluator/rename?name=eval1&new_name=eval2' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/evals/rename?eval_name=eval1&new_eval_name=eval2" +url = "https://api.unify.ai/v0/evaluator/rename?name=eval1&new_name=eval2" headers = {"Authorization": "Bearer "} diff --git a/api-reference/llm_queries/chat_completions.mdx b/api-reference/llm_queries/chat_completions.mdx new file mode 100644 index 000000000..1ef76e390 --- /dev/null +++ b/api-reference/llm_queries/chat_completions.mdx @@ -0,0 +1,236 @@ +--- +title: 'Chat Completions' +api: 'POST /v0/chat/completions' +--- +OpenAI compatible `/chat/completions` endpoint for LLM inference. +Check the OpenAI +[API reference](https://platform.openai.com/docs/api-reference/chat) +for the most updated documentation. The ground truth is always the latest OpenAI API +Reference. The arguments below are copied for convenience, but might not be fully +up-to-date at all times. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Body + + +A list of messages comprising the conversation so far. + + + + +
+ +[Unified Arguments](https://docs.unify.ai/universal_api/arguments#unified-arguments) + + +The endpoint to use, in the format `{model}@{provider}`, based on any of the supported endpoints as per the list returned by `/v0/endpoints` + + + +The maximum number of tokens that can be generated in the chat completion. + +The total length of input tokens and generated tokens is limited by the model's context length. + + + +Up to 4 sequences where the API will stop generating further tokens. + + + +If set, partial message deltas will be sent. Tokens will be sent as data-only server-sent events as they become available, with the stream terminated by a `data: [DONE]` message. + + + +What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. + +Generally recommended to alter this or `top_p`, but not both. + + + + +
+ +[Partially Unified Arguments](https://docs.unify.ai/universal_api/arguments#partially-unified-arguments) + + +Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. + + + +Modify the likelihood of specified tokens appearing in the completion. + +Accepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. + + + +Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the `content` of `message`. + + + +An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used. + + + +How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs. + + + +Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. + + + +An object specifying the format that the model must output. + +Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema. + +Setting to `{ "type": "json_object" }` enables JSON mode, which ensures the message the model generates is valid JSON. + +**Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length. + + + +If specified, the system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend. + + + +Options for streaming response. Only set this when you set `stream: true`. + + + +An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + +Generally recommended to alter this or `temperature` but not both. + + + +A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported. + + + +Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message.`auto` means the model can pick between generating a message or calling one or more tools. `required` means the model must call one or more tools. Specifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. + +`none` is the default when no tools are present. `auto` is the default if tools are present. + + + +Whether to enable parallel function calling during tool use. + + + +A unique identifier representing your end-user. + + + + +
+ +[Platform Arguments](https://docs.unify.ai/universal_api/arguments#platform-arguments) + + +A string used to represent where the request came from, for examples, did it come via the Python package, the NodeJS package, the chat interface etc. This should *not* be set by the user. + + + +Whether or not to use custom API keys with the specified provider, meaning that you will be using your own account with that provider in the backend. + + + +Whether or not to use custom API keys with the specified provider, meaning that you will be using your own account with that provider in the backend. + + + + +```bash cURL +curl --request POST \ + --url 'https://api.unify.ai/v0/chat/completions' \ + --header 'Authorization: Bearer ' \ + --header 'Content-Type: application/json' \ + --data '{ + "messages": [ + { + "content": "Tell me a joke", + "role": "user" + } + ], + "model": "gpt-4o-mini@openai", + "max_tokens": 1024, + "stop": [ + "The End.", + " is the answer." + ], + "stream": false, + "temperature": 0.9, + "frequency_penalty": 1.5, + "logit_bias": { + "0": 10, + "1": -75, + "2": 90 + }, + "logprobs": false, + "top_logprobs": 15, + "n": 15, + "presence_penalty": -1.1, + "response_format": "{ \"type\": \"json_mode\"}", + "seed": 11, + "stream_options": [ + true, + "include_usage" + ], + "top_p": 0.5, + "tool_choice": "{\"type\": \"function\", \"function\": {\"name\": \"my_function\"}}", + "parallel_tool_calls": true, + "user": "some_user", + "signature": "python", + "use_custom_keys": true, + "tags": true +}' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/chat/completions" + +headers = {"Authorization": "Bearer "} + +json_input = {"messages": [{"content": "Tell me a joke", "role": "user"}], "model": "gpt-4o-mini@openai", "max_tokens": 1024, "stop": ["The End.", " is the answer."], "stream": False, "temperature": 0.9, "frequency_penalty": 1.5, "logit_bias": {"0": 10, "1": -75, "2": 90}, "logprobs": False, "top_logprobs": 15, "n": 15, "presence_penalty": -1.1, "response_format": "{ "type": "json_mode"}", "seed": 11, "stream_options": [True, "include_usage"], "top_p": 0.5, "tool_choice": "{"type": "function", "function": {"name": "my_function"}}", "parallel_tool_calls": True, "user": "some_user", "signature": "python", "use_custom_keys": True, "tags": True} + +response = requests.request("POST", url, json=json_input, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{ + "model": "string", + "created": "integer | null", + "id": "string | null", + "object": "string", + "usage": "object", + "choices": "[object]" +} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/benchmarks/get_prompt_history.mdx b/api-reference/logging/get_prompt_history.mdx similarity index 72% rename from api-reference/benchmarks/get_prompt_history.mdx rename to api-reference/logging/get_prompt_history.mdx index 59337661a..b5fd747b6 100644 --- a/api-reference/benchmarks/get_prompt_history.mdx +++ b/api-reference/logging/get_prompt_history.mdx @@ -2,7 +2,7 @@ title: 'Get Prompt History' api: 'GET /v0/prompt_history' --- - +Get the prompt history, optionally for a given set of tags for a narrowed search. #### Authorizations @@ -12,22 +12,22 @@ api: 'GET /v0/prompt_history' #### Query Parameters - -Provide a tag to filter by prompts that are marked with this tag. + +Tags to filter for prompts that are marked with these tags. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/prompt_history' \ + --url 'https://api.unify.ai/v0/prompt_history?tags=None' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/prompt_history" +url = "https://api.unify.ai/v0/prompt_history?tags=None" headers = {"Authorization": "Bearer "} diff --git a/api-reference/querying_llms/get_query_metrics.mdx b/api-reference/logging/get_query_metrics.mdx similarity index 73% rename from api-reference/querying_llms/get_query_metrics.mdx rename to api-reference/logging/get_query_metrics.mdx index 53f0d4022..0d1a8ac08 100644 --- a/api-reference/querying_llms/get_query_metrics.mdx +++ b/api-reference/logging/get_query_metrics.mdx @@ -3,7 +3,9 @@ title: 'Get Query Metrics' api: 'GET /v0/metrics' --- Returns aggregated telemetry data from previous queries to the `/chat/completions` -endpoint. +endpoint, specifically the p50 and p95 for generation time and tokens per second, +and also the total prompt and completion tokens processed within the interval. The +user id and total request count within the interval are also returned. #### Authorizations @@ -13,27 +15,27 @@ endpoint. #### Query Parameters - + Timestamp of the earliest query to aggregate. Format is `YYYY-MM-DD hh:mm:ss`. - + Timestamp of the latest query to aggregate. Format is `YYYY-MM-DD hh:mm:ss`. - -Models to fetch metrics from. The list must be a set of comma-sparated strings. i.e. `gpt-3.5-turbo,gpt-4o` + +Models to fetch metrics from. The list must be a set of comma-separated strings. i.e. `gpt-3.5-turbo,gpt-4o` - -Providers to fetch metrics from. The list must be a set of comma-sparated strings. i.e. `openai,together-ai` + +Providers to fetch metrics from. The list must be a set of comma-separated strings. i.e. `openai,together-ai` - + Number of seconds in the aggregation interval. - + Secondary user id. The secondary user id will match any string previously sent in the `user` attribute of `/chat/completions`. diff --git a/api-reference/openapi.json b/api-reference/openapi.json index 185f0fc81..78705c89c 100644 --- a/api-reference/openapi.json +++ b/api-reference/openapi.json @@ -5,45 +5,75 @@ "version": "0.1.0" }, "paths": { - "/v0/get_credits": { + "/v0/providers": { "get": { "tags": [ - "User" + "Supported Endpoints" + ], + "summary": "List Providers", + "description": "Lists available providers. If `model` is specified,\nreturns the providers that support that model.", + "operationId": "list_providers_v0_providers_get", + "security": [ + { + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "model", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Model to get available providers for.", + "title": "Model" + }, + "description": "Model to get available providers for.", + "example": "llama-3.1-405b-chat" + } ], - "summary": "Get Credits", - "description": "Returns the number of available credits.", - "operationId": "get_credits_v0_get_credits_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreditsResponse" + "type": "array", + "items": { + "type": "string" + }, + "title": "Response List Providers V0 Providers Get" }, - "example": { - "id": "", - "credits": 10 + "example": [ + "openai", + "anthropic", + "together-ai", + "..." + ] + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" } } } } - }, - "security": [ - { - "HTTPBearer": [] - } - ] + } } }, - "/v0/promo": { - "post": { + "/v0/models": { + "get": { "tags": [ - "User" + "Supported Endpoints" ], - "summary": "Credits Code", - "description": "Activates a promotional code.", - "operationId": "credits_code_v0_promo_post", + "summary": "List Models", + "description": "Lists available models. If `provider` is specified,\nreturns the models that the provider supports.\nYou can also show all *custom* models by passing `custom` as the provider.", + "operationId": "list_models_v0_models_get", "security": [ { "HTTPBearer": [] @@ -51,35 +81,16 @@ ], "parameters": [ { - "name": "code", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Promo code to be activated.", - "title": "Code" - }, - "description": "Promo code to be activated.", - "example": "sample_code" - }, - { - "name": "user", + "name": "provider", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "ID of the user that receives the credits,defaults to the user making the request.", - "title": "User" + "type": "string", + "description": "Provider to get available models from.", + "title": "Provider" }, - "description": "ID of the user that receives the credits,defaults to the user making the request.", - "example": "sample_user_id" + "description": "Provider to get available models from.", + "example": "openai" } ], "responses": { @@ -88,35 +99,18 @@ "content": { "application/json": { "schema": { - "type": "object", - "additionalProperties": { + "type": "array", + "items": { "type": "string" }, - "title": "Response Credits Code V0 Promo Post" + "title": "Response List Models V0 Models Get" }, - "example": { - "info": "Code {code} activated succesfully!" - } - } - } - }, - "400": { - "description": "Already activated code", - "content": { - "application/json": { - "example": { - "detail": "This code has already been activated." - } - } - } - }, - "404": { - "description": "Code Not Found", - "content": { - "application/json": { - "example": { - "detail": "Invalid code." - } + "example": [ + "gpt-3.5-turbo", + "gpt-4", + "claude-3-haiku", + "..." + ] } } }, @@ -133,30 +127,42 @@ } } }, - "/v0/models": { + "/v0/endpoints": { "get": { "tags": [ - "Model and Endpoints" + "Supported Endpoints" ], - "summary": "Get Models", - "description": "Lists available models. If a provider is specified, returns the models that the provider supports.", - "operationId": "get_models_v0_models_get", + "summary": "List Endpoints", + "description": "Lists available endpoints in `model@provider` format.\nIf `model` or `provider` are specified, only the matching endpoints will be listed.\nYou can also show all *custom* endpoints by passing `custom` as the provider.", + "operationId": "list_endpoints_v0_endpoints_get", "security": [ { "HTTPBearer": [] } ], "parameters": [ + { + "name": "model", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Model to get available endpoints for.", + "title": "Model" + }, + "description": "Model to get available endpoints for.", + "example": "llama-3.1-405b-chat" + }, { "name": "provider", "in": "query", "required": false, "schema": { "type": "string", - "description": "Provider to get available models from.", + "description": "Provider to get available endpoints for.", "title": "Provider" }, - "description": "Provider to get available models from.", + "description": "Provider to get available endpoints for.", "example": "openai" } ], @@ -170,11 +176,12 @@ "items": { "type": "string" }, - "title": "Response Get Models V0 Models Get" + "title": "Response List Endpoints V0 Endpoints Get" }, "example": [ - "model_a", - "model_b", + "claude-3-haiku@anthropic", + "llama-3-70b-chat@groq", + "mistral-large@mistral-ai", "..." ] } @@ -193,19 +200,19 @@ } } }, - "/v0/inference": { + "/v0/chat/completions": { "post": { "tags": [ - "Querying LLMs" + "LLM Queries" ], - "summary": "Post Inference", - "description": "DEPRECATION WARNING: This endpoint is deprecated and will be removed. Instead,\nuse the /chat/completions endpoint for LLM inference.", - "operationId": "post_inference_v0_inference_post", + "summary": "Chat Completions", + "description": "OpenAI compatible `/chat/completions` endpoint for LLM inference.\nCheck the OpenAI\n[API reference](https://platform.openai.com/docs/api-reference/chat)\nfor the most updated documentation. The ground truth is always the latest OpenAI API\nReference. The arguments below are copied for convenience, but might not be fully\nup-to-date at all times.", + "operationId": "chat_completions_v0_chat_completions_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InferenceRequest" + "$ref": "#/components/schemas/ChatCompletionRequest" } } }, @@ -216,7 +223,9 @@ "description": "Successful Response", "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/ChatCompletionResponse" + } } } }, @@ -231,7 +240,6 @@ } } }, - "deprecated": true, "security": [ { "HTTPBearer": [] @@ -239,32 +247,48 @@ ] } }, - "/v0/chat/completions": { - "post": { + "/v0/prompt_history": { + "get": { "tags": [ - "Querying LLMs" + "Logging" + ], + "summary": "Get Prompt History", + "description": "Get the prompt history, optionally for a given set of tags for a narrowed search.", + "operationId": "get_prompt_history_v0_prompt_history_get", + "security": [ + { + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "tags", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "Tags to filter for prompts that are marked with these tags.", + "title": "Tags" + }, + "description": "Tags to filter for prompts that are marked with these tags." + } ], - "summary": "Get Completions", - "description": "OpenAI compatible /chat/completions endpoint for LLM inference.", - "operationId": "get_completions_v0_chat_completions_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChatCompletionRequest" - } - } - }, - "required": true - }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { - "schema": { - "$ref": "#/components/schemas/ChatCompletionResponse" - } + "schema": {} } } }, @@ -278,21 +302,16 @@ } } } - }, - "security": [ - { - "HTTPBearer": [] - } - ] + } } }, "/v0/metrics": { "get": { "tags": [ - "Querying LLMs" + "Logging" ], "summary": "Get Query Metrics", - "description": "Returns aggregated telemetry data from previous queries to the `/chat/completions`\nendpoint.", + "description": "Returns aggregated telemetry data from previous queries to the `/chat/completions`\nendpoint, specifically the p50 and p95 for generation time and tokens per second,\nand also the total prompt and completion tokens processed within the interval. The\nuser id and total request count within the interval are also returned.", "operationId": "get_query_metrics_v0_metrics_get", "security": [ { @@ -305,14 +324,7 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "description": "Timestamp of the earliest query to aggregate. Format is `YYYY-MM-DD hh:mm:ss`.", "title": "Start Time" }, @@ -324,14 +336,7 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "description": "Timestamp of the latest query to aggregate. Format is `YYYY-MM-DD hh:mm:ss`.", "title": "End Time" }, @@ -343,18 +348,11 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Models to fetch metrics from. The list must be a set of comma-sparated strings. i.e. `gpt-3.5-turbo,gpt-4o`", + "type": "string", + "description": "Models to fetch metrics from. The list must be a set of comma-separated strings. i.e. `gpt-3.5-turbo,gpt-4o`", "title": "Models" }, - "description": "Models to fetch metrics from. The list must be a set of comma-sparated strings. i.e. `gpt-3.5-turbo,gpt-4o`", + "description": "Models to fetch metrics from. The list must be a set of comma-separated strings. i.e. `gpt-3.5-turbo,gpt-4o`", "example": "gpt-4o,llama-3.1-405b-chat,claude-3.5-sonnet" }, { @@ -362,18 +360,11 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Providers to fetch metrics from. The list must be a set of comma-sparated strings. i.e. `openai,together-ai`", + "type": "string", + "description": "Providers to fetch metrics from. The list must be a set of comma-separated strings. i.e. `openai,together-ai`", "title": "Providers" }, - "description": "Providers to fetch metrics from. The list must be a set of comma-sparated strings. i.e. `openai,together-ai`", + "description": "Providers to fetch metrics from. The list must be a set of comma-separated strings. i.e. `openai,together-ai`", "example": "openai,anthropic,fireworks-ai" }, { @@ -381,14 +372,7 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "description": "Number of seconds in the aggregation interval.", "default": 300, "title": "Interval" @@ -401,14 +385,7 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "description": "Secondary user id. The secondary user id will match any string previously sent in the `user` attribute of `/chat/completions`.", "title": "Secondary User Id" }, @@ -442,42 +419,13 @@ } }, "/v0/custom_endpoint": { - "get": { - "tags": [ - "Custom Endpoints and API keys" - ], - "summary": "Get Custom Endpoints", - "description": "Returns a list of the available custom endpoints.", - "operationId": "get_custom_endpoints_v0_custom_endpoint_get", - "security": [ - { - "HTTPBearer": [] - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomEndpointModelResponse" - }, - "title": "Response Get Custom Endpoints V0 Custom Endpoint Get" - } - } - } - } - } - }, - "put": { + "post": { "tags": [ - "Custom Endpoints and API keys" + "Custom Endpoints" ], "summary": "Create Custom Endpoint", - "description": "Creates a custom endpoint. This endpoint must support the OpenAI `/chat/completions`\nformat. To query your custom endpoint, replace your endpoint string with `@custom`\nwhen querying the unified API.", - "operationId": "create_custom_endpoint_v0_custom_endpoint_put", + "description": "Creates a custom endpoint. This endpoint must either be a fine-tuned model from one\nof the supported providers (`/v0/providers`), in which case the \"provider\" argument\nmust be set accordingly. Otherwise, the endpoint must support the OpenAI\n`/chat/completions` format. To query your custom endpoint, replace your endpoint\nstring with `@custom` when querying any general custom endpoint. You can show\nall *custom* endpoints by querying `/v0/endpoints` and passing `custom` as the\n`provider` argument.", + "operationId": "create_custom_endpoint_v0_custom_endpoint_post", "security": [ { "HTTPBearer": [] @@ -521,23 +469,28 @@ "example": "key1" }, { - "name": "mdl_name", + "name": "model_name", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "description": "Named passed to the custom endpoint as model name. If not specified, it will default to the endpoint alias.", - "title": "Mdl Name" + "title": "Model Name" }, "description": "Named passed to the custom endpoint as model name. If not specified, it will default to the endpoint alias.", "example": "llama-3.1-8b-finetuned" + }, + { + "name": "provider", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "If the custom endpoint is for a fine-tuned model which is hosted directly via one of the supported providers, then this argument should be specified as the provider used. The unification logic will be used for this custom fine-tuned model behind the scenes, in the same manner as used for the foundation models with the same provider.", + "title": "Provider" + }, + "description": "If the custom endpoint is for a fine-tuned model which is hosted directly via one of the supported providers, then this argument should be specified as the provider used. The unification logic will be used for this custom fine-tuned model behind the scenes, in the same manner as used for the foundation models with the same provider.", + "example": "together-ai" } ], "responses": { @@ -547,7 +500,7 @@ "application/json": { "schema": {}, "example": { - "info": "Custom endpoint created succesfully!" + "info": "Custom endpoint created successfully!" } } } @@ -576,10 +529,10 @@ }, "delete": { "tags": [ - "Custom Endpoints and API keys" + "Custom Endpoints" ], "summary": "Delete Custom Endpoint", - "description": "Deletes a custom endpoint in your account.", + "description": "Deletes a custom endpoint from your account.", "operationId": "delete_custom_endpoint_v0_custom_endpoint_delete", "security": [ { @@ -607,7 +560,7 @@ "application/json": { "schema": {}, "example": { - "info": "Custom endpoint deleted succesfully!" + "info": "Custom endpoint deleted successfully!" } } } @@ -625,43 +578,88 @@ } } }, - "/v0/custom_api_key": { - "get": { + "/v0/custom_endpoint/rename": { + "post": { "tags": [ - "Custom Endpoints and API keys" + "Custom Endpoints" ], - "summary": "Get Custom Api Keys", - "description": "Returns a list of the available custom API keys.", - "operationId": "get_custom_api_keys_v0_custom_api_key_get", + "summary": "Rename Custom Endpoint", + "description": "Renames a custom endpoint in your account.", + "operationId": "rename_custom_endpoint_v0_custom_endpoint_rename_post", "security": [ { "HTTPBearer": [] } ], + "parameters": [ + { + "name": "name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Name of the custom endpoint to be updated.", + "title": "Name" + }, + "description": "Name of the custom endpoint to be updated.", + "example": "name1" + }, + { + "name": "new_name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "New name for the custom endpoint.", + "title": "New Name" + }, + "description": "New name for the custom endpoint.", + "example": "name2" + } + ], "responses": { "200": { "description": "Successful Response", + "content": { + "application/json": { + "schema": {}, + "example": { + "info": "Custom endpoint renamed successfully!" + } + } + } + }, + "404": { + "description": "Custom endpoint Not Found", + "content": { + "application/json": { + "example": { + "detail": "Custom endpoint not found." + } + } + } + }, + "422": { + "description": "Validation Error", "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomApiKeyModelResponse" - }, - "title": "Response Get Custom Api Keys V0 Custom Api Key Get" + "$ref": "#/components/schemas/HTTPValidationError" } } } } } - }, - "put": { + } + }, + "/v0/custom_api_key": { + "post": { "tags": [ - "Custom Endpoints and API keys" + "Custom API keys" ], "summary": "Create Custom Api Key", - "description": "Stores a custom API key from a LLM provider in your account. This can be used in two ways:\n1. As part of a custom endpoint. If you define a custom endpoint, you can reference a custom API\nkey. This will be sent to the endpoint as part of the request.\n2. To use your own API keys in standard providers. If any of your custom API keys matches a provider\nname and you pass `use_custom_keys=True` to the `/chat/completions` endpoint, this API key will\nbe used, charging your account directly instead of consuming Unify credits.", - "operationId": "create_custom_api_key_v0_custom_api_key_put", + "description": "Stores a custom API key from an LLM provider in your account. This can be done in\none of two ways:\n\n1. As part of a custom endpoint. If you define a custom endpoint, you can reference\na custom API key. This will be sent to the endpoint as part of the request.\n\n2. To use your own API keys with the standard providers. If any of your custom API\nkeys match a provider name and you pass `use_custom_keys=True` to the\n`/chat/completions` endpoint, then this API key will be used, using your own\naccount with the provider directly.", + "operationId": "create_custom_api_key_v0_custom_api_key_post", "security": [ { "HTTPBearer": [] @@ -669,13 +667,13 @@ ], "parameters": [ { - "name": "key", + "name": "name", "in": "query", "required": true, "schema": { "type": "string", "description": "Name of the API key.", - "title": "Key" + "title": "Name" }, "description": "Name of the API key.", "example": "key1" @@ -700,7 +698,7 @@ "application/json": { "schema": {}, "example": { - "info": "API key created succesfully!" + "info": "API key created successfully!" } } } @@ -717,13 +715,13 @@ } } }, - "delete": { + "get": { "tags": [ - "Custom Endpoints and API keys" + "Custom API keys" ], - "summary": "Delete Custom Api Key", - "description": "Deletes a custom API key in your account.", - "operationId": "delete_custom_api_key_v0_custom_api_key_delete", + "summary": "Get Custom Api Key", + "description": "Returns the value of the key for the specified custom API key name.", + "operationId": "get_custom_api_key_v0_custom_api_key_get", "security": [ { "HTTPBearer": [] @@ -731,15 +729,15 @@ ], "parameters": [ { - "name": "key", + "name": "name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the custom API key to delete.", - "title": "Key" + "description": "Name of the API key to get the value for.", + "title": "Name" }, - "description": "Name of the custom API key to delete.", + "description": "Name of the API key to get the value for.", "example": "key1" } ], @@ -748,19 +746,8 @@ "description": "Successful Response", "content": { "application/json": { - "schema": {}, - "example": { - "info": "API key deleted succesfully!" - } - } - } - }, - "404": { - "description": "Custom API key Not Found", - "content": { - "application/json": { - "example": { - "detail": "API key not found." + "schema": { + "$ref": "#/components/schemas/CustomApiKeyModelResponse" } } } @@ -776,16 +763,14 @@ } } } - } - }, - "/v0/custom_endpoint/rename": { - "post": { + }, + "delete": { "tags": [ - "Custom Endpoints and API keys" + "Custom API keys" ], - "summary": "Rename Custom Endpoint", - "description": "Renames a custom endpoint in your account.", - "operationId": "rename_custom_endpoint_v0_custom_endpoint_rename_post", + "summary": "Delete Custom Api Key", + "description": "Deletes the custom API key from your account.", + "operationId": "delete_custom_api_key_v0_custom_api_key_delete", "security": [ { "HTTPBearer": [] @@ -798,23 +783,11 @@ "required": true, "schema": { "type": "string", - "description": "Name of the custom endpoint to be updated.", + "description": "Name of the custom API key to delete.", "title": "Name" }, - "description": "Name of the custom endpoint to be updated.", - "example": "name1" - }, - { - "name": "new_name", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "New name for the custom endpoint.", - "title": "New Name" - }, - "description": "New name for the custom endpoint.", - "example": "name2" + "description": "Name of the custom API key to delete.", + "example": "key1" } ], "responses": { @@ -824,17 +797,17 @@ "application/json": { "schema": {}, "example": { - "info": "Custom endpoint renamed succesfully!" + "info": "API key deleted successfully!" } } } }, "404": { - "description": "Custom endpoint Not Found", + "description": "Custom API key Not Found", "content": { "application/json": { "example": { - "detail": "Custom endpoint not found." + "detail": "API key not found." } } } @@ -855,10 +828,10 @@ "/v0/custom_api_key/rename": { "post": { "tags": [ - "Custom Endpoints and API keys" + "Custom API keys" ], "summary": "Rename Custom Api Key", - "description": "Renames a custom API key from a LLM provider in your account.", + "description": "Renames the custom API key in your account.", "operationId": "rename_custom_api_key_v0_custom_api_key_rename_post", "security": [ { @@ -867,25 +840,25 @@ ], "parameters": [ { - "name": "key", + "name": "name", "in": "query", "required": true, "schema": { "type": "string", "description": "Name of the custom API key to be updated.", - "title": "Key" + "title": "Name" }, "description": "Name of the custom API key to be updated.", "example": "key1" }, { - "name": "new_key", + "name": "new_name", "in": "query", "required": true, "schema": { "type": "string", "description": "New name for the custom API key.", - "title": "New Key" + "title": "New Name" }, "description": "New name for the custom API key.", "example": "key2" @@ -898,7 +871,7 @@ "application/json": { "schema": {}, "example": { - "info": "API key renamed succesfully!" + "info": "API key renamed successfully!" } } } @@ -926,74 +899,84 @@ } } }, - "/v0/custom_endpoint/benchmark": { - "post": { + "/v0/custom_api_key/list": { + "get": { "tags": [ - "Custom Endpoints and API keys" + "Custom API keys" ], - "summary": "Upload Custom Benchmark", - "operationId": "upload_custom_benchmark_v0_custom_endpoint_benchmark_post", + "summary": "List Custom Api Keys", + "description": "Returns a list of the names for all custom API keys in your account.", + "operationId": "list_custom_api_keys_v0_custom_api_key_list_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/CustomApiKeyModelResponse" + }, + "type": "array", + "title": "Response List Custom Api Keys V0 Custom Api Key List Get" + } + } + } + } + }, "security": [ { "HTTPBearer": [] } + ] + } + }, + "/v0/dataset": { + "post": { + "tags": [ + "Datasets" ], - "parameters": [ - { - "name": "endpoint_name", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Name of the custom endpoint to submit a benchmark for.", - "title": "Endpoint Name" - }, - "description": "Name of the custom endpoint to submit a benchmark for.", - "example": "endpoint1" - }, - { - "name": "metric_name", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Name of the metric to submit. Allowed metrics are: \"input-cost\", \"output-cost\", \"tokens-per-second\", \"time-to-first-token\", \"inter-token-latency\", \"end-2-end-latency\", \"cold-start\".", - "title": "Metric Name" - }, - "description": "Name of the metric to submit. Allowed metrics are: \"input-cost\", \"output-cost\", \"tokens-per-second\", \"time-to-first-token\", \"inter-token-latency\", \"end-2-end-latency\", \"cold-start\".", - "example": "tokens-per-second" - }, + "summary": "Upload Dataset", + "description": "Uploads a custom dataset to your account.\n\nThe uploaded file must be a JSONL file with **at least** a `prompt` key for\neach prompt each:\n\n```\n{\"prompt\": \"This is the first prompt\"}\n{\"prompt\": \"This is the second prompt\"}\n{\"prompt\": \"This is the third prompt\"}\n```\n\nAdditionally, you can include any extra keys as desired, depending on the use case\nfor the dataset, and how it will be used by the evaluators and/or router training.\nFor example, you could include a reference answer to each prompt as follows:\n\n```\n{\"prompt\": \"This is the first prompt\", \"ref_answer\": \"First reference answer\"}\n{\"prompt\": \"This is the second prompt\", \"ref_answer\": \"Second reference answer\"}\n{\"prompt\": \"This is the third prompt\", \"ref_answer\": \"Third reference answer\"}\n```", + "operationId": "upload_dataset_v0_dataset_post", + "security": [ { - "name": "value", - "in": "query", - "required": true, - "schema": { - "type": "number", - "description": "Value of the metric to submit.", - "title": "Value" - }, - "description": "Value of the metric to submit.", - "example": 10 + "HTTPBearer": [] } ], + "requestBody": { + "required": true, + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_upload_dataset_v0_dataset_post" + } + } + } + }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { - "schema": {}, + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "Response Upload Dataset V0 Dataset Post" + }, "example": { - "info": "Custom endpoint benchmark uploaded succesfully!" + "info": "Dataset uploaded sucessfully!" } } } }, "400": { - "description": "Benchmark not valid", + "description": "Dataset already exists", "content": { "application/json": { "example": { - "detail": "Invalid data submitted" + "detail": "A dataset with this name already exists.Please, choose a different one." } } } @@ -1009,15 +992,14 @@ } } } - } - }, - "/v0/custom_endpoint/get_benchmark": { + }, "get": { "tags": [ - "Custom Endpoints and API keys" + "Datasets" ], - "summary": "Get Custom Benchmarks", - "operationId": "get_custom_benchmarks_v0_custom_endpoint_get_benchmark_get", + "summary": "Download Dataset", + "description": "Downloads a specific dataset from your account.", + "operationId": "download_dataset_v0_dataset_get", "security": [ { "HTTPBearer": [] @@ -1025,125 +1007,52 @@ ], "parameters": [ { - "name": "endpoint_name", + "name": "name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the custom endpoint to get a benchmark for.", - "title": "Endpoint Name" + "description": "Name of the dataset.", + "title": "Name" }, - "description": "Name of the custom endpoint to get a benchmark for.", - "example": "endpoint1" + "description": "Name of the dataset.", + "example": "dataset1" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {}, + "example": [ + { + "prompt": "This is the first prompt" + }, + { + "prompt": "This is the second prompt" + }, + "..." + ] + } + } }, - { - "name": "metric_name", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Name of the metric to get the benchmark of.", - "title": "Metric Name" - }, - "description": "Name of the metric to get the benchmark of.", - "example": "tokens-per-second" - }, - { - "name": "start_time", - "in": "query", - "required": false, - "schema": { - "type": "string", - "description": "Start time of window to get benchmarks between. Format YYYY-MM-DD", - "default": "2024-01-01", - "title": "Start Time" - }, - "description": "Start time of window to get benchmarks between. Format YYYY-MM-DD", - "example": "2024-01-01" - }, - { - "name": "end_time", - "in": "query", - "required": false, - "schema": { - "type": "string", - "description": "End time of window to get benchmarks between. Format YYYY-MM-DD", - "default": "2024-12-12", - "title": "End Time" - }, - "description": "End time of window to get benchmarks between. Format YYYY-MM-DD", - "example": "2024-12-12" - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v0/dataset": { - "post": { - "tags": [ - "Dataset" - ], - "summary": "Upload Dataset", - "description": "Uploads a custom dataset to the platform.\n\nThe uploaded file must be a JSONL file with **at least** a `prompt` key:\n\n```\n{\"prompt\": \"This is the first prompt\"}\n{\"prompt\": \"This is the second prompt\"}\n{\"prompt\": \"This is the third prompt\"}\n```\n\nAdditionally, you can include a `ref_answer` key, which will be accounted\nduring the evaluations.\n\n```\n{\"prompt\": \"This is the first prompt\", \"ref_answer\": \"First reference answer\"}\n{\"prompt\": \"This is the second prompt\", \"ref_answer\": \"Second reference answer\"}\n{\"prompt\": \"This is the third prompt\", \"ref_answer\": \"Third reference answer\"}\n```", - "operationId": "upload_dataset_v0_dataset_post", - "security": [ - { - "HTTPBearer": [] - } - ], - "requestBody": { - "required": true, - "content": { - "application/x-www-form-urlencoded": { - "schema": { - "$ref": "#/components/schemas/Body_upload_dataset_v0_dataset_post" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", + "400": { + "description": "Invalid dataset name", "content": { "application/json": { - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "Response Upload Dataset V0 Dataset Post" - }, "example": { - "info": "Dataset uploaded sucessfully!" + "detail": "Invalid name for a dataset. Please, choose a different one." } } } }, - "400": { - "description": "Dataset already exists", + "404": { + "description": "Dataset Not Found", "content": { "application/json": { "example": { - "detail": "A dataset with this name already exists. Please, choose a different one." + "detail": "This dataset does not exist." } } } @@ -1162,10 +1071,10 @@ }, "delete": { "tags": [ - "Dataset" + "Datasets" ], "summary": "Delete Dataset", - "description": "Deletes a previously updated dataset and any relevant artifacts from the platform.", + "description": "Deletes a previously updated dataset and any relevant artifacts from your account.", "operationId": "delete_dataset_v0_dataset_delete", "security": [ { @@ -1199,7 +1108,7 @@ "title": "Response Delete Dataset V0 Dataset Delete" }, "example": { - "info": "Dataset deleted succesfully!" + "info": "Dataset deleted successfully!" } } } @@ -1209,7 +1118,7 @@ "content": { "application/json": { "example": { - "detail": "Invalid name for a dataset. Please, choose a different one." + "detail": "Invalid name for a dataset.Please, choose a different one." } } } @@ -1225,14 +1134,16 @@ } } } - }, - "get": { + } + }, + "/v0/dataset/rename": { + "post": { "tags": [ - "Dataset" + "Datasets" ], - "summary": "Download Dataset", - "description": "Downloads a specific dataset from the platform.", - "operationId": "download_dataset_v0_dataset_get", + "summary": "Rename Dataset", + "description": "Renames a previously uploaded dataset.", + "operationId": "rename_dataset_v0_dataset_rename_post", "security": [ { "HTTPBearer": [] @@ -1245,11 +1156,23 @@ "required": true, "schema": { "type": "string", - "description": "Name of the dataset.", + "description": "Name of the dataset to be updated.", "title": "Name" }, - "description": "Name of the dataset.", + "description": "Name of the dataset to be updated.", "example": "dataset1" + }, + { + "name": "new_name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "New name for the dataset.", + "title": "New Name" + }, + "description": "New name for the dataset.", + "example": "dataset2" } ], "responses": { @@ -1257,35 +1180,25 @@ "description": "Successful Response", "content": { "application/json": { - "schema": {}, - "example": [ - { - "prompt": "This is the first prompt" - }, - { - "prompt": "This is the second prompt" + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" }, - "..." - ] - } - } - }, - "400": { - "description": "Invalid dataset name", - "content": { - "application/json": { + "title": "Response Rename Dataset V0 Dataset Rename Post" + }, "example": { - "detail": "Invalid name for a dataset. Please, choose a different one." + "info": "Dataset name updated sucessfully!" } } } }, - "404": { - "description": "Dataset Not Found", + "400": { + "description": "Dataset already exists", "content": { "application/json": { "example": { - "detail": "This dataset does not exist." + "detail": "A dataset with this name already exists.Please, choose a different one." } } } @@ -1306,10 +1219,10 @@ "/v0/dataset/list": { "get": { "tags": [ - "Dataset" + "Datasets" ], "summary": "List Datasets", - "description": "Lists all the custom datasets uploaded by the user to the platform.", + "description": "Lists all the datasets stored in the user account by name.", "operationId": "list_datasets_v0_dataset_list_get", "responses": { "200": { @@ -1339,14 +1252,57 @@ ] } }, - "/v0/dataset/rename": { + "/v0/evaluator": { "post": { "tags": [ - "Dataset" + "Evaluators" ], - "summary": "Rename Dataset", - "description": "Renames a previously updated dataset.", - "operationId": "rename_dataset_v0_dataset_rename_post", + "summary": "Create Evaluator", + "description": "Create a re-usable, named evaluator, and adds this to your account. This can be used\nto trigger an evaluation via `POST` requests to the `/evaluation` endpoint.", + "operationId": "create_evaluator_v0_evaluator_post", + "security": [ + { + "HTTPBearer": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluatorConfig" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "get": { + "tags": [ + "Evaluators" + ], + "summary": "Get Evaluator", + "description": "Returns the configuration JSON for an evaluator from your account. The configuration\ncontains the same information as the arguments passed to the `POST` function for the\nsame endpoint `/v0/evaluator`.", + "operationId": "get_evaluator_v0_evaluator_get", "security": [ { "HTTPBearer": [] @@ -1359,23 +1315,58 @@ "required": true, "schema": { "type": "string", - "description": "Name of the dataset to be updated.", + "description": "Name of the evaluator to return the configuration of.", "title": "Name" }, - "description": "Name of the dataset to be updated.", - "example": "dataset1" + "description": "Name of the evaluator to return the configuration of.", + "example": "eval1" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "Evaluators" + ], + "summary": "Delete Evaluator", + "description": "Deletes an evaluator from your account.", + "operationId": "delete_evaluator_v0_evaluator_delete", + "security": [ { - "name": "new_name", + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "name", "in": "query", "required": true, "schema": { "type": "string", - "description": "New name for the dataset.", - "title": "New Name" + "description": "Name of the evaluator to delete.", + "title": "Name" }, - "description": "New name for the dataset.", - "example": "dataset2" + "description": "Name of the evaluator to delete.", + "example": "eval1" } ], "responses": { @@ -1383,26 +1374,7 @@ "description": "Successful Response", "content": { "application/json": { - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "Response Rename Dataset V0 Dataset Rename Post" - }, - "example": { - "info": "Dataset name updated sucessfully!" - } - } - } - }, - "400": { - "description": "Dataset already exists", - "content": { - "application/json": { - "example": { - "detail": "A dataset with this name already exists. Please, choose a different one." - } + "schema": {} } } }, @@ -1419,24 +1391,45 @@ } } }, - "/v0/evals/create": { + "/v0/evaluator/rename": { "post": { "tags": [ - "Dataset Evaluation" + "Evaluators" ], - "summary": "Create Eval", - "description": "Create a re-useable, named eval configuration. This can be used to trigger an evaluation via the `/evals/trigger` endpoint.", - "operationId": "create_eval_v0_evals_create_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvalConfig" - } - } + "summary": "Rename Evaluator", + "description": "Renames an evaluator from `name` to `new_name` in your account.", + "operationId": "rename_evaluator_v0_evaluator_rename_post", + "security": [ + { + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Name of the evaluator to rename.", + "title": "Name" + }, + "description": "Name of the evaluator to rename.", + "example": "eval1" }, - "required": true - }, + { + "name": "new_name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "New name for the evaluator.", + "title": "New Name" + }, + "description": "New name for the evaluator.", + "example": "eval2" + } + ], "responses": { "200": { "description": "Successful Response", @@ -1456,22 +1449,17 @@ } } } - }, - "security": [ - { - "HTTPBearer": [] - } - ] + } } }, - "/v0/evals/list_configs": { + "/v0/evaluator/list": { "get": { "tags": [ - "Dataset Evaluation" + "Evaluators" ], - "summary": "List Evals", - "description": "Returns the names of the eval configurations you have created.", - "operationId": "list_evals_v0_evals_list_configs_get", + "summary": "List Evaluators", + "description": "Returns the names of all evaluators stored in your account.", + "operationId": "list_evaluators_v0_evaluator_list_get", "responses": { "200": { "description": "Successful Response", @@ -1489,14 +1477,14 @@ ] } }, - "/v0/evals/get_config": { - "get": { + "/v0/evaluation": { + "post": { "tags": [ - "Dataset Evaluation" + "Evaluations" ], - "summary": "Return Eval Config", - "description": "Returns the configuration JSON for a named eval.", - "operationId": "return_eval_config_v0_evals_get_config_get", + "summary": "Trigger Evaluation", + "description": "Uses the named `evaluator` to trigger an evaluation of quality scores for the\nselected LLM `endpoint` on the selected `dataset`. You can upload custom scores (and\nbypass the LLM judge entirely) by uploading a file via the `client_side_scores`\nargument. Once the evaluation has finished, you can access the scores using the\n`/v0/evaluation` endpoint.", + "operationId": "trigger_evaluation_v0_evaluation_post", "security": [ { "HTTPBearer": [] @@ -1504,24 +1492,91 @@ ], "parameters": [ { - "name": "eval_name", + "name": "evaluator", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the eval to return the configuration of", - "title": "Eval Name" + "description": "Name of the evaluator to use.", + "title": "Evaluator" }, - "description": "Name of the eval to return the configuration of", + "description": "Name of the evaluator to use.", "example": "eval1" + }, + { + "name": "dataset", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Name of the uploaded dataset to evaluate.", + "title": "Dataset" + }, + "description": "Name of the uploaded dataset to evaluate.", + "example": "dataset1" + }, + { + "name": "endpoint", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Name of the endpoint to evaluate. Endpoints must be specified using the `model@provider` format.", + "title": "Endpoint" + }, + "description": "Name of the endpoint to evaluate. Endpoints must be specified using the `model@provider` format.", + "example": "gpt-4o-mini@openai" } ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/Body_trigger_evaluation_v0_evaluation_post" + } + ], + "title": "Body" + } + } + } + }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { - "schema": {} + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "Response Trigger Evaluation V0 Evaluation Post" + }, + "example": { + "info": "Dataset evaluation started! You will receive an email soon!" + } + } + } + }, + "400": { + "description": "Invalid Endpoints", + "content": { + "application/json": { + "example": { + "detail": "Invalid input. Couldn't find endpoints [model_1@endpoint_1, model_2@endpoint_2]." + } + } + } + }, + "404": { + "description": "Dataset Not Found", + "content": { + "application/json": { + "example": { + "detail": "This dataset does not exist!" + } } } }, @@ -1536,16 +1591,14 @@ } } } - } - }, - "/v0/evals/rename": { - "post": { + }, + "get": { "tags": [ - "Dataset Evaluation" + "Evaluations" ], - "summary": "Rename Eval", - "description": "Renames a named eval from `eval_name` to `new_eval_name`.", - "operationId": "rename_eval_v0_evals_rename_post", + "summary": "Get Evaluations", + "description": "Fetches evaluation results on a given dataset, for a specific endpoint (optional)\nbased on a specific evaluator (optional). If no `evaluator` is provided, then scores\nare returned for all valid evaluators. Similarly, if no `endpoint` is provided, then\nscores are returned for all valid endpoints.", + "operationId": "get_evaluations_v0_evaluation_get", "security": [ { "HTTPBearer": [] @@ -1553,28 +1606,127 @@ ], "parameters": [ { - "name": "eval_name", + "name": "dataset", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the existing eval to rename", - "title": "Eval Name" + "description": "Name of the dataset to fetch evaluation from.", + "title": "Dataset" + }, + "description": "Name of the dataset to fetch evaluation from.", + "example": "dataset1" + }, + { + "name": "endpoint", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "The endpoint to fetch the evaluation for. If `None`, returns evaluations for all endpoints.", + "title": "Endpoint" }, - "description": "Name of the existing eval to rename", + "description": "The endpoint to fetch the evaluation for. If `None`, returns evaluations for all endpoints.", + "example": "gpt-4o-mini@openai" + }, + { + "name": "evaluator", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Name of the evaluator to fetch the evaluation for. If `None`, returns all available evaluations for the dataset and endpoint pair.", + "title": "Evaluator" + }, + "description": "Name of the evaluator to fetch the evaluation for. If `None`, returns all available evaluations for the dataset and endpoint pair.", "example": "eval1" }, { - "name": "new_eval_name", + "name": "per_prompt", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "If `True`, returns the scores on a per-prompt level. By default set to `False`. If `True` requires an eval name and endpoint to be set.", + "default": false, + "title": "Per Prompt" + }, + "description": "If `True`, returns the scores on a per-prompt level. By default set to `False`. If `True` requires an eval name and endpoint to be set.", + "example": false + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "title": "Response Get Evaluations V0 Evaluation Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "Evaluations" + ], + "summary": "Delete Evaluations", + "description": "Deletes evaluations on a given dataset, for a specific endpoint (optional) based on\na specific evaluator (optional). If no `evaluator` is provided, then evaluations for\nall valid evaluators are deleted. Similarly, if no `endpoint` is provided, then\nevaluations for all valid endpoints are deleted.", + "operationId": "delete_evaluations_v0_evaluation_delete", + "security": [ + { + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "dataset", "in": "query", "required": true, "schema": { "type": "string", - "description": "New name for the eval", - "title": "New Eval Name" + "description": "Name of the dataset to delete the evaluation for.", + "title": "Dataset" + }, + "description": "Name of the dataset to delete the evaluation for.", + "example": "dataset1" + }, + { + "name": "endpoint", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "The endpoint to delete the evaluation for. If `None`, deletes the evaluations for all endpoints.", + "title": "Endpoint" }, - "description": "New name for the eval", - "example": "eval2" + "description": "The endpoint to delete the evaluation for. If `None`, deletes the evaluations for all endpoints.", + "example": "gpt-4o-mini@openai" + }, + { + "name": "evaluator", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Name of the evaluator to delete the evaluation for. If `None`, deletes all available evaluations for the dataset and endpoint pair.", + "title": "Evaluator" + }, + "description": "Name of the evaluator to delete the evaluation for. If `None`, deletes all available evaluations for the dataset and endpoint pair.", + "example": "eval1" } ], "responses": { @@ -1599,14 +1751,14 @@ } } }, - "/v0/evals/delete": { - "delete": { + "/v0/evaluation/status": { + "get": { "tags": [ - "Dataset Evaluation" + "Evaluations" ], - "summary": "Delete Eval", - "description": "Deletes a named eval from your account.", - "operationId": "delete_eval_v0_evals_delete_delete", + "summary": "Eval Status", + "description": "Fetches the eval status on a given dataset. Returns object of the form:\n{\n \"responses\": {\n \"last_updated\": \"2024-08-19 13:58:20.866092\",\n \"num_failed\": 0,\n \"num_processed\": 3,\n \"num_remaining\": 0,\n },\n \"judgements\": {\n \"judge_model_a\": {\n \"last_updated\": \"2024-08-19 13:58:20.866092\",\n \"num_failed\": 0,\n \"num_processed\": 3,\n \"num_remaining\": 0,\n }\n },\n}", + "operationId": "eval_status_v0_evaluation_status_get", "security": [ { "HTTPBearer": [] @@ -1614,15 +1766,39 @@ ], "parameters": [ { - "name": "eval_name", + "name": "dataset", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Name of the dataset to get evaluation status of.", + "title": "Dataset" + }, + "description": "Name of the dataset to get evaluation status of.", + "example": "dataset1" + }, + { + "name": "endpoint", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Endpoint to get evaluation status of", + "title": "Endpoint" + }, + "description": "Endpoint to get evaluation status of", + "example": "llama-3-8b-chat@aws-bedrock" + }, + { + "name": "evaluator", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the eval to delete", - "title": "Eval Name" + "description": "Name of the evaluator to get status of.", + "title": "Evaluator" }, - "description": "Name of the eval to delete", + "description": "Name of the evaluator to get status of.", "example": "eval1" } ], @@ -1648,14 +1824,14 @@ } } }, - "/v0/evals/trigger": { + "/v0/benchmark": { "post": { "tags": [ - "Dataset Evaluation" + "Efficiency Benchmarks" ], - "summary": "Trigger Eval", - "description": "Uses the named `eval` to begin an evaluation of quality scores for the selected LLM `endpoint`, on the given `dataset`.\nYou can upload custom scores (and bypass the LLM judge) by uploading a file via the `client_side_scores` argument.\nOnce the evaluation has finished, you can access the scores using the `evals/get_scores` endpoint.", - "operationId": "trigger_eval_v0_evals_trigger_post", + "summary": "Append To Benchmark", + "description": "Append speed or cost data to the standardized time-series benchmarks for a custom\nendpoint (only custom endpoints are publishable by end users).", + "operationId": "append_to_benchmark_v0_benchmark_post", "security": [ { "HTTPBearer": [] @@ -1663,90 +1839,73 @@ ], "parameters": [ { - "name": "eval_name", + "name": "endpoint_name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the eval to use.", - "title": "Eval Name" + "description": "Name of the *custom* endpoint to append benchmark data for.", + "title": "Endpoint Name" }, - "description": "Name of the eval to use.", - "example": "eval1" + "description": "Name of the *custom* endpoint to append benchmark data for.", + "example": "my_endpoint" }, { - "name": "dataset", + "name": "metric_name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the uploaded dataset to evaluate.", - "title": "Dataset" + "description": "Name of the metric to submit. Allowed metrics are:\n \"input-cost\", \"output-cost\", \"time-to-first-token\", \"inter-token-latency\".", + "title": "Metric Name" }, - "description": "Name of the uploaded dataset to evaluate.", - "example": "dataset1" + "description": "Name of the metric to submit. Allowed metrics are:\n \"input-cost\", \"output-cost\", \"time-to-first-token\", \"inter-token-latency\".", + "example": "tokens-per-second" }, { - "name": "endpoint", + "name": "value", "in": "query", "required": true, + "schema": { + "type": "number", + "description": "Value of the metric to submit.", + "title": "Value" + }, + "description": "Value of the metric to submit.", + "example": 10 + }, + { + "name": "measured_at", + "in": "query", + "required": false, "schema": { "type": "string", - "description": "Name of the endpoint to evaluate. Endpoints must be specified using the `model@provider` format.", - "title": "Endpoint" + "format": "date-time", + "description": "The timestamp to associate with the submission. Defaults to current time if unspecified.", + "title": "Measured At" }, - "description": "Name of the endpoint to evaluate. Endpoints must be specified using the `model@provider` format.", - "example": "gpt-4o-mini@openai" + "description": "The timestamp to associate with the submission. Defaults to current time if unspecified.", + "example": "2024-08-12T04:20:32.808410" } ], - "requestBody": { - "content": { - "multipart/form-data": { - "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/Body_trigger_eval_v0_evals_trigger_post" - } - ], - "title": "Body" - } - } - } - }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "Response Trigger Eval V0 Evals Trigger Post" - }, + "schema": {}, "example": { - "info": "Dataset evaluation started! You will receive an email soon!" + "info": "Custom endpoint benchmark uploaded successfully!" } } } }, "400": { - "description": "Invalid Endpoints", - "content": { - "application/json": { - "example": { - "detail": "Invalid input. Couldn't find endpoints [model_1@endpoint_1, model_2@endpoint_2]." - } - } - } - }, - "404": { - "description": "Dataset Not Found", + "description": "Benchmark not valid", "content": { "application/json": { "example": { - "detail": "This dataset does not exist!" + "detail": "Invalid data submitted" } } } @@ -1762,16 +1921,14 @@ } } } - } - }, - "/v0/evals/get_scores": { + }, "get": { "tags": [ - "Dataset Evaluation" + "Efficiency Benchmarks" ], - "summary": "Get Eval Scores", - "description": "Fetches the results of an eval on a given dataset. If no `eval_name` is provided, returns scores for all completed evals on that dataset.", - "operationId": "get_eval_scores_v0_evals_get_scores_get", + "summary": "Get Benchmark", + "description": "Extracts cost and speed data for the provided endpoint via our standardized\nefficiency benchmarks, in the specified region, with the specified sequence length,\nwith all benchmark values returned within the specified time window.\n\nWhen extracting data for a *custom* endpoint, then `model` is the endpoint name, and\n`provider` must be set as `\"custom\"`. The arguments `region` and `seq_len` are\nignored for custom endpoints (they are not publishable).\n\nIf neither `start_time` nor `end_time` are provided, then only the *latest*\nbenchmark data is returned. If only `start_time` is provided, then `end_time` is\nassumed to be the current time. An exception is raised if only `end_time` is\nprovided.", + "operationId": "get_benchmark_v0_benchmark_get", "security": [ { "HTTPBearer": [] @@ -1779,66 +1936,78 @@ ], "parameters": [ { - "name": "dataset", + "name": "model", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the dataset to fetch evaluation from.", - "title": "Dataset" + "description": "Name of the model.", + "title": "Model" }, - "description": "Name of the dataset to fetch evaluation from.", - "example": "dataset1" + "description": "Name of the model.", + "example": "gpt-4o-mini" + }, + { + "name": "provider", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Name of the provider.", + "title": "Provider" + }, + "description": "Name of the provider.", + "example": "openai" }, { - "name": "eval_name", + "name": "region", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Name of the eval to fetch evaluation from. If `None`, returns all available evaluations for the dataset.", - "title": "Eval Name" + "type": "string", + "description": "Region where the benchmark is run.\n Options are: `\"Belgium\"`, `\"Hong Kong\"` or `\"Iowa\"`.", + "default": "Belgium", + "title": "Region" }, - "description": "Name of the eval to fetch evaluation from. If `None`, returns all available evaluations for the dataset.", - "example": "eval1" + "description": "Region where the benchmark is run.\n Options are: `\"Belgium\"`, `\"Hong Kong\"` or `\"Iowa\"`.", + "example": "Belgium" }, { - "name": "endpoint", + "name": "seq_len", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Name of endpoint to fetch eval from. If not set returns evals for all endpoints", - "title": "Endpoint" + "type": "string", + "description": "Length of the sequence used for benchmarking, can be short or long", + "default": "short", + "title": "Seq Len" }, - "description": "Name of endpoint to fetch eval from. If not set returns evals for all endpoints" + "description": "Length of the sequence used for benchmarking, can be short or long", + "example": "short" }, { - "name": "per_prompt", + "name": "start_time", "in": "query", "required": false, "schema": { - "type": "boolean", - "description": "If `True`, returns the scores on a per-prompt level. By default set to `False`. If `True` requires an eval name and endpoint to be set.", - "default": false, - "title": "Per Prompt" + "type": "string", + "description": "Window start time. Only returns the latest benchmark if unspecified", + "title": "Start Time" }, - "description": "If `True`, returns the scores on a per-prompt level. By default set to `False`. If `True` requires an eval name and endpoint to be set.", - "example": false + "description": "Window start time. Only returns the latest benchmark if unspecified", + "example": "2024-07-12T04:20:32.808410" + }, + { + "name": "end_time", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Window end time. Assumed to be the current time if this is unspecified *and* start_time *is* specified. Only the latest benchmark is returned if both are unspecified.", + "title": "End Time" + }, + "description": "Window end time. Assumed to be the current time if this is unspecified *and* start_time *is* specified. Only the latest benchmark is returned if both are unspecified.", + "example": "2024-08-12T04:20:32.808410" } ], "responses": { @@ -1847,8 +2016,28 @@ "content": { "application/json": { "schema": { - "type": "object", - "title": "Response Get Eval Scores V0 Evals Get Scores Get" + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "title": "Response Get Benchmark V0 Benchmark Get" + }, + "example": { + "ttft": 440.2323939998496, + "itl": 8.797065147959705, + "input_cost": 0.15, + "output_cost": 0.6, + "measured_at": "2024-08-17T19:19:37.289937" } } } @@ -1864,16 +2053,14 @@ } } } - } - }, - "/v0/evals/status": { - "get": { + }, + "delete": { "tags": [ - "Dataset Evaluation" + "Efficiency Benchmarks" ], - "summary": "Eval Status", - "description": "Fetches the eval status on a given dataset. Returns object of the form:\n\n```\n{\n \"responses\": {\n \"last_updated\": \"2024-08-19 13:58:20.866092\",\n \"num_failed\": 0,\n \"num_processed\": 3,\n \"num_remaining\": 0,\n },\n \"judgements\": {\n \"judge_model_a\": {\n \"last_updated\": \"2024-08-19 13:58:20.866092\",\n \"num_failed\": 0,\n \"num_processed\": 3,\n \"num_remaining\": 0,\n }\n },\n}\n```", - "operationId": "eval_status_v0_evals_status_get", + "summary": "Delete Benchmark", + "description": "Delete *all* benchmark time-series data for a given *custom* endpoint.\nThe time-series benchmark data for *public* endpoints are not deletable.", + "operationId": "delete_benchmark_v0_benchmark_delete", "security": [ { "HTTPBearer": [] @@ -1881,40 +2068,16 @@ ], "parameters": [ { - "name": "dataset", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Name of the dataset to get evaluation status of.", - "title": "Dataset" - }, - "description": "Name of the dataset to get evaluation status of.", - "example": "dataset1" - }, - { - "name": "endpoint", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Endpoint to get evaluation status of", - "title": "Endpoint" - }, - "description": "Endpoint to get evaluation status of", - "example": "llama-3-8b-chat@aws-bedrock" - }, - { - "name": "eval_name", + "name": "endpoint_name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Name of the eval to get evaluation status of.", - "title": "Eval Name" + "description": "Name of the *custom* endpoint to submit a benchmark for.", + "title": "Endpoint Name" }, - "description": "Name of the eval to get evaluation status of.", - "example": "eval1" + "description": "Name of the *custom* endpoint to submit a benchmark for.", + "example": "my_endpoint" } ], "responses": { @@ -1942,10 +2105,10 @@ "/v0/router/train": { "post": { "tags": [ - "Routing" + "Router Training" ], "summary": "Train Router", - "description": "Trains a router based on a dataset and a set of endpoints. To use a\ncustom-trained router, you will need to deploy the resulting artifacts to\na live endpoint. To do this, use the `/router/deploy` POST endpoint.", + "description": "Train a router based on a specified training dataset and a set of endpoints to route\nacross. To *use* a custom-trained router, you will need to deploy the resulting\nartifacts to a live endpoint, via the `/router/deploy` `POST` endpoint.", "operationId": "train_router_v0_router_train_post", "security": [ { @@ -2046,14 +2209,16 @@ } } } - }, + } + }, + "/v0/router": { "delete": { "tags": [ - "Routing" + "Router Training" ], - "summary": "Delete Router Train", - "description": "Deletes the training files of a specific router.", - "operationId": "delete_router_train_v0_router_train_delete", + "summary": "Delete Router", + "description": "Deletes a specific trained router, as well as all the training files etc.", + "operationId": "delete_router_v0_router_delete", "security": [ { "HTTPBearer": [] @@ -2069,8 +2234,88 @@ "description": "Name of the router to delete.", "title": "Name" }, - "description": "Name of the router to delete.", - "example": "router1" + "description": "Name of the router to delete.", + "example": "router1" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "Response Delete Router V0 Router Delete" + }, + "example": { + "info": "Trained router deleted!" + } + } + } + }, + "400": { + "description": "Router Training Does Not Exist", + "content": { + "application/json": { + "example": { + "detail": "This router training doesn't exist. Please, choose a different one or trigger the training first." + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/router/rename": { + "post": { + "tags": [ + "Router Training" + ], + "summary": "Rename Router", + "description": "Renames the specified router from `name` to `new_name`.", + "operationId": "rename_router_v0_router_rename_post", + "security": [ + { + "HTTPBearer": [] + } + ], + "parameters": [ + { + "name": "name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The original name of the router.", + "title": "Name" + }, + "description": "The original name of the router.", + "example": "original_name" + }, + { + "name": "new_name", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "The new name for the router.", + "title": "New Name" + }, + "description": "The new name for the router.", + "example": "new_name" } ], "responses": { @@ -2078,26 +2323,7 @@ "description": "Successful Response", "content": { "application/json": { - "schema": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "Response Delete Router Train V0 Router Train Delete" - }, - "example": { - "info": "Trained router deleted!" - } - } - } - }, - "400": { - "description": "Router Training Does Not Exist", - "content": { - "application/json": { - "example": { - "detail": "This router training doesn't exist. Please, choose a different one or trigger the training first." - } + "schema": {} } } }, @@ -2114,14 +2340,14 @@ } } }, - "/v0/router/train/list": { + "/v0/router/list": { "get": { "tags": [ - "Routing" + "Router Training" ], - "summary": "Get Trained Routers", - "description": "Fetches a list of the trained routers and relevant metadata. These routers are training\nartifacts and therefore don't imply an active, deployed router. To fetch a list of\ndeployed routers, you can use the /router/deploy/list GET endpoint.", - "operationId": "get_trained_routers_v0_router_train_list_get", + "summary": "List Routers", + "description": "Lists all the trained routers and the relevant metadata.\nThese routers are training artifacts and therefore don't imply an active,\ndeployed router. To fetch a list of deployed routers, you can use the\n`/router/deploy/list` `GET` endpoint.", + "operationId": "list_routers_v0_router_list_get", "responses": { "200": { "description": "Successful Response", @@ -2145,7 +2371,7 @@ "type": "object" }, "type": "object", - "title": "Response Get Trained Routers V0 Router Train List Get" + "title": "Response List Routers V0 Router List Get" }, "example": { "router_1": { @@ -2173,7 +2399,7 @@ "/v0/router/deploy": { "post": { "tags": [ - "Routing" + "Router Deployment" ], "summary": "Deploy Router", "description": "Deploys a trained router to a live endpoint.\n\nTo use this router, replace the model in the endpoint string with the\nrouter name. E.g. you can use `router-abc` by calling the\n`router-abc@q:1` endpoint.", @@ -2239,11 +2465,11 @@ }, "delete": { "tags": [ - "Routing" + "Router Deployment" ], - "summary": "Delete Router", - "description": "Deactivates and deletes a previously deployed router.", - "operationId": "delete_router_v0_router_deploy_delete", + "summary": "Undeploy Router", + "description": "Deactivates and deletes a previously deployed router, but keeps the training\nartifacts for this router, such that it can be redeployed if desired without needing\nto retrain.", + "operationId": "undeploy_router_v0_router_deploy_delete", "security": [ { "HTTPBearer": [] @@ -2273,7 +2499,7 @@ "additionalProperties": { "type": "string" }, - "title": "Response Delete Router V0 Router Deploy Delete" + "title": "Response Undeploy Router V0 Router Deploy Delete" }, "example": { "info": "Router deletion started! You will receive an email soon!" @@ -2307,11 +2533,11 @@ "/v0/router/deploy/list": { "get": { "tags": [ - "Routing" + "Router Deployment" ], - "summary": "Get Deployed Routers", - "description": "Fetches a list of the deployed routers and relevant metadata. These routers only\ninclude deployed routers. To fetch a list of all trained routers,\nyou can use the /router/train/list GET endpoint.\n\nTo use any of these routers, replace the model in the endpoint string with the\nrouter name. E.g. you can use `router-abc` with the endpoint `router-abc@q:1`.", - "operationId": "get_deployed_routers_v0_router_deploy_list_get", + "summary": "List Deployed Routers", + "description": "Fetches a list of the *deployed* routers and relevant metadata (excluding the\ntrained but undeployed routers). To fetch a list of *all* trained routers (both\ndeployed and undeployed), you can use the `/router/list` `GET` endpoint.\n\nTo use any of these routers, replace the model in the endpoint string with the\nrouter name. E.g. you can use `router-abc` with the endpoint `router-abc@q:1`.", + "operationId": "list_deployed_routers_v0_router_deploy_list_get", "responses": { "200": { "description": "Successful Response", @@ -2335,7 +2561,7 @@ "type": "object" }, "type": "object", - "title": "Response Get Deployed Routers V0 Router Deploy List Get" + "title": "Response List Deployed Routers V0 Router Deploy List Get" }, "example": { "router_1": { @@ -2360,14 +2586,14 @@ ] } }, - "/v0/endpoints": { - "get": { + "/v0/router/config": { + "post": { "tags": [ - "Model and Endpoints" + "Router Configurations" ], - "summary": "Get Endpoints", - "description": "Lists available endpoints in `model@provider` format. If `model` or `provider` are specified,\nonly the matching endpoints will be listed.", - "operationId": "get_endpoints_v0_endpoints_get", + "summary": "Create Router Config", + "description": "Creates a router configuration, which can be queried later using only the\nconfiguration name, such as `my_router_config@routers`.", + "operationId": "create_router_config_v0_router_config_post", "security": [ { "HTTPBearer": [] @@ -2375,28 +2601,28 @@ ], "parameters": [ { - "name": "model", + "name": "config_name", "in": "query", - "required": false, + "required": true, "schema": { "type": "string", - "description": "Model to get available endpoints from.", - "title": "Model" + "description": "The name of the router configuration to create.", + "title": "Config Name" }, - "description": "Model to get available endpoints from.", - "example": "llama-3.1-405b-chat" + "description": "The name of the router configuration to create.", + "example": "cost_and_speed_optimized" }, { - "name": "provider", + "name": "router_endpoint", "in": "query", - "required": false, + "required": true, "schema": { "type": "string", - "description": "Provider to get available endpoints from.", - "title": "Provider" + "description": "The raw string which fully defines the router endpoint, with all constraints applied, including the router name and any extra arguments.", + "title": "Router Endpoint" }, - "description": "Provider to get available endpoints from.", - "example": "openai" + "description": "The raw string which fully defines the router endpoint, with all constraints applied, including the router name and any extra arguments.", + "example": "router1|models:llama-3.1-8b-chat,mixtral-8x22b-instruct-v0.1|providers:fireworks-ai,together-ai|q:1|c:4.65e-03|t:2.08e-05|i:2.07e-03@routers" } ], "responses": { @@ -2404,18 +2630,7 @@ "description": "Successful Response", "content": { "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Response Get Endpoints V0 Endpoints Get" - }, - "example": [ - "model_a@provider_1", - "model_a@provider_2", - "..." - ] + "schema": {} } } }, @@ -2430,16 +2645,14 @@ } } } - } - }, - "/v0/providers": { + }, "get": { "tags": [ - "Model and Endpoints" + "Router Configurations" ], - "summary": "Get Providers", - "description": "Lists available providers. If a model is specified, returns the providers that support that model.", - "operationId": "get_providers_v0_providers_get", + "summary": "Get Router Config", + "description": "Returns the full router endpoint string for a given router configuration name.", + "operationId": "get_router_config_v0_router_config_get", "security": [ { "HTTPBearer": [] @@ -2447,16 +2660,16 @@ ], "parameters": [ { - "name": "model", + "name": "config_name", "in": "query", - "required": false, + "required": true, "schema": { "type": "string", - "description": "Model to get available providers from.", - "title": "Model" + "description": "The name of the router configuration to retrieve the full endpoint string for.", + "title": "Config Name" }, - "description": "Model to get available providers from.", - "example": "llama-3.1-405b-chat" + "description": "The name of the router configuration to retrieve the full endpoint string for.", + "example": "cost_and_speed_optimized" } ], "responses": { @@ -2465,17 +2678,9 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Response Get Providers V0 Providers Get" - }, - "example": [ - "provider_1", - "provider_2", - "..." - ] + "type": "string", + "title": "Response Get Router Config V0 Router Config Get" + } } } }, @@ -2490,15 +2695,14 @@ } } } - } - }, - "/v0/benchmarks": { - "get": { + }, + "delete": { "tags": [ - "benchmarks" + "Router Configurations" ], - "summary": "Get Latest Benchmark", - "operationId": "get_latest_benchmark_v0_benchmarks_get", + "summary": "Delete Router Config", + "description": "Deletes the specified router configuration from your account.", + "operationId": "delete_router_config_v0_router_config_delete", "security": [ { "HTTPBearer": [] @@ -2506,65 +2710,16 @@ ], "parameters": [ { - "name": "model", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Model name", - "title": "Model" - }, - "description": "Model name", - "example": "gpt-4o-mini" - }, - { - "name": "provider", + "name": "config_name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Provider name", - "title": "Provider" - }, - "description": "Provider name", - "example": "openai" - }, - { - "name": "regime", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "concurrent-1", - "title": "Regime" - }, - "example": "concurrent-1" - }, - { - "name": "region", - "in": "query", - "required": false, - "schema": { - "type": "string", - "description": "Region where the benchmark is run. Options are: \"Belgium\", \"Hong Kong\", \"Iowa\".", - "default": "Belgium", - "title": "Region" - }, - "description": "Region where the benchmark is run. Options are: \"Belgium\", \"Hong Kong\", \"Iowa\".", - "example": "Belgium" - }, - { - "name": "seq_len", - "in": "query", - "required": false, - "schema": { - "type": "string", - "description": "Length of the sequence used for benchmarking, can be short or long", - "default": "short", - "title": "Seq Len" + "description": "The name of the router configuration to delete.", + "title": "Config Name" }, - "description": "Length of the sequence used for benchmarking, can be short or long", - "example": "short" + "description": "The name of the router configuration to delete.", + "example": "cost_and_speed_optimized" } ], "responses": { @@ -2589,13 +2744,14 @@ } } }, - "/v0/benchmarks/filter": { + "/v0/router/config/rename": { "post": { "tags": [ - "benchmarks" + "Router Configurations" ], - "summary": "Filter Benchmark", - "operationId": "filter_benchmark_v0_benchmarks_filter_post", + "summary": "Rename Router Config", + "description": "Renames the specified router configuration in your account.", + "operationId": "rename_router_config_v0_router_config_rename_post", "security": [ { "HTTPBearer": [] @@ -2603,89 +2759,28 @@ ], "parameters": [ { - "name": "model", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Model name", - "title": "Model" - }, - "description": "Model name", - "example": "gpt-4o-mini" - }, - { - "name": "provider", - "in": "query", - "required": true, - "schema": { - "type": "string", - "description": "Provider name", - "title": "Provider" - }, - "description": "Provider name", - "example": "openai" - }, - { - "name": "start_time", + "name": "name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Window start time", - "title": "Start Time" + "description": "The original name of the router configuration.", + "title": "Name" }, - "description": "Window start time", - "example": "2024-07-12T04:20:32.808410" + "description": "The original name of the router configuration.", + "example": "original_name" }, { - "name": "end_time", + "name": "new_name", "in": "query", "required": true, "schema": { "type": "string", - "description": "Window end time", - "title": "End Time" - }, - "description": "Window end time", - "example": "2024-08-12T04:20:32.808410" - }, - { - "name": "regime", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "concurrent-1", - "title": "Regime" - }, - "example": "concurrent-1" - }, - { - "name": "region", - "in": "query", - "required": false, - "schema": { - "type": "string", - "description": "Region where the benchmark is run. Options are: \"Belgium\", \"Hong Kong\", \"Iowa\".", - "default": "Belgium", - "title": "Region" - }, - "description": "Region where the benchmark is run. Options are: \"Belgium\", \"Hong Kong\", \"Iowa\".", - "example": "Belgium" - }, - { - "name": "seq_len", - "in": "query", - "required": false, - "schema": { - "type": "string", - "description": "Length of the sequence used for benchmarking. Options are: \"short\", \"long\".", - "default": "short", - "title": "Seq Len" + "description": "The new name for the router configuration.", + "title": "New Name" }, - "description": "Length of the sequence used for benchmarking. Options are: \"short\", \"long\".", - "example": "short" + "description": "The new name for the router configuration.", + "example": "new_name" } ], "responses": { @@ -2710,13 +2805,76 @@ } } }, - "/v0/prompt_history": { + "/v0/router/config/list": { "get": { "tags": [ - "benchmarks" + "Router Configurations" ], - "summary": "Get Prompt History", - "operationId": "get_prompt_history_v0_prompt_history_get", + "summary": "List Router Configs", + "description": "Lists all saved router configurations by name.", + "operationId": "list_router_configs_v0_router_config_list_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Response List Router Configs V0 Router Config List Get" + } + } + } + } + }, + "security": [ + { + "HTTPBearer": [] + } + ] + } + }, + "/v0/credits": { + "get": { + "tags": [ + "Credits" + ], + "summary": "Get Credits", + "description": "Returns the number of available credits.", + "operationId": "get_credits_v0_credits_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreditsResponse" + }, + "example": { + "id": "", + "credits": 10 + } + } + } + } + }, + "security": [ + { + "HTTPBearer": [] + } + ] + } + }, + "/v0/promo": { + "post": { + "tags": [ + "Credits" + ], + "summary": "Promo Code", + "description": "Activates a promotional code.", + "operationId": "promo_code_v0_promo_post", "security": [ { "HTTPBearer": [] @@ -2724,22 +2882,28 @@ ], "parameters": [ { - "name": "tag", + "name": "code", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Promo code to be activated.", + "title": "Code" + }, + "description": "Promo code to be activated.", + "example": "sample_code" + }, + { + "name": "user", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Provide a tag to filter by prompts that are marked with this tag.", - "title": "Tag" + "type": "string", + "description": "ID of the user that receives the credits,defaults to the user making the request.", + "title": "User" }, - "description": "Provide a tag to filter by prompts that are marked with this tag." + "description": "ID of the user that receives the credits,defaults to the user making the request.", + "example": "sample_user_id" } ], "responses": { @@ -2747,7 +2911,36 @@ "description": "Successful Response", "content": { "application/json": { - "schema": {} + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "Response Promo Code V0 Promo Post" + }, + "example": { + "info": "Code {code} activated successfully!" + } + } + } + }, + "400": { + "description": "Already activated code", + "content": { + "application/json": { + "example": { + "detail": "This code has already been activated." + } + } + } + }, + "404": { + "description": "Code Not Found", + "content": { + "application/json": { + "example": { + "detail": "Invalid code." + } } } }, @@ -2767,25 +2960,18 @@ }, "components": { "schemas": { - "Body_trigger_eval_v0_evals_trigger_post": { + "Body_trigger_evaluation_v0_evaluation_post": { "properties": { "client_side_scores": { - "anyOf": [ - { - "type": "string", - "format": "binary" - }, - { - "type": "null" - } - ], + "type": "string", + "format": "binary", "title": "Client Side Scores", - "description": "An optional file upload for client-side scores. The file must be in JSONL format and the prompts must match the order of the `dataset`. Each entry should include `prompt` and `score` keys, with `score` being a float between 0 and 1. The evaluation corresponding to the `eval_name` must have `client_side=True`.", + "description": "An optional file upload for client-side scores. The file must be in JSONL format and the prompts must match the order of the `dataset`. Each entry should include `prompt` and `score` keys, with `score` being a float between 0 and 1. The evaluation corresponding to the `evaluator` must have `client_side=True`.", "example": "client_scores.jsonl" } }, "type": "object", - "title": "Body_trigger_eval_v0_evals_trigger_post" + "title": "Body_trigger_evaluation_v0_evaluation_post" }, "Body_upload_dataset_v0_dataset_post": { "properties": { @@ -2793,11 +2979,13 @@ "type": "string", "format": "binary", "title": "File", + "description": "The contents of the `.jsonl` file being uploaded.", "example": "dataset.jsonl" }, "name": { "type": "string", "title": "Name", + "description": "The name to give to this dataset.", "example": "dataset1" } }, @@ -2810,17 +2998,13 @@ }, "ChatCompletionRequest": { "properties": { - "model": { - "type": "string", - "title": "Model", - "example": "gpt-4o-mini@openai" - }, "messages": { "items": { "type": "object" }, "type": "array", "title": "Messages", + "description": "A list of messages comprising the conversation so far.", "example": [ { "content": "Tell me a joke", @@ -2828,151 +3012,129 @@ } ] }, - "temperature": { - "type": "number", - "title": "Temperature", - "default": 0.9, - "example": 0.9 - }, - "stream": { - "type": "boolean", - "title": "Stream", - "default": false, - "example": false + "model": { + "type": "string", + "title": "Model", + "description": "The endpoint to use, in the format `{model}@{provider}`, based on any of the supported endpoints as per the list returned by `/v0/endpoints`", + "example": "gpt-4o-mini@openai" }, "max_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], + "type": "integer", "title": "Max Tokens", + "description": "The maximum number of tokens that can be generated in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length.", "example": 1024 }, - "frequency_penalty": { + "stop": { "anyOf": [ { - "type": "number" + "type": "string" }, { - "type": "null" - } - ], - "title": "Frequency Penalty" - }, - "logit_bias": { - "anyOf": [ - { - "additionalProperties": { - "type": "number" + "items": { + "type": "string" }, - "type": "object" + "type": "array" }, { "type": "null" } ], - "title": "Logit Bias" + "title": "Stop", + "description": "Up to 4 sequences where the API will stop generating further tokens.", + "example": [ + "The End.", + " is the answer." + ] + }, + "stream": { + "type": "boolean", + "title": "Stream", + "description": "If set, partial message deltas will be sent. Tokens will be sent as data-only server-sent events as they become available, with the stream terminated by a `data: [DONE]` message.", + "default": false, + "example": false + }, + "temperature": { + "type": "number", + "title": "Temperature", + "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nGenerally recommended to alter this or `top_p`, but not both.", + "default": 1.0, + "example": 0.9 + }, + "frequency_penalty": { + "type": "number", + "title": "Frequency Penalty", + "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.", + "example": 1.5 + }, + "logit_bias": { + "additionalProperties": { + "type": "number" + }, + "type": "object", + "title": "Logit Bias", + "description": "Modify the likelihood of specified tokens appearing in the completion.\n\nAccepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token.", + "example": { + "0": 10, + "1": -75, + "2": 90 + } }, "logprobs": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "title": "Logprobs" + "type": "boolean", + "title": "Logprobs", + "description": "Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the `content` of `message`.", + "example": false }, "top_logprobs": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Top Logprobs" + "type": "integer", + "title": "Top Logprobs", + "description": "An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used.", + "example": 15 }, "n": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "N" + "type": "integer", + "title": "N", + "description": "How many chat completion choices to generate for each input message. Note that you will be charged based on the number of generated tokens across all of the choices. Keep `n` as `1` to minimize costs.", + "example": 15 }, "presence_penalty": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Presence Penalty" + "type": "number", + "title": "Presence Penalty", + "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.", + "example": -1.1 }, "response_format": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Response Format" + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "Response Format", + "description": "An object specifying the format that the model must output.\n\nSetting to `{ \"type\": \"json_schema\", \"json_schema\": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema.\n\nSetting to `{ \"type\": \"json_object\" }` enables JSON mode, which ensures the message the model generates is valid JSON.\n\n**Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly \"stuck\" request. Also note that the message content may be partially cut off if `finish_reason=\"length\"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length.", + "example": "{ \"type\": \"json_mode\"}" }, "seed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Seed" + "type": "integer", + "title": "Seed", + "description": "If specified, the system will make a best effort to sample deterministically, such that repeated requests with the same `seed` and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend.", + "example": 11 }, - "stop": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Stop" + "stream_options": { + "additionalProperties": { + "type": "boolean" + }, + "type": "object", + "title": "Stream Options", + "description": "Options for streaming response. Only set this when you set `stream: true`.", + "example": [ + true, + "include_usage" + ] }, "top_p": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ], - "title": "Top P" + "type": "number", + "title": "Top P", + "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nGenerally recommended to alter this or `temperature` but not both.", + "example": 0.5 }, "tools": { "anyOf": [ @@ -2984,7 +3146,8 @@ "type": "null" } ], - "title": "Tools" + "title": "Tools", + "description": "A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. A max of 128 functions are supported." }, "tool_choice": { "anyOf": [ @@ -2993,48 +3156,50 @@ "type": "null" } ], - "title": "Tool Choice" + "title": "Tool Choice", + "description": "Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message.`auto` means the model can pick between generating a message or calling one or more tools. `required` means the model must call one or more tools. Specifying a particular tool via `{\"type\": \"function\", \"function\": {\"name\": \"my_function\"}}` forces the model to call that tool.\n\n`none` is the default when no tools are present. `auto` is the default if tools are present.", + "example": "{\"type\": \"function\", \"function\": {\"name\": \"my_function\"}}" + }, + "parallel_tool_calls": { + "type": "boolean", + "title": "Parallel Tool Calls", + "description": "Whether to enable parallel function calling during tool use.", + "default": true, + "example": true }, "user": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "User" + "type": "string", + "title": "User", + "description": "A unique identifier representing your end-user.", + "example": "some_user" }, "signature": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Signature" + "type": "string", + "title": "Signature", + "description": "A string used to represent where the request came from, for examples, did it come via the Python package, the NodeJS package, the chat interface etc. This should *not* be set by the user.", + "example": "python" }, "use_custom_keys": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], + "type": "boolean", "title": "Use Custom Keys", - "default": false + "description": "Whether or not to use custom API keys with the specified provider, meaning that you will be using your own account with that provider in the backend.", + "example": true + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Tags", + "description": "Whether or not to use custom API keys with the specified provider, meaning that you will be using your own account with that provider in the backend.", + "example": true } }, "additionalProperties": true, "type": "object", "required": [ - "model", - "messages" + "messages", + "model" ], "title": "ChatCompletionRequest", "description": "Request model for chat completion based language model.\n\nAttributes:\n model (str): The model identifier.\n messages (List[Dict[str]]): List of messages for completion.\n temperature (float): The temperature parameter for generating completions.\n stream (bool): Whether to stream the response." @@ -3130,40 +3295,12 @@ ], "title": "CustomApiKeyModelResponse" }, - "CustomEndpointModelResponse": { + "EvaluatorConfig": { "properties": { "name": { "type": "string", - "title": "Name" - }, - "mdl_name": { - "type": "string", - "title": "Mdl Name" - }, - "url": { - "type": "string", - "title": "Url" - }, - "key": { - "type": "string", - "title": "Key" - } - }, - "type": "object", - "required": [ - "name", - "mdl_name", - "url", - "key" - ], - "title": "CustomEndpointModelResponse" - }, - "EvalConfig": { - "properties": { - "eval_name": { - "type": "string", - "title": "Eval Name", - "description": "A unique, user-defined name used when referencing and triggering the eval.", + "title": "Name", + "description": "A unique, user-defined name used when referencing and triggering the evaluation.", "example": "eval1" }, "system_prompt": { @@ -3189,7 +3326,7 @@ } ], "title": "Class Config", - "description": "If set, describes the list of classifications that the LLM judge uses to score each prompt. For example:\n```\n[{\"label\": \"Excellent\", \"score\": 1.0, \"description\": \"A perfect answer with no factual mistakes\"},\n{\"label\": \"Good\", \"score\": 0.5, \"description\": \"An average answer\"},\n{\"label\": \"Bad\", \"score\": 0.0, \"description\": \"An incorrect answer, containing a significant factual mistake\"}]\n```\n" + "description": "If set, describes the list of classifications that the LLM judge uses to\n score each prompt. For example:\n```\n[{\"label\": \"Excellent\", \"score\": 1.0, \"description\": \"A perfect answer with no factual\nmistakes\"},\n{\"label\": \"Good\", \"score\": 0.5, \"description\": \"An average answer\"},\n{\"label\": \"Bad\", \"score\": 0.0, \"description\": \"An incorrect answer, containing a\nsignificant factual mistake\"}]\n```\n" }, "judge_models": { "anyOf": [ @@ -3204,7 +3341,7 @@ } ], "title": "Judge Models", - "description": "Specifies the LLM(s) to be used as the judge. This can be a string containining a single model name or a list of model names.", + "description": "Specifies the LLM(s) to be used as the judge. This can be a string containing a single model name or a list of model names.", "default": "claude-3.5-sonnet@aws-bedrock", "example": "claude-3.5-sonnet@aws-bedrock" }, @@ -3217,9 +3354,9 @@ }, "type": "object", "required": [ - "eval_name" + "name" ], - "title": "EvalConfig" + "title": "EvaluatorConfig" }, "HTTPValidationError": { "properties": { @@ -3234,41 +3371,6 @@ "type": "object", "title": "HTTPValidationError" }, - "InferenceRequest": { - "properties": { - "model": { - "type": "string", - "title": "Model" - }, - "provider": { - "type": "string", - "title": "Provider" - }, - "arguments": { - "type": "object", - "title": "Arguments" - }, - "signature": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Signature" - } - }, - "type": "object", - "required": [ - "model", - "provider", - "arguments" - ], - "title": "InferenceRequest", - "description": "Request model for any model in the hub.\nAttributes:\n model (str): The model identifier.\n provider (str): The provider identifier.\n arguments (Dict[str, Any]]): Model-specific arguments." - }, "ValidationError": { "properties": { "loc": { diff --git a/api-reference/querying_llms/get_completions.mdx b/api-reference/querying_llms/get_completions.mdx deleted file mode 100644 index e221d4991..000000000 --- a/api-reference/querying_llms/get_completions.mdx +++ /dev/null @@ -1,176 +0,0 @@ ---- -title: 'Get Completions' -api: 'POST /v0/chat/completions' ---- -OpenAI compatible /chat/completions endpoint for LLM inference. - -#### Authorizations - - - Bearer authentication header of the form `Bearer `, where `` is your auth token. - - -#### Body - - - -
- -[Unified Arguments](https://docs.unify.ai/universal_api/arguments#unified-arguments) - - - - - - - - - - - - - - - - - - - - - - - -
- -[Partially Unified Arguments](https://docs.unify.ai/universal_api/arguments#partially-unified-arguments) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -[Platform Arguments](https://docs.unify.ai/universal_api/arguments#platform-arguments) - - - - - - - - - - - -```bash cURL -curl --request POST \ - --url 'https://api.unify.ai/v0/chat/completions' \ - --header 'Authorization: Bearer ' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "gpt-4o-mini@openai", - "messages": [ - { - "content": "Tell me a joke", - "role": "user" - } - ], - "temperature": 0.9, - "stream": false, - "max_tokens": 1024 -}' -``` - -```python Python -import requests - -url = "https://api.unify.ai/v0/chat/completions" - -headers = {"Authorization": "Bearer "} - -json_input = {"model": "gpt-4o-mini@openai", "messages": [{"content": "Tell me a joke", "role": "user"}], "temperature": 0.9, "stream": False, "max_tokens": 1024} - -response = requests.request("POST", url, json=json_input, headers=headers) - -print(response.text) -``` - - - - -```json 200 -{ - "model": "string", - "created": "integer | null", - "id": "string | null", - "object": "string", - "usage": "object", - "choices": "[object]" -} -``` - -```json 422 -{ - "detail": [ - { - "loc": [ - "string" - ], - "msg": "string", - "type": "string" - } - ] -} -``` - - diff --git a/api-reference/querying_llms/post_inference.mdx b/api-reference/querying_llms/post_inference.mdx deleted file mode 100644 index 74ca2d5d1..000000000 --- a/api-reference/querying_llms/post_inference.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: 'Post Inference' -api: 'POST /v0/inference' ---- -DEPRECATION WARNING: This endpoint is deprecated and will be removed. Instead, -use the /chat/completions endpoint for LLM inference. - -#### Authorizations - - - Bearer authentication header of the form `Bearer `, where `` is your auth token. - - -#### Body - - - - - - - - - - - - - - - - - - - -```bash cURL -curl --request POST \ - --url 'https://api.unify.ai/v0/inference' \ - --header 'Authorization: Bearer ' \ - --header 'Content-Type: application/json' \ - --data '{}' -``` - -```python Python -import requests - -url = "https://api.unify.ai/v0/inference" - -headers = {"Authorization": "Bearer "} - -json_input = {} - -response = requests.request("POST", url, json=json_input, headers=headers) - -print(response.text) -``` - - - - -```json 200 -{} -``` - -```json 422 -{ - "detail": [ - { - "loc": [ - "string" - ], - "msg": "string", - "type": "string" - } - ] -} -``` - - diff --git a/api-reference/router_configurations/create_router_config.mdx b/api-reference/router_configurations/create_router_config.mdx new file mode 100644 index 000000000..7013cede8 --- /dev/null +++ b/api-reference/router_configurations/create_router_config.mdx @@ -0,0 +1,65 @@ +--- +title: 'Create Router Config' +api: 'POST /v0/router/config' +--- +Creates a router configuration, which can be queried later using only the +configuration name, such as `my_router_config@routers`. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +The name of the router configuration to create. + + + +The raw string which fully defines the router endpoint, with all constraints applied, including the router name and any extra arguments. + + + + +```bash cURL +curl --request POST \ + --url 'https://api.unify.ai/v0/router/config?config_name=cost_and_speed_optimized&router_endpoint=router1|models:llama-3.1-8b-chat,mixtral-8x22b-instruct-v0.1|providers:fireworks-ai,together-ai|q:1|c:4.65e-03|t:2.08e-05|i:2.07e-03@routers' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/router/config?config_name=cost_and_speed_optimized&router_endpoint=router1|models:llama-3.1-8b-chat,mixtral-8x22b-instruct-v0.1|providers:fireworks-ai,together-ai|q:1|c:4.65e-03|t:2.08e-05|i:2.07e-03@routers" + +headers = {"Authorization": "Bearer "} + +response = requests.request("POST", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/router_configurations/delete_router_config.mdx b/api-reference/router_configurations/delete_router_config.mdx new file mode 100644 index 000000000..fcd805340 --- /dev/null +++ b/api-reference/router_configurations/delete_router_config.mdx @@ -0,0 +1,60 @@ +--- +title: 'Delete Router Config' +api: 'DELETE /v0/router/config' +--- +Deletes the specified router configuration from your account. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +The name of the router configuration to delete. + + + + +```bash cURL +curl --request DELETE \ + --url 'https://api.unify.ai/v0/router/config?config_name=cost_and_speed_optimized' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/router/config?config_name=cost_and_speed_optimized" + +headers = {"Authorization": "Bearer "} + +response = requests.request("DELETE", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/router_configurations/get_router_config.mdx b/api-reference/router_configurations/get_router_config.mdx new file mode 100644 index 000000000..1bb32a2a1 --- /dev/null +++ b/api-reference/router_configurations/get_router_config.mdx @@ -0,0 +1,60 @@ +--- +title: 'Get Router Config' +api: 'GET /v0/router/config' +--- +Returns the full router endpoint string for a given router configuration name. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +The name of the router configuration to retrieve the full endpoint string for. + + + + +```bash cURL +curl --request GET \ + --url 'https://api.unify.ai/v0/router/config?config_name=cost_and_speed_optimized' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/router/config?config_name=cost_and_speed_optimized" + +headers = {"Authorization": "Bearer "} + +response = requests.request("GET", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/custom_endpoints_and_api_keys/get_custom_endpoints.mdx b/api-reference/router_configurations/list_router_configs.mdx similarity index 64% rename from api-reference/custom_endpoints_and_api_keys/get_custom_endpoints.mdx rename to api-reference/router_configurations/list_router_configs.mdx index c85d3d6e1..179cb43fc 100644 --- a/api-reference/custom_endpoints_and_api_keys/get_custom_endpoints.mdx +++ b/api-reference/router_configurations/list_router_configs.mdx @@ -1,8 +1,8 @@ --- -title: 'Get Custom Endpoints' -api: 'GET /v0/custom_endpoint' +title: 'List Router Configs' +api: 'GET /v0/router/config/list' --- -Returns a list of the available custom endpoints. +Lists all saved router configurations by name. #### Authorizations @@ -14,14 +14,14 @@ Returns a list of the available custom endpoints. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/custom_endpoint' \ + --url 'https://api.unify.ai/v0/router/config/list' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/custom_endpoint" +url = "https://api.unify.ai/v0/router/config/list" headers = {"Authorization": "Bearer "} @@ -34,12 +34,7 @@ print(response.text) ```json 200 -{ - "name": "string", - "mdl_name": "string", - "url": "string", - "key": "string" -} +{} ``` diff --git a/api-reference/router_configurations/rename_router_config.mdx b/api-reference/router_configurations/rename_router_config.mdx new file mode 100644 index 000000000..bda6f0825 --- /dev/null +++ b/api-reference/router_configurations/rename_router_config.mdx @@ -0,0 +1,64 @@ +--- +title: 'Rename Router Config' +api: 'POST /v0/router/config/rename' +--- +Renames the specified router configuration in your account. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +The original name of the router configuration. + + + +The new name for the router configuration. + + + + +```bash cURL +curl --request POST \ + --url 'https://api.unify.ai/v0/router/config/rename?name=original_name&new_name=new_name' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/router/config/rename?name=original_name&new_name=new_name" + +headers = {"Authorization": "Bearer "} + +response = requests.request("POST", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/routing/deploy_router.mdx b/api-reference/router_deployment/deploy_router.mdx similarity index 100% rename from api-reference/routing/deploy_router.mdx rename to api-reference/router_deployment/deploy_router.mdx diff --git a/api-reference/routing/get_deployed_routers.mdx b/api-reference/router_deployment/list_deployed_routers.mdx similarity index 79% rename from api-reference/routing/get_deployed_routers.mdx rename to api-reference/router_deployment/list_deployed_routers.mdx index bd7fab147..b01654a57 100644 --- a/api-reference/routing/get_deployed_routers.mdx +++ b/api-reference/router_deployment/list_deployed_routers.mdx @@ -1,10 +1,10 @@ --- -title: 'Get Deployed Routers' +title: 'List Deployed Routers' api: 'GET /v0/router/deploy/list' --- -Fetches a list of the deployed routers and relevant metadata. These routers only -include deployed routers. To fetch a list of all trained routers, -you can use the /router/train/list GET endpoint. +Fetches a list of the *deployed* routers and relevant metadata (excluding the +trained but undeployed routers). To fetch a list of *all* trained routers (both +deployed and undeployed), you can use the `/router/list` `GET` endpoint. To use any of these routers, replace the model in the endpoint string with the router name. E.g. you can use `router-abc` with the endpoint `router-abc@q:1`. diff --git a/api-reference/routing/delete_router.mdx b/api-reference/router_deployment/undeploy_router.mdx similarity index 85% rename from api-reference/routing/delete_router.mdx rename to api-reference/router_deployment/undeploy_router.mdx index 2348b19a2..1ad84e9c0 100644 --- a/api-reference/routing/delete_router.mdx +++ b/api-reference/router_deployment/undeploy_router.mdx @@ -1,8 +1,10 @@ --- -title: 'Delete Router' +title: 'Undeploy Router' api: 'DELETE /v0/router/deploy' --- -Deactivates and deletes a previously deployed router. +Deactivates and deletes a previously deployed router, but keeps the training +artifacts for this router, such that it can be redeployed if desired without needing +to retrain. #### Authorizations diff --git a/api-reference/routing/delete_router_train.mdx b/api-reference/router_training/delete_router.mdx similarity index 81% rename from api-reference/routing/delete_router_train.mdx rename to api-reference/router_training/delete_router.mdx index dea45e9c6..a858588a9 100644 --- a/api-reference/routing/delete_router_train.mdx +++ b/api-reference/router_training/delete_router.mdx @@ -1,8 +1,8 @@ --- -title: 'Delete Router Train' -api: 'DELETE /v0/router/train' +title: 'Delete Router' +api: 'DELETE /v0/router' --- -Deletes the training files of a specific router. +Deletes a specific trained router, as well as all the training files etc. #### Authorizations @@ -20,14 +20,14 @@ Name of the router to delete. ```bash cURL curl --request DELETE \ - --url 'https://api.unify.ai/v0/router/train?name=router1' \ + --url 'https://api.unify.ai/v0/router?name=router1' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/router/train?name=router1" +url = "https://api.unify.ai/v0/router?name=router1" headers = {"Authorization": "Bearer "} diff --git a/api-reference/routing/get_trained_routers.mdx b/api-reference/router_training/list_routers.mdx similarity index 64% rename from api-reference/routing/get_trained_routers.mdx rename to api-reference/router_training/list_routers.mdx index 61c5dbf72..a18ea3c63 100644 --- a/api-reference/routing/get_trained_routers.mdx +++ b/api-reference/router_training/list_routers.mdx @@ -1,10 +1,11 @@ --- -title: 'Get Trained Routers' -api: 'GET /v0/router/train/list' +title: 'List Routers' +api: 'GET /v0/router/list' --- -Fetches a list of the trained routers and relevant metadata. These routers are training -artifacts and therefore don't imply an active, deployed router. To fetch a list of -deployed routers, you can use the /router/deploy/list GET endpoint. +Lists all the trained routers and the relevant metadata. +These routers are training artifacts and therefore don't imply an active, +deployed router. To fetch a list of deployed routers, you can use the +`/router/deploy/list` `GET` endpoint. #### Authorizations @@ -16,14 +17,14 @@ deployed routers, you can use the /router/deploy/list GET endpoint. ```bash cURL curl --request GET \ - --url 'https://api.unify.ai/v0/router/train/list' \ + --url 'https://api.unify.ai/v0/router/list' \ --header 'Authorization: Bearer ' ``` ```python Python import requests -url = "https://api.unify.ai/v0/router/train/list" +url = "https://api.unify.ai/v0/router/list" headers = {"Authorization": "Bearer "} diff --git a/api-reference/router_training/rename_router.mdx b/api-reference/router_training/rename_router.mdx new file mode 100644 index 000000000..32c74de1b --- /dev/null +++ b/api-reference/router_training/rename_router.mdx @@ -0,0 +1,64 @@ +--- +title: 'Rename Router' +api: 'POST /v0/router/rename' +--- +Renames the specified router from `name` to `new_name`. + +#### Authorizations + + + Bearer authentication header of the form `Bearer `, where `` is your auth token. + + +#### Query Parameters + + +The original name of the router. + + + +The new name for the router. + + + + +```bash cURL +curl --request POST \ + --url 'https://api.unify.ai/v0/router/rename?name=original_name&new_name=new_name' \ + --header 'Authorization: Bearer ' +``` + +```python Python +import requests + +url = "https://api.unify.ai/v0/router/rename?name=original_name&new_name=new_name" + +headers = {"Authorization": "Bearer "} + +response = requests.request("POST", url, headers=headers) + +print(response.text) +``` + + + + +```json 200 +{} +``` + +```json 422 +{ + "detail": [ + { + "loc": [ + "string" + ], + "msg": "string", + "type": "string" + } + ] +} +``` + + diff --git a/api-reference/routing/train_router.mdx b/api-reference/router_training/train_router.mdx similarity index 89% rename from api-reference/routing/train_router.mdx rename to api-reference/router_training/train_router.mdx index 9b35909a8..c5fd933c9 100644 --- a/api-reference/routing/train_router.mdx +++ b/api-reference/router_training/train_router.mdx @@ -2,9 +2,9 @@ title: 'Train Router' api: 'POST /v0/router/train' --- -Trains a router based on a dataset and a set of endpoints. To use a -custom-trained router, you will need to deploy the resulting artifacts to -a live endpoint. To do this, use the `/router/deploy` POST endpoint. +Train a router based on a specified training dataset and a set of endpoints to route +across. To *use* a custom-trained router, you will need to deploy the resulting +artifacts to a live endpoint, via the `/router/deploy` `POST` endpoint. #### Authorizations diff --git a/api-reference/model_and_endpoints/get_endpoints.mdx b/api-reference/supported_endpoints/list_endpoints.mdx similarity index 71% rename from api-reference/model_and_endpoints/get_endpoints.mdx rename to api-reference/supported_endpoints/list_endpoints.mdx index 55966cfd4..81e850791 100644 --- a/api-reference/model_and_endpoints/get_endpoints.mdx +++ b/api-reference/supported_endpoints/list_endpoints.mdx @@ -1,9 +1,10 @@ --- -title: 'Get Endpoints' +title: 'List Endpoints' api: 'GET /v0/endpoints' --- -Lists available endpoints in `model@provider` format. If `model` or `provider` are specified, -only the matching endpoints will be listed. +Lists available endpoints in `model@provider` format. +If `model` or `provider` are specified, only the matching endpoints will be listed. +You can also show all *custom* endpoints by passing `custom` as the provider. #### Authorizations @@ -14,11 +15,11 @@ only the matching endpoints will be listed. #### Query Parameters -Model to get available endpoints from. +Model to get available endpoints for. -Provider to get available endpoints from. +Provider to get available endpoints for. @@ -46,8 +47,9 @@ print(response.text) ```json 200 [ - "model_a@provider_1", - "model_a@provider_2", + "claude-3-haiku@anthropic", + "llama-3-70b-chat@groq", + "mistral-large@mistral-ai", "..." ] ``` diff --git a/api-reference/model_and_endpoints/get_models.mdx b/api-reference/supported_endpoints/list_models.mdx similarity index 80% rename from api-reference/model_and_endpoints/get_models.mdx rename to api-reference/supported_endpoints/list_models.mdx index f9e16ea60..292f4ef96 100644 --- a/api-reference/model_and_endpoints/get_models.mdx +++ b/api-reference/supported_endpoints/list_models.mdx @@ -1,8 +1,10 @@ --- -title: 'Get Models' +title: 'List Models' api: 'GET /v0/models' --- -Lists available models. If a provider is specified, returns the models that the provider supports. +Lists available models. If `provider` is specified, +returns the models that the provider supports. +You can also show all *custom* models by passing `custom` as the provider. #### Authorizations @@ -41,8 +43,9 @@ print(response.text) ```json 200 [ - "model_a", - "model_b", + "gpt-3.5-turbo", + "gpt-4", + "claude-3-haiku", "..." ] ``` diff --git a/api-reference/model_and_endpoints/get_providers.mdx b/api-reference/supported_endpoints/list_providers.mdx similarity index 82% rename from api-reference/model_and_endpoints/get_providers.mdx rename to api-reference/supported_endpoints/list_providers.mdx index 781e3453c..e9601c0f4 100644 --- a/api-reference/model_and_endpoints/get_providers.mdx +++ b/api-reference/supported_endpoints/list_providers.mdx @@ -1,8 +1,9 @@ --- -title: 'Get Providers' +title: 'List Providers' api: 'GET /v0/providers' --- -Lists available providers. If a model is specified, returns the providers that support that model. +Lists available providers. If `model` is specified, +returns the providers that support that model. #### Authorizations @@ -13,7 +14,7 @@ Lists available providers. If a model is specified, returns the providers that s #### Query Parameters -Model to get available providers from. +Model to get available providers for. @@ -41,8 +42,9 @@ print(response.text) ```json 200 [ - "provider_1", - "provider_2", + "openai", + "anthropic", + "together-ai", "..." ] ``` diff --git a/mint.json b/mint.json index b79752c23..e8ee1ffba 100644 --- a/mint.json +++ b/mint.json @@ -104,83 +104,113 @@ "group": "API Reference", "pages": [ { - "group": "User", + "group": "Supported Endpoints", "pages": [ - "api-reference/user/get_credits", - "api-reference/user/credits_code" + "api-reference/supported_endpoints/list_providers", + "api-reference/supported_endpoints/list_models", + "api-reference/supported_endpoints/list_endpoints" ] }, { - "group": "Model and Endpoints", + "group": "LLM Queries", "pages": [ - "api-reference/model_and_endpoints/get_models", - "api-reference/model_and_endpoints/get_endpoints", - "api-reference/model_and_endpoints/get_providers" + "api-reference/llm_queries/chat_completions" ] }, { - "group": "Querying LLMs", + "group": "Logging", "pages": [ - "api-reference/querying_llms/post_inference", - "api-reference/querying_llms/get_completions", - "api-reference/querying_llms/get_query_metrics" + "api-reference/logging/get_prompt_history", + "api-reference/logging/get_query_metrics" ] }, { - "group": "Custom Endpoints and API keys", + "group": "Custom Endpoints", "pages": [ - "api-reference/custom_endpoints_and_api_keys/get_custom_endpoints", - "api-reference/custom_endpoints_and_api_keys/create_custom_endpoint", - "api-reference/custom_endpoints_and_api_keys/delete_custom_endpoint", - "api-reference/custom_endpoints_and_api_keys/get_custom_api_keys", - "api-reference/custom_endpoints_and_api_keys/create_custom_api_key", - "api-reference/custom_endpoints_and_api_keys/delete_custom_api_key", - "api-reference/custom_endpoints_and_api_keys/rename_custom_endpoint", - "api-reference/custom_endpoints_and_api_keys/rename_custom_api_key", - "api-reference/custom_endpoints_and_api_keys/upload_custom_benchmark", - "api-reference/custom_endpoints_and_api_keys/get_custom_benchmarks" + "api-reference/custom_endpoints/create_custom_endpoint", + "api-reference/custom_endpoints/delete_custom_endpoint", + "api-reference/custom_endpoints/rename_custom_endpoint" ] }, { - "group": "Dataset", + "group": "Custom API keys", "pages": [ - "api-reference/dataset/upload_dataset", - "api-reference/dataset/delete_dataset", - "api-reference/dataset/download_dataset", - "api-reference/dataset/list_datasets", - "api-reference/dataset/rename_dataset" + "api-reference/custom_api_keys/create_custom_api_key", + "api-reference/custom_api_keys/get_custom_api_key", + "api-reference/custom_api_keys/delete_custom_api_key", + "api-reference/custom_api_keys/rename_custom_api_key", + "api-reference/custom_api_keys/list_custom_api_keys" ] }, { - "group": "Dataset Evaluation", + "group": "Datasets", "pages": [ - "api-reference/dataset_evaluation/create_eval", - "api-reference/dataset_evaluation/list_evals", - "api-reference/dataset_evaluation/return_eval_config", - "api-reference/dataset_evaluation/rename_eval", - "api-reference/dataset_evaluation/delete_eval", - "api-reference/dataset_evaluation/trigger_eval", - "api-reference/dataset_evaluation/get_eval_scores", - "api-reference/dataset_evaluation/eval_status" + "api-reference/datasets/upload_dataset", + "api-reference/datasets/download_dataset", + "api-reference/datasets/delete_dataset", + "api-reference/datasets/rename_dataset", + "api-reference/datasets/list_datasets" ] }, { - "group": "Routing", + "group": "Evaluators", "pages": [ - "api-reference/routing/train_router", - "api-reference/routing/delete_router_train", - "api-reference/routing/get_trained_routers", - "api-reference/routing/deploy_router", - "api-reference/routing/delete_router", - "api-reference/routing/get_deployed_routers" + "api-reference/evaluators/create_evaluator", + "api-reference/evaluators/get_evaluator", + "api-reference/evaluators/delete_evaluator", + "api-reference/evaluators/rename_evaluator", + "api-reference/evaluators/list_evaluators" ] }, { - "group": "benchmarks", + "group": "Evaluations", "pages": [ - "api-reference/benchmarks/get_latest_benchmark", - "api-reference/benchmarks/filter_benchmark", - "api-reference/benchmarks/get_prompt_history" + "api-reference/evaluations/trigger_evaluation", + "api-reference/evaluations/get_evaluations", + "api-reference/evaluations/delete_evaluations", + "api-reference/evaluations/eval_status" + ] + }, + { + "group": "Efficiency Benchmarks", + "pages": [ + "api-reference/efficiency_benchmarks/append_to_benchmark", + "api-reference/efficiency_benchmarks/get_benchmark", + "api-reference/efficiency_benchmarks/delete_benchmark" + ] + }, + { + "group": "Router Training", + "pages": [ + "api-reference/router_training/train_router", + "api-reference/router_training/delete_router", + "api-reference/router_training/rename_router", + "api-reference/router_training/list_routers" + ] + }, + { + "group": "Router Deployment", + "pages": [ + "api-reference/router_deployment/deploy_router", + "api-reference/router_deployment/undeploy_router", + "api-reference/router_deployment/list_deployed_routers" + ] + }, + { + "group": "Router Configurations", + "pages": [ + "api-reference/router_configurations/create_router_config", + "api-reference/router_configurations/get_router_config", + "api-reference/router_configurations/delete_router_config", + "api-reference/router_configurations/rename_router_config", + "api-reference/router_configurations/list_router_configs" + ] + }, + { + "group": "Credits", + "pages": [ + "api-reference/credits/get_credits", + "api-reference/credits/promo_code" ] } ]