Skip to content

Commit

Permalink
Merge branch 'main' into experiments-prompt-data-set-linking
Browse files Browse the repository at this point in the history
  • Loading branch information
@chitalian
chitalian committed May 13, 2024
2 parents f846bcb + 8ff1400 commit d30356b
Show file tree
Hide file tree
Showing 16 changed files with 386 additions and 177 deletions.
137 changes: 137 additions & 0 deletions costs/__tests__/ensureOnlyOne.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,14 @@ test("cost calc snapshot test", () => {
"name": "gpt-4-1106-vision-preview",
"provider": "OPENAI"
},
{
"name": "gpt-4o",
"provider": "OPENAI"
},
{
"name": "gpt-4o-2024-05-13",
"provider": "OPENAI"
},
{
"name": "gpt-3.5-turbo-0613",
"provider": "OPENAI"
Expand Down Expand Up @@ -126,6 +134,94 @@ test("cost calc snapshot test", () => {
{
"name": "claude-3-haiku-20240307",
"provider": "ANTHROPIC"
},
{
"name": "gpt-3.5-turbo",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-0301",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-1106",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-instruct",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-instruct-0914",
"provider": "AZURE"
},
{
"name": "gpt-4",
"provider": "AZURE"
},
{
"name": "gpt-4-0314",
"provider": "AZURE"
},
{
"name": "gpt-4-0613",
"provider": "AZURE"
},
{
"name": "gpt-4-32k",
"provider": "AZURE"
},
{
"name": "gpt-4-32k-0314",
"provider": "AZURE"
},
{
"name": "gpt-4-32k-0613",
"provider": "AZURE"
},
{
"name": "gpt-4-0125-preview",
"provider": "AZURE"
},
{
"name": "gpt-4-1106-preview",
"provider": "AZURE"
},
{
"name": "gpt-4-1106-vision-preview",
"provider": "AZURE"
},
{
"name": "gpt-4o",
"provider": "AZURE"
},
{
"name": "gpt-4o-2024-05-13",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-0613",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-16k-0613",
"provider": "AZURE"
},
{
"name": "gpt-3.5-turbo-0125",
"provider": "AZURE"
},
{
"name": "gpt-4-turbo",
"provider": "AZURE"
},
{
"name": "gpt-4-turbo-2024-04-09",
"provider": "AZURE"
},
{
"name": "gpt-4-turbo-0125-preview",
"provider": "AZURE"
}
]"
`);
Expand Down Expand Up @@ -163,6 +259,45 @@ WHEN (request_response_log.model = 'gpt35') THEN 1500 * request_response_log.pro
WHEN (request_response_log.model = 'gpt-35-turbo-0613') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-35-16k') THEN 3000 * request_response_log.prompt_tokens + 4000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-vision') THEN 10000 * request_response_log.prompt_tokens + 300000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'ada') THEN 400 * request_response_log.prompt_tokens + 400 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-ada-001') THEN 400 * request_response_log.prompt_tokens + 400 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'babbage') THEN 500 * request_response_log.prompt_tokens + 500 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'curie') THEN 2000 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-curie-001') THEN 2000 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'davinci') THEN 20000 * request_response_log.prompt_tokens + 20000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-davinci-001') THEN 20000 * request_response_log.prompt_tokens + 20000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-davinci-002') THEN 20000 * request_response_log.prompt_tokens + 20000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-davinci-003') THEN 20000 * request_response_log.prompt_tokens + 20000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-0301') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-35-turbo') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-1106') THEN 1000 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-instruct') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-instruct-0914') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4') THEN 30000 * request_response_log.prompt_tokens + 60000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-0314') THEN 30000 * request_response_log.prompt_tokens + 60000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-0613') THEN 30000 * request_response_log.prompt_tokens + 60000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-32k') THEN 60000 * request_response_log.prompt_tokens + 120000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-32k-0314') THEN 60000 * request_response_log.prompt_tokens + 120000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-32k-0613') THEN 60000 * request_response_log.prompt_tokens + 120000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-0125-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-1106-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-1106-vision-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4o') THEN 15000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4o-2024-05-13') THEN 15000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-0613') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-35-turbo-16k') THEN 3000 * request_response_log.prompt_tokens + 4000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-16k-0613') THEN 3000 * request_response_log.prompt_tokens + 4000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-0125') THEN 500 * request_response_log.prompt_tokens + 1500 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-turbo') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-turbo-2024-04-09') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-turbo-0125-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-embedding-ada-002') THEN 100 * request_response_log.prompt_tokens + 0 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-embedding-ada') THEN 100 * request_response_log.prompt_tokens + 0 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-embedding-ada-002-v2') THEN 100 * request_response_log.prompt_tokens + 0 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'text-embedding-3-small') THEN 20 * request_response_log.prompt_tokens + 0 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-vision-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-35-turbo-16k-0613') THEN 3000 * request_response_log.prompt_tokens + 4000 * request_response_log.completion_tokens
ELSE 0
END
)
Expand Down Expand Up @@ -449,6 +584,8 @@ WHEN (request_response_log.model = 'gpt-4-32k-0613') THEN 60000 * request_respon
WHEN (request_response_log.model = 'gpt-4-0125-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-1106-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4-1106-vision-preview') THEN 10000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4o') THEN 15000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-4o-2024-05-13') THEN 15000 * request_response_log.prompt_tokens + 30000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-0613') THEN 1500 * request_response_log.prompt_tokens + 2000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-35-turbo-16k') THEN 3000 * request_response_log.prompt_tokens + 4000 * request_response_log.completion_tokens
WHEN (request_response_log.model = 'gpt-3.5-turbo-16k-0613') THEN 3000 * request_response_log.prompt_tokens + 4000 * request_response_log.completion_tokens
Expand Down
2 changes: 1 addition & 1 deletion costs/src/providers/mappings.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ export const providers: {
{
pattern: azurePattern,
provider: "AZURE",
costs: azureCosts,
costs: [...azureCosts, ...openaiCosts],
},
{
pattern: localProxyPattern,
Expand Down
22 changes: 22 additions & 0 deletions costs/src/providers/openai/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,28 @@ export const costs: ModelRow[] = [
},
showInPlayground: true,
},
{
model: {
operator: "equals",
value: "gpt-4o",
},
cost: {
prompt_token: 0.000015,
completion_token: 0.00003,
},
showInPlayground: true,
},
{
model: {
operator: "equals",
value: "gpt-4o-2024-05-13",
},
cost: {
prompt_token: 0.000015,
completion_token: 0.00003,
},
showInPlayground: true,
},
{
model: {
operator: "equals",
Expand Down
2 changes: 1 addition & 1 deletion docs/features/prompts.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ Let's say we have an app that generates a short story, where users are able to i

Assigning an id allows us to associate your prompt with future versions of your prompt, and automatically manage versions on your behalf.

Depending on the package you are using, you will need to add a header. For more information on adding headers to packages, please see [Header Directory](/header-directory).
Depending on the package you are using, you will need to add a header. For more information on adding headers to packages, please see [Header Directory](/helicone-headers/header-directory).

```tsx
headers: {
Expand Down
14 changes: 3 additions & 11 deletions docs/getting-started/quick-start.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,6 @@ description: "Helicone is the observability platform purpose-built for LLM devel
## Quick Start

Integrate with Helicone and send your first events in seconds.

<CardGroup cols={1}>
<Card title="Gateway" href="/getting-started/integration-method/gateway" icon="flag">
The optimal method to integrate with Helicone is through our Gateway.
</Card>
</CardGroup>

<CardGroup cols={3}>
<Card title="OpenAI" href="/getting-started/integration-method/openai-proxy">
Python, Node, cURL, Langchain
Expand All @@ -44,14 +37,15 @@ Integrate with Helicone and send your first events in seconds.
</Card>
<Card title="LiteLLM" href="/getting-started/integration-method/litellm">
</Card>
<Card title="Gateway" href="/getting-started/integration-method/gateway" icon="flag">
Don't see your provider above? Try Helicone's universal Gateway.
</Card>
</CardGroup>

## New to Helicone?

To help you get the most out of Helicone, we curated a list of actions that you can do next. Our users typically find themselves doing majority of the following, but you're welcome to explore the product on your own!



<CardGroup cols={1}>
<Card title="Add a custom property" href="features/advanced-usage/custom-properties" icon="square-1">
Label your requests. We will walk you through how to segment, analyze, and visualize them.
Expand Down Expand Up @@ -182,7 +176,6 @@ Discover features for monitoring and experimenting with your prompts.

</CardGroup>


## Further Reading

<CardGroup cols={3}>
Expand All @@ -197,7 +190,6 @@ Discover features for monitoring and experimenting with your prompts.
</Card>
</CardGroup>


## Questions?
<Check>Although we designed the docs to be as self-serving as possible, you are welcome to join our [Discord](https://discord.com/invite/HwUbV3Q8qz) or contact [[email protected]]([email protected]) with any questions or feedback you have. </Check>

Expand Down
Loading

0 comments on commit d30356b

Please sign in to comment.