-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
300 additions
and
42 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,233 @@ | ||
import pytz | ||
import httpx | ||
import numpy as np | ||
from datetime import datetime | ||
import typing | ||
from dataclasses import dataclass | ||
import asyncio | ||
from typing import Optional, Dict, Any | ||
import pandas as pd | ||
import pandas_gbq as gbq | ||
import logging | ||
from pydantic import BaseModel | ||
from typing import List | ||
|
||
logging.basicConfig( | ||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" | ||
) | ||
|
||
|
||
@dataclass | ||
class TokenPriceConfig: | ||
"""Configuration class for token price fetching""" | ||
|
||
base_url: str = "https://coins.llama.fi/chart" | ||
chain: str = "ethereum" | ||
search_width: int = 100 | ||
period: str = "1m" | ||
span: int = 30 | ||
|
||
|
||
class TokenPriceClient: | ||
"""Client for fetching token prices from DefiLlama""" | ||
|
||
def __init__(self, config: TokenPriceConfig = TokenPriceConfig()): | ||
self.config = config | ||
self.client = httpx.AsyncClient(timeout=30.0) | ||
|
||
async def __aenter__(self): | ||
return self | ||
|
||
async def __aexit__(self, exc_type, exc_val, exc_tb): | ||
await self.client.aclose() | ||
|
||
def _build_url(self, token_address: str) -> str: | ||
"""Build URL for API request""" | ||
token_identifier = f"{self.config.chain}:{token_address}" | ||
return f"{self.config.base_url}/{token_identifier}" | ||
|
||
def _build_params(self, start_timestamp: int) -> Dict[str, Any]: | ||
"""Build query parameters""" | ||
return { | ||
"start": start_timestamp, | ||
"span": self.config.span, | ||
"period": self.config.period, | ||
"searchWidth": self.config.search_width, | ||
} | ||
|
||
async def get_token_prices(self, token_address: str, start_timestamp: int): | ||
""" | ||
Fetch token prices from DefiLlama | ||
Args: | ||
token_address: Token contract address | ||
start_timestamp: Start timestamp in unix format | ||
Returns: | ||
TokenPriceResponse object containing price data | ||
""" | ||
url = self._build_url(token_address) | ||
params = self._build_params(start_timestamp) | ||
|
||
try: | ||
response = await self.client.get(url, params=params) | ||
response.raise_for_status() | ||
data = response.json() | ||
|
||
return data | ||
|
||
except httpx.HTTPError as e: | ||
raise Exception(f"HTTP error occurred: {str(e)}") | ||
except Exception as e: | ||
raise Exception(f"Error fetching token prices: {str(e)}") | ||
|
||
|
||
class PricePoint(BaseModel): | ||
"""Model for individual price points""" | ||
|
||
timestamp: int | ||
price: float | ||
|
||
|
||
class TokenData(BaseModel): | ||
"""Model for token information""" | ||
|
||
symbol: str | ||
confidence: float | ||
decimals: int | ||
prices: List[PricePoint] | ||
|
||
|
||
class TokenResponse(BaseModel): | ||
"""Root response model""" | ||
|
||
coins: Dict[str, TokenData] | ||
|
||
|
||
def process_token_data(json_data: dict) -> pd.DataFrame: | ||
""" | ||
Process token price data into a DataFrame | ||
Args: | ||
json_data: Raw JSON response from the API | ||
Returns: | ||
pd.DataFrame: Processed and validated token price data | ||
""" | ||
# Validate data using Pydantic | ||
validated_data = TokenResponse(coins=json_data["coins"]) | ||
|
||
# Process into rows for DataFrame | ||
rows = [] | ||
for contract_address, token_data in validated_data.coins.items(): | ||
# Split contract address into chain and address | ||
chain = contract_address.split(":")[0] | ||
address = contract_address.split(":")[1] | ||
|
||
# Create a row for each price point | ||
for price_data in token_data.prices: | ||
# minute,blockchain, contract_address, decimals, symbol, price | ||
rows.append( | ||
{ | ||
"minute": datetime.fromtimestamp(price_data.timestamp, tz=pytz.UTC), | ||
"blockchain": chain, | ||
"contract_address": address.lower(), | ||
"decimals": token_data.decimals, | ||
"symbol": token_data.symbol.upper(), | ||
"price": price_data.price, | ||
} | ||
) | ||
|
||
return pd.DataFrame(rows) | ||
|
||
|
||
def get_max_token_price_timestamp_from_bq() -> int: | ||
sql = "SELECT max(CAST(minute AS TIMESTAMP)) AS max_timestamp FROM `mainnet-bigq.dune.all_everclear_tokens_prices`" | ||
df = gbq.read_gbq(sql) | ||
final_start_date = np.array(df["max_timestamp"])[0] | ||
# convert to int | ||
return int(final_start_date.timestamp()) | ||
|
||
|
||
async def pull_defilamma_coingecko_price(start_timestamp: int): | ||
""" | ||
Fetch token prices from DefiLlama and push to BigQuery | ||
""" | ||
|
||
token_addresses = [ | ||
"0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", # WETH | ||
"0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", # USDC | ||
"0xdac17f958d2ee523a2206206994597c13d831ec7", # USDT | ||
"0xB8C77482E45F1F44DE1745F52C74426C631BDD52", # BNB | ||
] | ||
|
||
logging.info(f"Starting from {start_timestamp}") | ||
config = TokenPriceConfig() | ||
all_price_data = [] | ||
|
||
async with TokenPriceClient(config) as client: | ||
for token_address in token_addresses: | ||
try: | ||
response = await client.get_token_prices(token_address, start_timestamp) | ||
logging.info(f"Fetching price data for token: {token_address}") | ||
|
||
# Process data for this token | ||
token_price_data = process_token_data(response) | ||
all_price_data.append(token_price_data) | ||
|
||
# Add delay to avoid rate limiting | ||
await asyncio.sleep(10) | ||
|
||
except Exception as e: | ||
logging.error( | ||
f"Error fetching data for token {token_address}: {str(e)}" | ||
) | ||
continue | ||
|
||
if all_price_data: | ||
# Combine all dataframes | ||
combined_price_data = pd.concat(all_price_data, ignore_index=True) | ||
return combined_price_data | ||
else: | ||
logging.error("No price data was collected") | ||
|
||
|
||
def get_all_everclear_tokens_prices_pipeline(): | ||
"""date as UTC, convert the number to timestamp from BQ""" | ||
|
||
max_date_bq = get_max_token_price_timestamp_from_bq() | ||
from_date, to_date = max_date_bq, int(datetime.now(pytz.UTC).timestamp()) | ||
logging.info( | ||
f"Pulling data for Everclear tokens prices from {from_date} to {to_date}" | ||
) | ||
df = asyncio.run(pull_defilamma_coingecko_price(start_timestamp=max_date_bq)) | ||
if not df.empty: | ||
logging.info(df.head()) | ||
|
||
max_timestamps_by_symbol = df.groupby("symbol")["minute"].max().reset_index() | ||
min_of_max_timestamps = np.array(max_timestamps_by_symbol["minute"].min()) | ||
logging.info( | ||
f"Min of max timestamps: {min_of_max_timestamps}, filter out all data above this" | ||
) | ||
|
||
df_final = df[ | ||
(df["minute"] <= min_of_max_timestamps) | ||
& (df["minute"] > pd.to_datetime(max_date_bq, unit="s", utc=True)) | ||
].reset_index(drop=True) | ||
if not df_final.empty: | ||
logging.info(f"Adding data to BigQuery: length {len(df_final)}") | ||
gbq.to_gbq( | ||
dataframe=df_final, | ||
project_id="mainnet-bigq", | ||
destination_table="dune.all_everclear_tokens_prices", | ||
if_exists="append", | ||
api_method="load_csv", | ||
) | ||
else: | ||
logging.info( | ||
f"""data not up to date for all symbols, | ||
from {from_date} to {to_date} with min timestamp {min_of_max_timestamps} | ||
""" | ||
) | ||
else: | ||
logging.info(f"No data fetched for the period {from_date} to {to_date}") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
from pydantic import BaseModel, Field | ||
from typing import List, Optional | ||
from datetime import datetime | ||
import pandas as pd | ||
|
||
|
||
class TokenPrice(BaseModel): | ||
"""Individual price point for a token""" | ||
|
||
timestamp: int | ||
price: float | ||
|
||
|
||
class TokenData(BaseModel): | ||
"""Token information and price data""" | ||
|
||
symbol: str | ||
confidence: float | ||
decimals: int | ||
prices: List[TokenPrice] | ||
|
||
|
||
class TokenResponse(BaseModel): | ||
"""Root response model""" | ||
|
||
coins: dict[str, TokenData] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters