-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
710 additions
and
216 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -34,6 +34,7 @@ sqlalchemy = "*" | |
dbt = "*" | ||
dbt-core = "*" | ||
dbt-bigquery = "*" | ||
bs4 = "*" | ||
|
||
[dev-packages] | ||
black="*" | ||
|
Large diffs are not rendered by default.
Oops, something went wrong.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
import requests | ||
import pandas as pd | ||
from bs4 import BeautifulSoup | ||
|
||
|
||
class ChainsAssetsMetadata: | ||
def __init__(self, url: str): | ||
self.url = url | ||
|
||
def pull_registered_assets_data(self) -> pd.DataFrame: | ||
""" | ||
Fetches and parses the registered assets table from the specified URL. | ||
Returns: | ||
pd.DataFrame: A DataFrame containing the registered assets metadata. | ||
""" | ||
# Step 1: Fetch the webpage content | ||
response = requests.get(self.url) | ||
response.raise_for_status() # Raises HTTPError for bad responses | ||
content = response.content | ||
|
||
# Step 2: Parse the HTML content using BeautifulSoup | ||
soup = BeautifulSoup(content, "html.parser") | ||
|
||
# Step 3: Locate the 'Registered Assets' section and find the table | ||
registered_assets_header = soup.find("h2", id="registered-assets") | ||
if not registered_assets_header: | ||
raise ValueError( | ||
"Couldn't find the 'Registered Assets' section in the HTML." | ||
) | ||
|
||
# Assuming the table is immediately after the header | ||
table = registered_assets_header.find_next("table") | ||
if not table: | ||
raise ValueError( | ||
"Couldn't find the table under the 'Registered Assets' section." | ||
) | ||
|
||
# Step 4: Loop through the rows and columns of the table to gather the data | ||
table_data = [] | ||
for row in table.find_all("tr"): | ||
cols = row.find_all("td") | ||
if cols: # Ensure the row has data cells | ||
cols_text = [col.get_text(strip=True) for col in cols] | ||
table_data.append(cols_text) | ||
|
||
# Step 5: Convert the data to a pandas DataFrame | ||
df = pd.DataFrame( | ||
table_data, | ||
columns=[ | ||
"asset_name", | ||
"symbol", | ||
"decimals", | ||
"domain_id", | ||
"address", | ||
"faucet", | ||
"faucet_limit", | ||
], | ||
) | ||
|
||
return df | ||
|
||
def save_to_csv(self, df: pd.DataFrame, filepath: str): | ||
""" | ||
Saves the DataFrame to a specified CSV file. | ||
Args: | ||
df (pd.DataFrame): The DataFrame to save. | ||
filepath (str): The path where the CSV will be saved. | ||
""" | ||
df.to_csv(filepath, index=False) | ||
print(f"Data has been saved to '{filepath}'.") | ||
|
||
|
||
if __name__ == "__main__": | ||
|
||
# Initialize the class with the target URL | ||
metadata_scraper = ChainsAssetsMetadata( | ||
url="https://docs.everclear.org/resources/contracts/mainnet" | ||
) | ||
|
||
# Pull the registered assets data into a DataFrame | ||
df_assets = metadata_scraper.pull_registered_assets_data() | ||
|
||
# Save the DataFrame to a CSV file | ||
metadata_scraper.save_to_csv(df_assets, "data/chains_assets_metadata.csv") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.