Skip to content

Commit

Permalink
Remove erroneous number of records count function for NLS
Browse files Browse the repository at this point in the history
  • Loading branch information
JackGilmore committed Nov 10, 2023
1 parent e1849ea commit 59dca44
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions nls_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ def fetch_file_size(page: BeautifulSoup) -> list:
return list_of_filesizes


# TODO: This doesn't accurately get the number of records and needs to be fixed
def fetch_num_recs(page: BeautifulSoup) -> list:
"""
Fetches the number of files of each dataset on that page.
Expand Down Expand Up @@ -367,7 +368,8 @@ def main():
soup = BeautifulSoup(req.content, "html.parser")
list_of_asset_urls = fetch_asset_urls(soup)
fetched_file_size = fetch_file_size(soup)
fetched_num_recs = fetch_num_recs(soup)
# TODO: This doesn't accurately get the number of records and needs to be fixed
# fetched_num_recs = fetch_num_recs(soup)
counter = 0
nls_licence = fetch_licences(soup)
# print("nls_licence:", nls_licence)
Expand Down Expand Up @@ -395,7 +397,7 @@ def main():
### fetch_data_types is more accurate & useful, but file extension is consistent with other listings
data_type = asset_url.rsplit(".", 1)[1] # fetch_data_types(soup)
# print("data_type:", data_type)
num_recs = fetched_num_recs[counter]
num_recs = None
# print(("num_recs:", num_recs))
if len(indiv_descriptions) > 1:
description = (
Expand Down

0 comments on commit 59dca44

Please sign in to comment.