Skip to content

Commit

Permalink
Skip broken/outdated import scripts in run_all_import_scripts
Browse files Browse the repository at this point in the history
Also, run the scripts alphabetically for easier debugging.
  • Loading branch information
zarino committed Sep 24, 2024
1 parent 57195e0 commit d85d048
Showing 1 changed file with 20 additions and 4 deletions.
24 changes: 20 additions & 4 deletions hub/management/commands/run_all_import_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,23 @@
class Command(BaseCommand):
help = "Run all of the import scripts"

priority_imports = [
"import_areas",
"import_mps",
"import_mps_election_results",
]
skip_imports = [
"import_2024_ppcs", # no longer relevant post-election
"import_air_quality_data", # ValueError because it checks for truthiness of a Pandas dataframe?
"import_christian_aid_group_locations", # JSONDecodeError because parlparse JSON file not found
"import_mps_appg_data", # hasn't been updated for Autumn 2024 APPGs
"import_mps_relevant_votes", # hasn't been updated for a while (and we import EDMs separately now)
"import_mps_standing_down_2024", # no longer relevant post-election
"import_mps_select_committee_membership", # ValueError because it checks for truthiness of a Pandas dataframe?
"import_nt_property_locations", # ValueError because it checks for truthiness of a Pandas dataframe?
"import_onward_polling_data", # JSONDecodeError because parlparse JSON file not found
]

def get_scripts(self, *args, **options):
path = "hub/management/commands"
files = [f for f in listdir(path) if isfile(join(path, f))]
Expand All @@ -19,10 +36,10 @@ def get_scripts(self, *args, **options):
]
generators = []
importers = []
for script in scripts:
for script in sorted(scripts):
if "generate_" in script:
generators.append(script)
else:
elif script not in self.skip_imports:
importers.append(script)

return {"generators": generators, "importers": importers}
Expand Down Expand Up @@ -56,8 +73,7 @@ def run_importer_scripts(self, imports, *args, **options):
total = str(len(imports))
i = 1
failed_imports = {}
priority_imports = ["import_areas", "import_mps", "import_mps_election_results"]
for importer in priority_imports:
for importer in self.priority_imports:
imports.remove(importer)
print(f"Running command: {importer} ({str(i)}/{total})")
try:
Expand Down

0 comments on commit d85d048

Please sign in to comment.