Skip to content

Commit

Permalink
fix(find-dependencies): http->https redirections weren't followed
Browse files Browse the repository at this point in the history
  • Loading branch information
Ned Batchelder committed Aug 17, 2023
1 parent f9ed7f7 commit 4040308
Showing 1 changed file with 18 additions and 5 deletions.
23 changes: 18 additions & 5 deletions edx_repo_tools/find_dependencies/find_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ def find_real_url(url: str) -> Optional[str]:
# I didn't know you could get 429 from https://github.com, but you can...
wait = int(resp.headers.get("Retry-After", 10))
time.sleep(wait + 1)
elif resp.status_code in {301, 302}:
url = resp.headers.get("Location")
else:
break

Expand Down Expand Up @@ -303,13 +305,25 @@ def process_directory():
repo_urls.update(check_py_dependencies())
return repo_urls

FIRST_PARTY_ORGS = ["openedx"]

SECOND_PARTY_ORGS = [
"edx", "edx-unsupported", "edx-solutions",
"mitodl",
"overhangio",
"open-craft", "eduNEXT", "raccoongang",
]

def urls_in_orgs(urls, orgs):
"""
Find urls that are in any of the `orgs`.
"""
return sorted(
url for url in urls
if any(f"/{org}/" in url for org in orgs)
)


def main(dirs=None):
"""
Analyze the requirements in all of the directories mentioned on the command line.
Expand All @@ -331,11 +345,10 @@ def main(dirs=None):

write_list(WORK_DIR / "repo_urls.txt", sorted(repo_urls))

seconds = sorted(
url for url in repo_urls
if any(f"/{org}/" in url for org in SECOND_PARTY_ORGS)
)
write_list(WORK_DIR / "second_party_urls.txt", sorted(seconds))
firsts = urls_in_orgs(repo_urls, FIRST_PARTY_ORGS)
write_list(WORK_DIR / "first_party_urls.txt", firsts)
seconds = urls_in_orgs(repo_urls, SECOND_PARTY_ORGS)
write_list(WORK_DIR / "second_party_urls.txt", seconds)

print("== DONE ==============")
print("Second-party:")
Expand Down

0 comments on commit 4040308

Please sign in to comment.