Skip to content

Commit

Permalink
feat: Use different extensions for partially-downloaded files
Browse files Browse the repository at this point in the history
  • Loading branch information
JoshuaVandaele committed Mar 6, 2024
1 parent f482caf commit 5627569
Showing 1 changed file with 32 additions and 17 deletions.
49 changes: 32 additions & 17 deletions modules/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,23 +239,38 @@ def download_file(url: str, local_file: Path, progress_bar: bool = True) -> None
Returns:
None
"""
logging.debug(f"[download_file] Downloading {url} to {local_file.resolve()}")
with requests.get(url, stream=True) as r:
total_size = int(r.headers.get("content-length", 0)) # Sizes in bytes

with open(local_file, "wb") as f:
if progress_bar:
with tqdm(
total=total_size,
unit="B",
desc=local_file.name,
) as pbar:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
pbar.update(len(chunk))
else:
shutil.copyfileobj(r.raw, f)
part_file = local_file.with_suffix(".part")
logging.debug(f"[download_file] Downloading {url} to {part_file.resolve()}")

try:
with requests.get(url, stream=True) as r:
total_size = int(r.headers.get("content-length", 0)) # Sizes in bytes

with open(part_file, "wb") as f:
if progress_bar:
with tqdm(
total=total_size,
unit="B",
desc=part_file.name,
) as pbar:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
pbar.update(len(chunk))
else:
shutil.copyfileobj(r.raw, f)
except requests.exceptions.RequestException:
logging.exception(f"Failed to download {url} to {part_file.resolve()}")
if part_file.exists():
part_file.unlink()
raise
except KeyboardInterrupt:
logging.info(f"Download of {url} to {part_file.resolve()} was cancelled")
if part_file.exists():
part_file.unlink()
raise

part_file.rename(local_file)


def windows_consumer_download(
Expand Down

0 comments on commit 5627569

Please sign in to comment.