Skip to content

Commit

Permalink
Daily sync: add extra logging
Browse files Browse the repository at this point in the history
  • Loading branch information
raphodn committed May 6, 2024
1 parent e6f667d commit 89b8d47
Showing 1 changed file with 9 additions and 1 deletion.
10 changes: 9 additions & 1 deletion app/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,21 @@ def import_product_db(
"""
logger.info(f"Launching import_product_db ({flavor})")
existing_codes = set(db.execute(select(Product.code)).scalars())
logger.info("Number of existing codes: %d", len(existing_codes))
logger.info(f"OP: number of existing codes: {len(existing_codes)}")
existing_codes_flavor = set(
db.execute(select(Product.code).where(Product.source == flavor)).scalars()
)
logger.info(
f"OP: number of existing codes from {flavor}: {len(existing_codes_flavor)}"
)

dataset = ProductDataset(
flavor=flavor,
dataset_type=DatasetType.jsonl,
force_download=True,
download_newer=True,
)
logger.info(f"Dataset size: {dataset.count()}")

added_count = 0
updated_count = 0
Expand Down

0 comments on commit 89b8d47

Please sign in to comment.