Set interval limit for end date

okfn-brasil · Aug 7, 2024 · 8c275fb · 8c275fb
1 parent 70d5b10
commit 8c275fb
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/data_collection/gazette/spiders/base/barcodigital.py b/data_collection/gazette/spiders/base/barcodigital.py
@@ -30,8 +30,15 @@ def start_requests(self):
     def parse(self, response):
         for documents in response.json().values():
             for document in documents:
+                document_date = datetime.strptime(
+                    document.get("data"), "%Y-%m-%d"
+                ).date()
+
+                if document_date > self.end_date:
+                    continue
+
                 yield Gazette(
-                    date=datetime.strptime(document.get("data"), "%Y-%m-%d").date(),
+                    date=document_date,
                     edition_number=document.get("edicao"),
                     is_extra_edition=document.get("tipo_edicao_id")
                     != self.EDITION_TYPE_NORMAL,