From 8c275fbd913fc64b7dc03a3e83fa2d04714cb415 Mon Sep 17 00:00:00 2001 From: daniel carvalho Date: Wed, 3 Jul 2024 13:54:20 -0300 Subject: [PATCH] Set interval limit for end date --- data_collection/gazette/spiders/base/barcodigital.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/data_collection/gazette/spiders/base/barcodigital.py b/data_collection/gazette/spiders/base/barcodigital.py index d4cf05492..009497eee 100644 --- a/data_collection/gazette/spiders/base/barcodigital.py +++ b/data_collection/gazette/spiders/base/barcodigital.py @@ -30,8 +30,15 @@ def start_requests(self): def parse(self, response): for documents in response.json().values(): for document in documents: + document_date = datetime.strptime( + document.get("data"), "%Y-%m-%d" + ).date() + + if document_date > self.end_date: + continue + yield Gazette( - date=datetime.strptime(document.get("data"), "%Y-%m-%d").date(), + date=document_date, edition_number=document.get("edicao"), is_extra_edition=document.get("tipo_edicao_id") != self.EDITION_TYPE_NORMAL,