From 323aeece893bae67a677bad31a9ee6f2310ebbd7 Mon Sep 17 00:00:00 2001 From: Jonathan Schweder Date: Thu, 5 Oct 2023 19:11:53 +0100 Subject: [PATCH 1/4] =?UTF-8?q?Corre=C3=A7=C3=A3o=20de=20raspadores=20Inst?= =?UTF-8?q?ar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data_collection/gazette/spiders/sp/sp_alto_alegre.py | 5 ++++- data_collection/gazette/spiders/sp/sp_aracariguama.py | 5 ++++- data_collection/gazette/spiders/sp/sp_coronel_macedo.py | 5 ++++- data_collection/gazette/spiders/sp/sp_glicerio.py | 5 ++++- data_collection/gazette/spiders/sp/sp_itapirapua_paulista.py | 5 ++++- data_collection/gazette/spiders/sp/sp_lavinia.py | 5 ++++- data_collection/gazette/spiders/sp/sp_monte_alto.py | 5 ++++- data_collection/gazette/spiders/sp/sp_parisi.py | 5 ++++- data_collection/gazette/spiders/sp/sp_patrocinio_paulista.py | 5 ++++- data_collection/gazette/spiders/sp/sp_pratania.py | 5 ++++- data_collection/gazette/spiders/sp/sp_santa_ernestina.py | 5 ++++- data_collection/gazette/spiders/sp/sp_sao_manuel.py | 5 ++++- data_collection/gazette/spiders/sp/sp_sarutaia.py | 5 ++++- 13 files changed, 52 insertions(+), 13 deletions(-) diff --git a/data_collection/gazette/spiders/sp/sp_alto_alegre.py b/data_collection/gazette/spiders/sp/sp_alto_alegre.py index 7f7ba1862..f1820e5cb 100644 --- a/data_collection/gazette/spiders/sp/sp_alto_alegre.py +++ b/data_collection/gazette/spiders/sp/sp_alto_alegre.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpAltoAlegreSpider(BaseInstarSpider): TERRITORY_ID = "3501103" name = "sp_alto_alegre" allowed_domains = ["altoalegre.sp.gov.br"] - start_urls = ["http://www.altoalegre.sp.gov.br/portal/diario-oficial"] + start_date = date(2018, 7, 3) + base_url = "http://www.altoalegre.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_aracariguama.py b/data_collection/gazette/spiders/sp/sp_aracariguama.py index d0d3097ef..fd83ee0a3 100644 --- a/data_collection/gazette/spiders/sp/sp_aracariguama.py +++ b/data_collection/gazette/spiders/sp/sp_aracariguama.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpAracariguamaSpider(BaseInstarSpider): TERRITORY_ID = "3502754" name = "sp_aracariguama" allowed_domains = ["aracariguama.sp.gov.br"] - start_urls = ["https://www.aracariguama.sp.gov.br/portal/diario-oficial"] + start_date = date(2019, 9, 6) + base_url = "https://www.aracariguama.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_coronel_macedo.py b/data_collection/gazette/spiders/sp/sp_coronel_macedo.py index 500b76f3f..96c817468 100644 --- a/data_collection/gazette/spiders/sp/sp_coronel_macedo.py +++ b/data_collection/gazette/spiders/sp/sp_coronel_macedo.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpCoronelMacedoSpider(BaseInstarSpider): TERRITORY_ID = "3512605" name = "sp_coronel_macedo" allowed_domains = ["coronelmacedo.sp.gov.br"] - start_urls = ["https://www.coronelmacedo.sp.gov.br/portal/diario-oficial"] + start_date = date(2017, 5, 29) + base_url = "https://www.coronelmacedo.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_glicerio.py b/data_collection/gazette/spiders/sp/sp_glicerio.py index 9e3017845..ee1bfe764 100644 --- a/data_collection/gazette/spiders/sp/sp_glicerio.py +++ b/data_collection/gazette/spiders/sp/sp_glicerio.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpGlicerioSpider(BaseInstarSpider): TERRITORY_ID = "3517109" name = "sp_glicerio" allowed_domains = ["glicerio.sp.gov.br"] - start_urls = ["https://www.glicerio.sp.gov.br/portal/diario-oficial/"] + start_date = date(2019, 1, 8) + base_url = "https://www.glicerio.sp.gov.br/portal/diario-oficial/" diff --git a/data_collection/gazette/spiders/sp/sp_itapirapua_paulista.py b/data_collection/gazette/spiders/sp/sp_itapirapua_paulista.py index 6be7aaf25..590c7f885 100644 --- a/data_collection/gazette/spiders/sp/sp_itapirapua_paulista.py +++ b/data_collection/gazette/spiders/sp/sp_itapirapua_paulista.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpItapirapuaPaulistaSpider(BaseInstarSpider): TERRITORY_ID = "3522653" name = "sp_itapirapua_paulista" allowed_domains = ["itapirapuapaulista.sp.gov.br"] - start_urls = ["https://www.itapirapuapaulista.sp.gov.br/portal/diario-oficial"] + start_date = date(2019, 5, 24) + base_url = "https://www.itapirapuapaulista.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_lavinia.py b/data_collection/gazette/spiders/sp/sp_lavinia.py index 67402a932..f571ccde0 100644 --- a/data_collection/gazette/spiders/sp/sp_lavinia.py +++ b/data_collection/gazette/spiders/sp/sp_lavinia.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpLaviniaSpider(BaseInstarSpider): TERRITORY_ID = "3526506" name = "sp_lavinia" allowed_domains = ["lavinia.sp.gov.br"] - start_urls = ["https://www.lavinia.sp.gov.br/portal/diario-oficial"] + start_date = date(2018, 8, 2) + base_url = "https://www.lavinia.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_monte_alto.py b/data_collection/gazette/spiders/sp/sp_monte_alto.py index 07194878b..c1f986f73 100644 --- a/data_collection/gazette/spiders/sp/sp_monte_alto.py +++ b/data_collection/gazette/spiders/sp/sp_monte_alto.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider from gazette.spiders.base.sigpub import SigpubGazetteSpider @@ -6,7 +8,8 @@ class SpMonteAltoSpider(BaseInstarSpider): TERRITORY_ID = "3531308" name = "sp_monte_alto" allowed_domains = ["montealto.instaridc.com.br"] - start_urls = ["http://montealto.instaridc.com.br/portal/diario-oficial"] + start_date = date(2017, 9, 11) + base_url = "http://montealto.instaridc.com.br/portal/diario-oficial" class SpMonteAltoSigpubSpider(SigpubGazetteSpider): diff --git a/data_collection/gazette/spiders/sp/sp_parisi.py b/data_collection/gazette/spiders/sp/sp_parisi.py index a5d433c30..7d781e9cf 100644 --- a/data_collection/gazette/spiders/sp/sp_parisi.py +++ b/data_collection/gazette/spiders/sp/sp_parisi.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpParisiSpider(BaseInstarSpider): TERRITORY_ID = "3536257" name = "sp_parisi" allowed_domains = ["parisi.sp.gov.br"] - start_urls = ["https://www.parisi.sp.gov.br/portal/diario-oficial"] + start_date = date(2015, 2, 27) + base_url = "https://www.parisi.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_patrocinio_paulista.py b/data_collection/gazette/spiders/sp/sp_patrocinio_paulista.py index 73894627a..4a8116484 100644 --- a/data_collection/gazette/spiders/sp/sp_patrocinio_paulista.py +++ b/data_collection/gazette/spiders/sp/sp_patrocinio_paulista.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpPatrocinioPaulistaSpider(BaseInstarSpider): TERRITORY_ID = "3536307" name = "sp_patrocinio_paulista" allowed_domains = ["patrociniopaulista.sp.gov.br"] - start_urls = ["https://www.patrociniopaulista.sp.gov.br/portal/diario-oficial"] + start_date = date(2017, 8, 18) + base_url = "https://www.patrociniopaulista.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_pratania.py b/data_collection/gazette/spiders/sp/sp_pratania.py index f1e407814..96d9b04e7 100644 --- a/data_collection/gazette/spiders/sp/sp_pratania.py +++ b/data_collection/gazette/spiders/sp/sp_pratania.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpPrataniaSpider(BaseInstarSpider): TERRITORY_ID = "3541059" name = "sp_pratania" allowed_domains = ["pratania.sp.gov.br"] - start_urls = ["https://www.pratania.sp.gov.br/portal/diario-oficial"] + start_date = date(2019, 5, 13) + base_url = "https://www.pratania.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_santa_ernestina.py b/data_collection/gazette/spiders/sp/sp_santa_ernestina.py index 82fcc9ee4..886f97f35 100644 --- a/data_collection/gazette/spiders/sp/sp_santa_ernestina.py +++ b/data_collection/gazette/spiders/sp/sp_santa_ernestina.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpSantaErnestinaPaulistaSpider(BaseInstarSpider): TERRITORY_ID = "3546504" name = "sp_santa_ernestina" allowed_domains = ["santaernestina.sp.gov.br"] - start_urls = ["https://www.santaernestina.sp.gov.br/portal/diario-oficial"] + start_date = date(2019, 8, 19) + base_url = "https://www.santaernestina.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_sao_manuel.py b/data_collection/gazette/spiders/sp/sp_sao_manuel.py index 033428e1c..f11330453 100644 --- a/data_collection/gazette/spiders/sp/sp_sao_manuel.py +++ b/data_collection/gazette/spiders/sp/sp_sao_manuel.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpSaoManuelSpider(BaseInstarSpider): TERRITORY_ID = "3550100" name = "sp_sao_manuel" allowed_domains = ["saomanuel.sp.gov.br"] - start_urls = ["https://www.saomanuel.sp.gov.br/portal/diario-oficial"] + start_date = date(2016, 6, 7) + base_url = "https://www.saomanuel.sp.gov.br/portal/diario-oficial" diff --git a/data_collection/gazette/spiders/sp/sp_sarutaia.py b/data_collection/gazette/spiders/sp/sp_sarutaia.py index 5458f0d04..1ab1e840c 100644 --- a/data_collection/gazette/spiders/sp/sp_sarutaia.py +++ b/data_collection/gazette/spiders/sp/sp_sarutaia.py @@ -1,3 +1,5 @@ +from datetime import date + from gazette.spiders.base.instar import BaseInstarSpider @@ -5,4 +7,5 @@ class SpSarutaiaSpider(BaseInstarSpider): TERRITORY_ID = "3551207" name = "sp_sarutaia" allowed_domains = ["sarutaia.sp.gov.br"] - start_urls = ["https://www.sarutaia.sp.gov.br/portal/diario-oficial"] + start_date = date(2020, 3, 27) + start_urls = "https://www.sarutaia.sp.gov.br/portal/diario-oficial" From 6567dac6473d4717d19490014abd88f17f8d0a88 Mon Sep 17 00:00:00 2001 From: trevineju Date: Thu, 5 Oct 2023 19:30:26 -0300 Subject: [PATCH 2/4] Separa os raspadores para Monte Alto (SP) --- data_collection/gazette/spiders/sp/sp_monte_alto.py | 11 ----------- .../gazette/spiders/sp/sp_monte_alto_2017.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 data_collection/gazette/spiders/sp/sp_monte_alto_2017.py diff --git a/data_collection/gazette/spiders/sp/sp_monte_alto.py b/data_collection/gazette/spiders/sp/sp_monte_alto.py index c1f986f73..0ab3de7d0 100644 --- a/data_collection/gazette/spiders/sp/sp_monte_alto.py +++ b/data_collection/gazette/spiders/sp/sp_monte_alto.py @@ -1,17 +1,6 @@ -from datetime import date - -from gazette.spiders.base.instar import BaseInstarSpider from gazette.spiders.base.sigpub import SigpubGazetteSpider -class SpMonteAltoSpider(BaseInstarSpider): - TERRITORY_ID = "3531308" - name = "sp_monte_alto" - allowed_domains = ["montealto.instaridc.com.br"] - start_date = date(2017, 9, 11) - base_url = "http://montealto.instaridc.com.br/portal/diario-oficial" - - class SpMonteAltoSigpubSpider(SigpubGazetteSpider): name = "sp_monte_alto_sigpub" TERRITORY_ID = "3531308" diff --git a/data_collection/gazette/spiders/sp/sp_monte_alto_2017.py b/data_collection/gazette/spiders/sp/sp_monte_alto_2017.py new file mode 100644 index 000000000..a07f0ae10 --- /dev/null +++ b/data_collection/gazette/spiders/sp/sp_monte_alto_2017.py @@ -0,0 +1,11 @@ +from datetime import date + +from gazette.spiders.base.instar import BaseInstarSpider + + +class SpMonteAltoSpider(BaseInstarSpider): + TERRITORY_ID = "3531308" + name = "sp_monte_alto_2017" + allowed_domains = ["montealto.instaridc.com.br"] + start_date = date(2017, 9, 11) + base_url = "http://montealto.instaridc.com.br/portal/diario-oficial" From 3f832e48645eee8f5c319cfdd9f7334456a8f6a2 Mon Sep 17 00:00:00 2001 From: trevineju Date: Thu, 5 Oct 2023 19:31:05 -0300 Subject: [PATCH 3/4] Ajusta sp_sarutaia.py --- data_collection/gazette/spiders/sp/sp_sarutaia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_collection/gazette/spiders/sp/sp_sarutaia.py b/data_collection/gazette/spiders/sp/sp_sarutaia.py index 1ab1e840c..d9fbc4d36 100644 --- a/data_collection/gazette/spiders/sp/sp_sarutaia.py +++ b/data_collection/gazette/spiders/sp/sp_sarutaia.py @@ -8,4 +8,4 @@ class SpSarutaiaSpider(BaseInstarSpider): name = "sp_sarutaia" allowed_domains = ["sarutaia.sp.gov.br"] start_date = date(2020, 3, 27) - start_urls = "https://www.sarutaia.sp.gov.br/portal/diario-oficial" + base_url = "https://www.sarutaia.sp.gov.br/portal/diario-oficial" From efa77e9536865a562f1a4be44ecb374f3367489f Mon Sep 17 00:00:00 2001 From: trevineju Date: Thu, 5 Oct 2023 19:31:36 -0300 Subject: [PATCH 4/4] =?UTF-8?q?Habilita=2013=20novas=20cidades=20em=20prod?= =?UTF-8?q?u=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/enabled_spiders.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/enabled_spiders.py b/scripts/enabled_spiders.py index e0e1328a4..c1a9f9d2f 100644 --- a/scripts/enabled_spiders.py +++ b/scripts/enabled_spiders.py @@ -94,28 +94,41 @@ "sc_joinville", "se_nossa_senhora_do_socorro", "sp_adolfo", + "sp_alto_alegre", + "sp_aracariguama", "sp_barao_de_antonina", "sp_birigui", "sp_braganca_paulista", "sp_campinas", "sp_catanduva", + "sp_coronel_macedo", + "sp_glicerio", "sp_guaracai", "sp_guarulhos", "sp_ibitinga", "sp_itapevi", + "sp_itapirapua_paulista", "sp_jaboticabal", "sp_jandira", "sp_jundiai", + "sp_lavinia", "sp_marilia", + "sp_monte_alto_2017", "sp_osasco", + "sp_parisi", + "sp_patrocinio_paulista", "sp_paulinia", "sp_penapolis", "sp_piedade", + "sp_pratania", "sp_rio_claro", + "sp_santa_ernestina", "sp_santo_andre", "sp_santos", "sp_sao_bernardo_do_campo", + "sp_sao_manuel", "sp_sao_roque", + "sp_sarutaia", "sp_sumare", "sp_valinhos", "sp_vera_cruz",