From b6b6f19a8d0ba8b99419e5819c15726623173364 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Sat, 9 Sep 2023 08:44:17 -0500 Subject: [PATCH 1/8] Use file extension from content disposition header instead of giving up --- openaddr/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openaddr/cache.py b/openaddr/cache.py index aac53f22..97108a12 100644 --- a/openaddr/cache.py +++ b/openaddr/cache.py @@ -207,8 +207,8 @@ def guess_url_file_extension(url): if path_ext == attachment_ext: _L.debug('Content-Disposition agrees: "{}"'.format(match.group('filename'))) else: - _L.debug('Content-Disposition disagrees: "{}"'.format(match.group('filename'))) - path_ext = False + _L.debug('Content-Disposition disagrees: "{}", using "{}" instead'.format(match.group('filename'), attachment_ext)) + path_ext = attachment_ext if not path_ext: # From 263afce59c2d236bf0d1e4d26263329d5af53eff Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Sat, 9 Sep 2023 08:52:28 -0500 Subject: [PATCH 2/8] Use file extension from content-type instead --- openaddr/cache.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openaddr/cache.py b/openaddr/cache.py index 97108a12..f06f0ef3 100644 --- a/openaddr/cache.py +++ b/openaddr/cache.py @@ -207,8 +207,11 @@ def guess_url_file_extension(url): if path_ext == attachment_ext: _L.debug('Content-Disposition agrees: "{}"'.format(match.group('filename'))) else: - _L.debug('Content-Disposition disagrees: "{}", using "{}" instead'.format(match.group('filename'), attachment_ext)) - path_ext = attachment_ext + _L.debug('Content-Disposition disagrees: "{}" says we should use "{}", using "{}" instead'.format( + match.group('filename'), + attachment_ext, + path_ext, + )) if not path_ext: # From 6e1fd4e0635e2162ca0df11b7ff63346d4500640 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Sat, 9 Sep 2023 09:07:38 -0500 Subject: [PATCH 3/8] Update README --- CHANGELOG | 3 +++ openaddr/VERSION | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 6524fe8a..042d60cc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +2023-09-09 v8.2.5 +- Use the file extension from the content-type header when it disagrees with content-disposition in https://github.com/openaddresses/batch-machine/pull/54 + 2023-09-05 v8.2.4 - Update TippeCanoe to latest version - Make GeoJSON+LD output a default diff --git a/openaddr/VERSION b/openaddr/VERSION index 1365b923..fc8336ce 100644 --- a/openaddr/VERSION +++ b/openaddr/VERSION @@ -1 +1 @@ -8.2.3 +8.2.5 From 4f57e3e6be53695ef0a4a8db60139154de09b6cf Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Sat, 9 Sep 2023 09:11:56 -0500 Subject: [PATCH 4/8] Update Changelog, I used the wrong tag --- CHANGELOG | 9 +++++---- openaddr/VERSION | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 042d60cc..73fb7389 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,10 @@ -2023-09-09 v8.2.5 +2023-09-09 v8.3.1 - Use the file extension from the content-type header when it disagrees with content-disposition in https://github.com/openaddresses/batch-machine/pull/54 -2023-09-05 v8.2.4 -- Update TippeCanoe to latest version -- Make GeoJSON+LD output a default +2023-09-05 v8.3.0 +- Update Mapbox Vector Tile dependency in https://github.com/openaddresses/batch-machine/pull/52 +- Make GeoJSON+LD output a default in https://github.com/openaddresses/batch-machine/pull/51 +- Update TippeCanoe to latest version in https://github.com/openaddresses/batch-machine/pull/50 2023-09-02 v8.2.3 - Fix properties in geojson-ld features in https://github.com/openaddresses/batch-machine/pull/49 diff --git a/openaddr/VERSION b/openaddr/VERSION index fc8336ce..56b6be4e 100644 --- a/openaddr/VERSION +++ b/openaddr/VERSION @@ -1 +1 @@ -8.2.5 +8.3.1 From facf91a7563789ed27fe3217cd98ca21db6fbc3e Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 09:19:02 +0100 Subject: [PATCH 5/8] Remove field limit --- openaddr/tests/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 2455e18c..7a61260c 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -989,10 +989,7 @@ def test_single_tx_waco(self): source = join(self.src_dir, 'us/tx/city_of_waco.json') with HTTMock(self.response_content): - ofs = csv.field_size_limit() - csv.field_size_limit(1) state_path = process_one.process(source, self.testdir, "addresses", "default", False, False) - csv.field_size_limit(ofs) with open(state_path) as file: state = dict(zip(*json.load(file))) From d11c6b069ff1bab6d295bd8023992723aa452a28 Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 09:26:44 +0100 Subject: [PATCH 6/8] Fix Empty Geom Error --- openaddr/conform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 5b240079..b2b3240c 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -1083,7 +1083,7 @@ def row_convert_to_out(source_config, row): "Convert a row from the source schema to OpenAddresses output schema" geom = row.get(GEOM_FIELDNAME.lower(), None) - if geom == "POINT EMPTY": + if geom == "POINT EMPTY" or geom == '': geom = None output = { From 8775cc14c1d2c5ecff42c1ac5c7994d15a27cb0d Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 11:43:16 +0100 Subject: [PATCH 7/8] Finish getting tests passing --- openaddr/conform.py | 2 +- openaddr/tests/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index b2b3240c..0b084140 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -388,7 +388,7 @@ def convert(self, source_config, source_paths, workdir): rc = conform_cli(source_config, source_path, dest_path) if rc == 0: with open(dest_path) as file: - addr_count = sum(1 for line in file) - 1 + addr_count = sum(1 for line in file) # Success! Return the path of the output CSV return dest_path, addr_count diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index 7a61260c..be71dbca 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -994,8 +994,8 @@ def test_single_tx_waco(self): with open(state_path) as file: state = dict(zip(*json.load(file))) - self.assertEqual(state["source problem"], "Could not conform source data") - self.assertIsNone(state["processed"]) + self.assertIsNone(state["source problem"]) + self.assertEqual(state["processed"], 'out.geojson') source = join(self.src_dir, 'us/tx/city_of_waco.json') @@ -1019,7 +1019,7 @@ def test_single_tx_waco(self): self.assertEqual(rows[0]['properties']['region'], u'TX') self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'308') - self.assertEqual(rows[0]['properties']['hash'], u'c782a47f7d52d99f') + self.assertEqual(rows[0]['properties']['hash'], u'431f816eebac0000') self.assertEqual(rows[0]['properties']['city'], u'Mcgregor') self.assertEqual(rows[0]['geometry']['coordinates'], [-97.3961768, 31.4432706]), self.assertEqual(rows[0]['properties']['street'], u'PULLEN ST') From 20325d6817a2a37f86bceade124c086e02bcc92c Mon Sep 17 00:00:00 2001 From: ingalls Date: Mon, 18 Sep 2023 10:40:34 -0500 Subject: [PATCH 8/8] Fix CHANGELOG --- CHANGELOG | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d606f7b1..3f51f62b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,19 +1,9 @@ -<<<<<<< HEAD 2023-09-17 v9.0.0 - Remove GeoJSONLD Render Functions - Remove MBTiles Render Functions - Rewrite confrom to natively output GeoJSON+LD - Rewrite Preview module to expect GeoJSON+LD -2023-09-06 v8.3.0 -- Update to next major alpine version -- Remove mock library in favour of unittest.mock -- Update all python deps to latest versions - -2023-09-05 v8.2.4 -- Update TippeCanoe to latest version -- Make GeoJSON+LD output a default -======= 2023-09-09 v8.3.1 - Use the file extension from the content-type header when it disagrees with content-disposition in https://github.com/openaddresses/batch-machine/pull/54 @@ -21,8 +11,10 @@ - Update Mapbox Vector Tile dependency in https://github.com/openaddresses/batch-machine/pull/52 - Make GeoJSON+LD output a default in https://github.com/openaddresses/batch-machine/pull/51 - Update TippeCanoe to latest version in https://github.com/openaddresses/batch-machine/pull/50 ->>>>>>> 4f57e3e6be53695ef0a4a8db60139154de09b6cf +2023-09-05 v8.2.4 +- Update TippeCanoe to latest version +- Make GeoJSON+LD output a default 2023-09-02 v8.2.3 - Fix properties in geojson-ld features in https://github.com/openaddresses/batch-machine/pull/49 - Remove extra whitespace in geojson-ld output in https://github.com/openaddresses/batch-machine/pull/48