From 83c06629ab440969bb7ee09decf31bd61c3a3d9c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 4 Nov 2019 09:58:45 -0500 Subject: [PATCH] Support for --gzip compression levels (#84) --- Makefile | 3 +++ bin/generate-sqltomvt | 6 +++--- bin/postserve | 9 +++++---- bin/test-perf | 6 +++--- openmaptiles/sqltomvt.py | 13 ++++++++++--- testdata/expected/mvttile_query_gzip9.sql | 6 ++++++ 6 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 testdata/expected/mvttile_query_gzip9.sql diff --git a/Makefile b/Makefile index 0148aad0..427286bf 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ build-tests: \ build/mvttile_prep.sql \ build/mvttile_query.sql \ build/mvttile_query_gzip.sql \ + build/mvttile_query_gzip9.sql \ build/mvttile_query_no_feat_ids.sql \ build/doc/doc.md \ build/sqlquery.sql \ @@ -80,6 +81,8 @@ build/mvttile_query.sql: prepare $(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query > build/mvttile_query.sql build/mvttile_query_gzip.sql: prepare $(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query --gzip > build/mvttile_query_gzip.sql +build/mvttile_query_gzip9.sql: prepare + $(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query --gzip 9 > build/mvttile_query_gzip9.sql build/mvttile_query_no_feat_ids.sql: prepare $(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query --no-feature-ids > build/mvttile_query_no_feat_ids.sql build/doc/doc.md: prepare diff --git a/bin/generate-sqltomvt b/bin/generate-sqltomvt index 987a96fe..7db22687 100755 --- a/bin/generate-sqltomvt +++ b/bin/generate-sqltomvt @@ -6,7 +6,7 @@ to generate a vector tile (MVT) for the given zoom and bounding box. Usage: generate-sqltomvt [--fname ] [--function | --prepared | --query | --psql | --raw] - [--layer=]... [--key] [--gzip] [--no-feature-ids] + [--layer=]... [--key] [--gzip []] [--no-feature-ids] generate-sqltomvt --help generate-sqltomvt --version @@ -21,7 +21,7 @@ Options: -r --raw Generate raw query without any wrappers (good for debugging SQL) -l --layer= If set, limit tile generation to just this layer (could be multiple) --key If set, the result will also have a `key` column (md5 of the mvt data) - --gzip If set, MVT will be compressed using `gzip()` function. + --gzip If set, compress MVT with gzip, with optional level=0..9. `gzip()` is available from https://github.com/pramsey/pgsql-gzip --no-feature-ids Disable feature ID generation, e.g. from osm_id. You must use this flag when generating SQL for PostGIS before v3. @@ -38,7 +38,7 @@ if __name__ == '__main__': tileset=args[''], layer_ids=args['--layer'], key_column=args['--key'], - gzip=args['--gzip'], + gzip=args['--gzip'] and (args[''] or True), use_feature_id=not args['--no-feature-ids'], ) diff --git a/bin/postserve b/bin/postserve index bf3f8e18..99cb9c05 100755 --- a/bin/postserve +++ b/bin/postserve @@ -3,10 +3,11 @@ This is a simple vector tile server that returns a PBF tile for /tiles/{z}/{x}/{y}.pbf requests Usage: - postserve [--serve=] [--port=] [--key] [--gzip] [--no-feature-ids] - [--file=] [--layer=]... [--verbose] + postserve [--serve=] [--port=] [--key] [--gzip []] + [--no-feature-ids] [--file=] [--layer=]... [--pghost=] [--pgport=] [--dbname=] [--user=] [--password=] + [--verbose] postserve --help postserve --version @@ -17,7 +18,7 @@ Options: -s --serve= Return this hostname as tileserver URL in metadata [default: localhost] -p --port= Serve on this port [default: 8090] --key If set, print md5 of the data to console (generated by Postgres) - --gzip If set, MVT will be compressed using `gzip()` function. + --gzip If set, compress MVT with gzip, with optional level=0..9. --no-feature-ids Disable feature ID generation, e.g. from osm_id. Feature IDS are automatically disabled with PostGIS before v3. -v --verbose Print additional debugging information @@ -86,7 +87,7 @@ if __name__ == '__main__': tileset_path=args[''], sql_file=args.get('--file'), key_column=args['--key'], - gzip=args['--gzip'], + gzip=args['--gzip'] and (args[''] or True), disable_feature_ids=args.get('--no-feature-ids'), verbose=args.get('--verbose'), ).serve() diff --git a/bin/test-perf b/bin/test-perf index 12284049..b94af5ec 100755 --- a/bin/test-perf +++ b/bin/test-perf @@ -7,7 +7,7 @@ Usage: [--per-layer] [--summary] [--test-all] ([--zoom=]... | [--minzoom=] [--maxzoom=]) [--record=] [--compare=] [--buckets=] - [--key] [--gzip] [--no-color] [--no-feature-ids] [--verbose] + [--key] [--gzip []] [--no-color] [--no-feature-ids] [--verbose] test-perf --help test-perf --version @@ -26,7 +26,7 @@ Options: -c --compare= Compare performance run results with a previous run -b --buckets= Show up to this many buckets in a graph [default: 10] --key Generate md5 keys for all tiles (resulting key is ignored) - --gzip If set, MVT will be compressed using `gzip()` function. + --gzip If set, compress MVT with gzip, with optional level=0..9. --no-color Disable ANSI colors --no-feature-ids Disable feature ID generation, e.g. from osm_id. Feature IDS are automatically disabled with PostGIS before v3. @@ -88,7 +88,7 @@ def main(args): disable_colors=args.get('--no-color'), disable_feature_ids=args.get('--no-feature-ids'), key_column=args['--key'], - gzip=args['--gzip'], + gzip=args['--gzip'] and (args[''] or True), verbose=args.get('--verbose'), ) asyncio.run(perf.run()) diff --git a/openmaptiles/sqltomvt.py b/openmaptiles/sqltomvt.py index 662e7f73..a4b3934b 100644 --- a/openmaptiles/sqltomvt.py +++ b/openmaptiles/sqltomvt.py @@ -13,7 +13,7 @@ class MvtGenerator: layer_ids: Set[str] def __init__(self, tileset: Union[str, Tileset], layer_ids: Iterable[str] = None, - key_column=False, gzip=False, use_feature_id=True): + key_column=False, gzip: Union[int, bool] = False, use_feature_id=True): if isinstance(tileset, str): self.tileset = Tileset.parse(tileset) else: @@ -84,9 +84,16 @@ def generate_query(self, bbox: str = None, zoom: str = None): raise DocoptExit('Could not find any layer definitions') concatenate_layers = "STRING_AGG(mvtl, '')" - if self.gzip: + # Handle when gzip is True or a number + # Note that any bool is an int, but not reverse: isinstance(False, int) == True + if not isinstance(self.gzip, bool) or self.gzip: # GZIP function is available from https://github.com/pramsey/pgsql-gzip - concatenate_layers = f"GZIP({concatenate_layers})" + if isinstance(self.gzip, bool): + concatenate_layers = f"GZIP({concatenate_layers})" + else: + self.gzip = int(self.gzip) + assert 0 <= self.gzip <= 9 + concatenate_layers = f"GZIP({concatenate_layers}, {self.gzip})" union_layers = "\n UNION ALL\n ".join(queries) query = f"""\ SELECT {concatenate_layers} AS mvt FROM ( diff --git a/testdata/expected/mvttile_query_gzip9.sql b/testdata/expected/mvttile_query_gzip9.sql new file mode 100644 index 00000000..b1c8dfe5 --- /dev/null +++ b/testdata/expected/mvttile_query_gzip9.sql @@ -0,0 +1,6 @@ +SELECT GZIP(STRING_AGG(mvtl, ''), 9) AS mvt FROM ( + SELECT COALESCE(ST_AsMVT(t, 'housenumber', 4096, 'mvtgeometry'), '') as mvtl FROM (SELECT ST_AsMVTGeom(geometry, TileBBox($1, $2, $3), 4096, 8, true) AS mvtgeometry, housenumber, NULLIF(tags->'name:en', '') AS "name:en", NULLIF(tags->'name:de', '') AS "name:de", NULLIF(tags->'name:cs', '') AS "name:cs", NULLIF(tags->'name_int', '') AS "name_int", NULLIF(tags->'name:latin', '') AS "name:latin", NULLIF(tags->'name:nonlatin', '') AS "name:nonlatin" FROM layer_housenumber(TileBBox($1, $2, $3), $1)) AS t + UNION ALL + SELECT COALESCE(ST_AsMVT(t, 'enumfield', 4096, 'mvtgeometry', 'osm_id'), '') as mvtl FROM (SELECT osm_id, ST_AsMVTGeom(geometry, TileBBox($1, $2, $3), 4096, 0, true) AS mvtgeometry, enumfield FROM layer_enumfields(TileBBox($1, $2, $3), $1)) AS t +) AS all_layers +