Support for --gzip compression levels (#84)

Makefile

@@ -36,6 +36,7 @@ build-tests: \
     build/mvttile_prep.sql \
     build/mvttile_query.sql \
     build/mvttile_query_gzip.sql \
+    build/mvttile_query_gzip9.sql \
     build/mvttile_query_no_feat_ids.sql \
     build/doc/doc.md \
     build/sqlquery.sql \
@@ -80,6 +81,8 @@ build/mvttile_query.sql: prepare
 	$(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query                              > build/mvttile_query.sql
 build/mvttile_query_gzip.sql: prepare
 	$(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query --gzip                       > build/mvttile_query_gzip.sql
+build/mvttile_query_gzip9.sql: prepare
+	$(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query --gzip 9                     > build/mvttile_query_gzip9.sql
 build/mvttile_query_no_feat_ids.sql: prepare
 	$(RUN_CMD) generate-sqltomvt testdata/testlayers/testmaptiles.yaml --query --no-feature-ids             > build/mvttile_query_no_feat_ids.sql
 build/doc/doc.md: prepare

bin/generate-sqltomvt

@@ -6,7 +6,7 @@ to generate a vector tile (MVT) for the given zoom and bounding box.
 Usage:
   generate-sqltomvt <tileset> [--fname <name>]
                     [--function | --prepared | --query | --psql | --raw]
-                    [--layer=<layer>]... [--key] [--gzip] [--no-feature-ids]
+                    [--layer=<layer>]... [--key] [--gzip [<gzlevel>]] [--no-feature-ids]
   generate-sqltomvt --help
   generate-sqltomvt --version
 
@@ -21,7 +21,7 @@ Options:
   -r --raw               Generate raw query without any wrappers (good for debugging SQL)
   -l --layer=<layer>     If set, limit tile generation to just this layer (could be multiple)
   --key                  If set, the result will also have a `key` column (md5 of the mvt data)
-  --gzip                 If set, MVT will be compressed using `gzip()` function.
+  --gzip                 If set, compress MVT with gzip, with optional level=0..9.
                          `gzip()` is available from https://github.com/pramsey/pgsql-gzip
   --no-feature-ids       Disable feature ID generation, e.g. from osm_id.
                          You must use this flag when generating SQL for PostGIS before v3.
@@ -38,7 +38,7 @@ if __name__ == '__main__':
         tileset=args['<tileset>'],
         layer_ids=args['--layer'],
         key_column=args['--key'],
-        gzip=args['--gzip'],
+        gzip=args['--gzip'] and (args['<gzlevel>'] or True),
         use_feature_id=not args['--no-feature-ids'],
     )
 

bin/postserve

@@ -3,10 +3,11 @@
 This is a simple vector tile server that returns a PBF tile for  /tiles/{z}/{x}/{y}.pbf  requests
 
 Usage:
-  postserve <tileset> [--serve=<host>] [--port=<port>] [--key] [--gzip] [--no-feature-ids]
-                      [--file=<sql-file>] [--layer=<layer>]... [--verbose]
+  postserve <tileset> [--serve=<host>] [--port=<port>] [--key] [--gzip [<gzlevel>]]
+                      [--no-feature-ids] [--file=<sql-file>] [--layer=<layer>]...
                       [--pghost=<host>] [--pgport=<port>] [--dbname=<db>]
                       [--user=<user>] [--password=<password>]
+                      [--verbose]
   postserve --help
   postserve --version
 
@@ -17,7 +18,7 @@ Options:
   -s --serve=<port>     Return this hostname as tileserver URL in metadata  [default: localhost]
   -p --port=<port>      Serve on this port  [default: 8090]
   --key                 If set, print md5 of the data to console (generated by Postgres)
-  --gzip                If set, MVT will be compressed using `gzip()` function.
+  --gzip                If set, compress MVT with gzip, with optional level=0..9.
   --no-feature-ids      Disable feature ID generation, e.g. from osm_id.
                         Feature IDS are automatically disabled with PostGIS before v3.
   -v --verbose          Print additional debugging information
@@ -86,7 +87,7 @@ if __name__ == '__main__':
         tileset_path=args['<tileset>'],
         sql_file=args.get('--file'),
         key_column=args['--key'],
-        gzip=args['--gzip'],
+        gzip=args['--gzip'] and (args['<gzlevel>'] or True),
         disable_feature_ids=args.get('--no-feature-ids'),
         verbose=args.get('--verbose'),
     ).serve()

bin/test-perf

@@ -7,7 +7,7 @@ Usage:
               [--per-layer] [--summary] [--test-all]
               ([--zoom=<zoom>]... | [--minzoom=<min>] [--maxzoom=<max>])
               [--record=<file>] [--compare=<file>] [--buckets=<count>]
-              [--key] [--gzip] [--no-color] [--no-feature-ids] [--verbose]
+              [--key] [--gzip [<gzlevel>]] [--no-color] [--no-feature-ids] [--verbose]
   test-perf --help
   test-perf --version
 
@@ -26,7 +26,7 @@ Options:
   -c --compare=<file>   Compare performance run results with a previous run
   -b --buckets=<count>  Show up to this many buckets in a graph  [default: 10]
   --key                 Generate md5 keys for all tiles (resulting key is ignored)
-  --gzip                If set, MVT will be compressed using `gzip()` function.
+  --gzip                If set, compress MVT with gzip, with optional level=0..9.
   --no-color            Disable ANSI colors
   --no-feature-ids      Disable feature ID generation, e.g. from osm_id.
                         Feature IDS are automatically disabled with PostGIS before v3.
@@ -88,7 +88,7 @@ def main(args):
         disable_colors=args.get('--no-color'),
         disable_feature_ids=args.get('--no-feature-ids'),
         key_column=args['--key'],
-        gzip=args['--gzip'],
+        gzip=args['--gzip'] and (args['<gzlevel>'] or True),
         verbose=args.get('--verbose'),
     )
     asyncio.run(perf.run())

openmaptiles/sqltomvt.py

@@ -13,7 +13,7 @@ class MvtGenerator:
     layer_ids: Set[str]
 
     def __init__(self, tileset: Union[str, Tileset], layer_ids: Iterable[str] = None,
-                 key_column=False, gzip=False, use_feature_id=True):
+                 key_column=False, gzip: Union[int, bool] = False, use_feature_id=True):
         if isinstance(tileset, str):
             self.tileset = Tileset.parse(tileset)
         else:
@@ -84,9 +84,16 @@ def generate_query(self, bbox: str = None, zoom: str = None):
             raise DocoptExit('Could not find any layer definitions')
 
         concatenate_layers = "STRING_AGG(mvtl, '')"
-        if self.gzip:
+        # Handle when gzip is True or a number
+        # Note that any bool is an int, but not reverse: isinstance(False, int) == True
+        if not isinstance(self.gzip, bool) or self.gzip:
             # GZIP function is available from https://github.com/pramsey/pgsql-gzip
-            concatenate_layers = f"GZIP({concatenate_layers})"
+            if isinstance(self.gzip, bool):
+                concatenate_layers = f"GZIP({concatenate_layers})"
+            else:
+                self.gzip = int(self.gzip)
+                assert 0 <= self.gzip <= 9
+                concatenate_layers = f"GZIP({concatenate_layers}, {self.gzip})"
         union_layers = "\n    UNION ALL\n  ".join(queries)
         query = f"""\
 SELECT {concatenate_layers} AS mvt FROM (

testdata/expected/mvttile_query_gzip9.sql

@@ -0,0 +1,6 @@
+SELECT GZIP(STRING_AGG(mvtl, ''), 9) AS mvt FROM (
+  SELECT COALESCE(ST_AsMVT(t, 'housenumber', 4096, 'mvtgeometry'), '') as mvtl FROM (SELECT ST_AsMVTGeom(geometry, TileBBox($1, $2, $3), 4096, 8, true) AS mvtgeometry, housenumber, NULLIF(tags->'name:en', '') AS "name:en", NULLIF(tags->'name:de', '') AS "name:de", NULLIF(tags->'name:cs', '') AS "name:cs", NULLIF(tags->'name_int', '') AS "name_int", NULLIF(tags->'name:latin', '') AS "name:latin", NULLIF(tags->'name:nonlatin', '') AS "name:nonlatin" FROM layer_housenumber(TileBBox($1, $2, $3), $1)) AS t
+    UNION ALL
+  SELECT COALESCE(ST_AsMVT(t, 'enumfield', 4096, 'mvtgeometry', 'osm_id'), '') as mvtl FROM (SELECT osm_id, ST_AsMVTGeom(geometry, TileBBox($1, $2, $3), 4096, 0, true) AS mvtgeometry, enumfield FROM layer_enumfields(TileBBox($1, $2, $3), $1)) AS t
+) AS all_layers
+