diff --git a/celery-cmd b/celery-cmd index a1c7fbd98fc..a600163fc77 100644 --- a/celery-cmd +++ b/celery-cmd @@ -3,7 +3,7 @@ # Luca Pasquali CELERY_BIN=${CELERY_BIN:-"$(which celery||echo celery)"} CELERY_APP=${CELERY_APP:-"geonode.celery_app:app"} -CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker.state"} +CELERY__STATE_DB=${CELERY__STATE_DB:-"/mnt/volumes/statics/worker@%h.state"} # expressed in KB CELERY__MAX_MEMORY_PER_CHILD=${CELERY__MAX_MEMORY_PER_CHILD:-"200000"} CELERY__AUTOSCALE_VALUES=${CELERY__AUTOSCALE_VALUES:-"15,10"} diff --git a/geonode/storage/tests.py b/geonode/storage/tests.py index 89eea61d4ad..48d1f3ebfe2 100644 --- a/geonode/storage/tests.py +++ b/geonode/storage/tests.py @@ -197,6 +197,7 @@ def test_google_size(self, gcs): gcs.assert_called_once_with("name") +@override_settings(AWS_STORAGE_BUCKET_NAME="my-bucket-name") class TestAwsStorageManager(SimpleTestCase): def setUp(self): self.sut = AwsStorageManager diff --git a/geonode/thumbs/background.py b/geonode/thumbs/background.py index 8896a5353c3..85bceb64159 100644 --- a/geonode/thumbs/background.py +++ b/geonode/thumbs/background.py @@ -21,13 +21,16 @@ import ast import typing import logging +import math import mercantile +import requests from io import BytesIO from pyproj import Transformer from abc import ABC, abstractmethod from math import ceil, floor, copysign from PIL import Image, UnidentifiedImageError +from owslib.wmts import WebMapTileService from django.conf import settings from django.utils.html import strip_tags @@ -464,3 +467,200 @@ def __init__( self.url = "https://tile.openstreetmap.org/{z}/{x}/{y}.png" self.tile_size = 256 + + +WMTS_TILEMATRIXSET_LEVELS = None + + +class GenericWMTSBackground(BaseThumbBackground): + def __init__(self, thumbnail_width: int, thumbnail_height: int, max_retries: int = 3, retry_delay: int = 1): + super().__init__(thumbnail_width, thumbnail_height, max_retries, retry_delay) + self.options = settings.THUMBNAIL_BACKGROUND.get("options", {}) + self.levels = self.get_levels_for_tilematrix() + + self.thumbnail_width = thumbnail_width + self.thumbnail_height = thumbnail_height + + def fetch(self, bbox: typing.List, *args, **kwargs): + bbox = [bbox[0], bbox[2], bbox[1], bbox[3]] + target_pixelspan = self.get_target_pixelspan(bbox) + level = self.get_level_for_targetpixelspan(target_pixelspan) + + tilewidth = level["tilewidth"] + tileheight = level["tileheight"] + zoom = level["zoom"] + pixelspan = level["pixelspan"] + tilespanx = level["tilespanx"] + tilespany = level["tilespany"] + + pixelspan_ratio = level["pixelspan"] / target_pixelspan + + tile_rowcols = self.get_tiles_coords(level, bbox) + tiles_cols_list = set([tile_rowcol[0] for tile_rowcol in tile_rowcols]) + tiles_mincol = min(tiles_cols_list) + tiles_maxcol = max(tiles_cols_list) + tiles_minx = level["bounds"][0] + (tiles_mincol * tilespanx) + tiles_rows_list = set([tile_rowcol[1] for tile_rowcol in tile_rowcols]) + tiles_minrow = min(tiles_rows_list) + tiles_maxrow = max(tiles_rows_list) + tiles_maxy = level["bounds"][3] - (tiles_minrow * tilespany) + + tiles_width = (tiles_maxcol - tiles_mincol + 1) * tilewidth + tiles_height = (tiles_maxrow - tiles_minrow + 1) * tileheight + + background = Image.new("RGB", (tiles_width, tiles_height), (250, 250, 250)) + + for tile_coord in tile_rowcols: + try: + im = None + imgurl = self.build_request([tile_coord[0], tile_coord[1], zoom]) + resp = requests.get(imgurl) + if resp.status_code > 400: + raise Exception(f"{strip_tags(resp.content)}") + im = BytesIO(resp.content) + Image.open(im).verify() + if im: + offsetx = (tile_coord[0] - tiles_mincol) * tilewidth + offsety = (tile_coord[1] - tiles_minrow) * tileheight + image = Image.open(im) + background.paste(image, (offsetx, offsety)) + except Exception as e: + logger.error(f"Error fetching {imgurl} for thumbnail: {e}") + + left = abs(tiles_minx - bbox[0]) / pixelspan + right = left + self.thumbnail_width + top = abs(tiles_maxy - bbox[3]) / pixelspan + bottom = top + self.thumbnail_height + background = background.crop((left, top, right, bottom)) + + width = round(self.thumbnail_width * pixelspan_ratio) + height = round(self.thumbnail_height * pixelspan_ratio) + + background = background.resize((width, height)) + background.crop((left, top, right, bottom)) + + return background + + def build_kvp_request(self, baseurl, layer, style, xyz): + return f"{baseurl}?&Service=WMTS&Request=GetTile&Version=1.0.0&Format=image/png&layer={layer}&style={style} \ + &tilematrixset={self.options['tilematrixset']}&TileMatrix={xyz[2]}&TileRow={xyz[1]}&TileCol={xyz[0]}" + + def build_request(self, xyz): + request_encoding = self.options.get("requestencoding", "KVP") + baseurl = self.options["url"] + layer = self.options["layer"] + style = self.options["style"] + + imgurl = None + if request_encoding == "KVP": + imgurl = self.build_kvp_request(baseurl, layer, style, xyz) + + return imgurl + + def get_image_bbox_for_level(self, level, bbox): + image_width = self.thumbnail_width + image_height = self.thumbnail_height + + half_imagespanx = image_width * level["pixelspan"] / 2 + half_imagespany = image_height * level["pixelspan"] / 2 + + ( + boundsminx, + boundsminy, + boundsmaxx, + boundsmaxy, + ) = bbox + + bboxcentrex = boundsminx + ((boundsmaxx - boundsminx) / 2) + bboxcentrey = boundsminy + ((boundsmaxy - boundsminy) / 2) + + image_minx = bboxcentrex - half_imagespanx + image_maxx = bboxcentrex + half_imagespanx + image_miny = bboxcentrey - half_imagespany + image_maxy = bboxcentrey + half_imagespany + + return [image_minx, image_miny, image_maxx, image_maxy] + + def get_tiles_coords(self, level, bbox): + tile_coords = [] + + tilematrixminx = level["bounds"][0] + tilematrixmaxy = level["bounds"][3] + tilespanx = level["tilespanx"] + tilespany = level["tilespany"] + + boundsminx, boundsminy, boundsmaxx, boundsmaxy = bbox + + tile_coord_minx = int(math.floor(boundsminx - tilematrixminx) / tilespanx) + # min tile coord corresponds to the maxy coordinate + tile_coord_miny = int(math.floor(tilematrixmaxy - boundsmaxy) / tilespany) + tile_coord_maxx = int(math.floor(boundsmaxx - tilematrixminx) / tilespanx) + # max tile coord corresponds to the miny coordinate + tile_coord_maxy = int(math.floor(tilematrixmaxy - boundsminy) / tilespany) + + for x in range(tile_coord_minx, tile_coord_maxx + 1): + for y in range(tile_coord_miny, tile_coord_maxy + 1): + tile_coords.append([x, y]) + + return tile_coords + + def get_level_for_targetpixelspan(self, target_pixelspan): + level = None + for _level in self.levels: + is_level_under_minscaledenominator = False + minscaledenominator = self.options.get("minscaledenominator") + if minscaledenominator: + is_level_under_minscaledenominator = _level["scaledenominator"] < self.options.get( + "minscaledenominator" + ) + if _level["pixelspan"] < target_pixelspan or is_level_under_minscaledenominator: + return level + level = _level + + def get_target_pixelspan(self, bbox): + x_min, y_min, x_max, y_max = bbox + return (x_max - x_min) / self.thumbnail_width + + def get_levels_for_tilematrix(self): + url = self.options["url"] + tilematrixset = self.options["tilematrixset"] + global WMTS_TILEMATRIXSET_LEVELS + if not WMTS_TILEMATRIXSET_LEVELS: + service = WebMapTileService(url=url) + tilematrixsset = service.tilematrixsets[tilematrixset] + + levels = [] + for index, tilematrix in tilematrixsset.tilematrix.items(): + scaledenominator = tilematrix.scaledenominator * 1 # here we assume 3857 + matrixheight = tilematrix.matrixheight + matrixwidth = tilematrix.matrixwidth + tileheight = tilematrix.tileheight + tilewidth = tilematrix.tilewidth + tilematrixminx = tilematrix.topleftcorner[0] # here we assume 3857 + tilematrixmaxy = tilematrix.topleftcorner[1] # here we assume 3857 + + pixelspan = scaledenominator * 0.00028 # OGC standardized rendering pixel size + tilespanx = tilewidth * pixelspan + tilespany = tileheight * pixelspan + tilematrixmaxx = tilematrixminx + tilespanx * matrixwidth + tilematrixminy = tilematrixmaxy - tilespany * matrixheight + + levels.append( + { + "zoom": int(index), + "bounds": [ + tilematrixminx, + tilematrixminy, + tilematrixmaxx, + tilematrixmaxy, + ], + "scaledenominator": scaledenominator, + "tilewidth": tilewidth, + "tileheight": tileheight, + "pixelspan": pixelspan, + "tilespanx": tilespanx, + "tilespany": tilespany, + } + ) + WMTS_TILEMATRIXSET_LEVELS = levels + return WMTS_TILEMATRIXSET_LEVELS diff --git a/geonode/thumbs/tests/expected_results/tiles/background.png b/geonode/thumbs/tests/expected_results/tiles/background.png new file mode 100644 index 00000000000..d345f1ce362 Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/background.png differ diff --git a/geonode/thumbs/tests/expected_results/tiles/wmts_7_5_4.png b/geonode/thumbs/tests/expected_results/tiles/wmts_7_5_4.png new file mode 100644 index 00000000000..3ee80d90d4b Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/wmts_7_5_4.png differ diff --git a/geonode/thumbs/tests/expected_results/tiles/wmts_7_6_4.png b/geonode/thumbs/tests/expected_results/tiles/wmts_7_6_4.png new file mode 100644 index 00000000000..de1b9c8d700 Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/wmts_7_6_4.png differ diff --git a/geonode/thumbs/tests/expected_results/tiles/wmts_8_5_4.png b/geonode/thumbs/tests/expected_results/tiles/wmts_8_5_4.png new file mode 100644 index 00000000000..f6f4a57d281 Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/wmts_8_5_4.png differ diff --git a/geonode/thumbs/tests/expected_results/tiles/wmts_8_6_4.png b/geonode/thumbs/tests/expected_results/tiles/wmts_8_6_4.png new file mode 100644 index 00000000000..778ae94ceaf Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/wmts_8_6_4.png differ diff --git a/geonode/thumbs/tests/expected_results/tiles/wmts_9_5_4.png b/geonode/thumbs/tests/expected_results/tiles/wmts_9_5_4.png new file mode 100644 index 00000000000..5ce6c42cc42 Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/wmts_9_5_4.png differ diff --git a/geonode/thumbs/tests/expected_results/tiles/wmts_9_6_4.png b/geonode/thumbs/tests/expected_results/tiles/wmts_9_6_4.png new file mode 100644 index 00000000000..7c46fbcc0ea Binary files /dev/null and b/geonode/thumbs/tests/expected_results/tiles/wmts_9_6_4.png differ diff --git a/geonode/thumbs/tests/test_backgrounds.py b/geonode/thumbs/tests/test_backgrounds.py new file mode 100644 index 00000000000..896ef0685dc --- /dev/null +++ b/geonode/thumbs/tests/test_backgrounds.py @@ -0,0 +1,371 @@ +######################################################################### +# +# Copyright (C) 2023 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import os +import logging +from unittest.mock import patch + +from pixelmatch.contrib.PIL import pixelmatch +from PIL import Image + +from django.test import override_settings + +from geonode.tests.base import GeoNodeBaseTestSupport +from geonode.thumbs.background import GenericWMTSBackground + +logger = logging.getLogger(__name__) + +WMTS_TILEMATRIX_LEVELS = [ + { + "zoom": 0, + "bounds": [-20037508.342787, -60112525.02833891, 20037508.342775952, 20037508.342787], + "scaledenominator": 559082264.028501, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 156543.0339279803, + "tilespanx": 40075016.68556295, + "tilespany": 40075016.68556295, + }, + { + "zoom": 1, + "bounds": [-20037508.342787, -40075016.68555735, 20037508.3427759, 20037508.342787], + "scaledenominator": 279541132.01425016, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 78271.51696399004, + "tilespanx": 20037508.34278145, + "tilespany": 20037508.34278145, + }, + { + "zoom": 2, + "bounds": [-20037508.342787, -40075016.685557604, 20037508.342776064, 20037508.342787], + "scaledenominator": 139770566.00712565, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 39135.75848199518, + "tilespanx": 10018754.171390766, + "tilespany": 10018754.171390766, + }, + { + "zoom": 3, + "bounds": [-20037508.342787, -35065639.599861786, 20037508.34277575, 20037508.342787], + "scaledenominator": 69885283.00356229, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 19567.879240997438, + "tilespanx": 5009377.085695344, + "tilespany": 5009377.085695344, + }, + { + "zoom": 4, + "bounds": [-20037508.342787, -32560951.057014164, 20037508.34277579, 20037508.342787], + "scaledenominator": 34942641.50178117, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 9783.939620498728, + "tilespanx": 2504688.5428476743, + "tilespany": 2504688.5428476743, + }, + { + "zoom": 5, + "bounds": [-20037508.342787, -31308606.785590325, 20037508.34277579, 20037508.342787], + "scaledenominator": 17471320.750890587, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 4891.969810249364, + "tilespanx": 1252344.2714238372, + "tilespany": 1252344.2714238372, + }, + { + "zoom": 6, + "bounds": [-20037508.342787, -30682434.6498784, 20037508.34277579, 20037508.342787], + "scaledenominator": 8735660.375445293, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 2445.984905124682, + "tilespanx": 626172.1357119186, + "tilespany": 626172.1357119186, + }, + { + "zoom": 7, + "bounds": [-20037508.342787, -30369348.58202224, 20037508.342775624, 20037508.342787], + "scaledenominator": 4367830.187722629, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 1222.992452562336, + "tilespanx": 313086.067855958, + "tilespany": 313086.067855958, + }, + { + "zoom": 8, + "bounds": [-20037508.342787, -30369348.58203337, 20037508.342784476, 20037508.342787], + "scaledenominator": 2183915.093861797, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 611.496226281303, + "tilespanx": 156543.03392801358, + "tilespany": 156543.03392801358, + }, + { + "zoom": 9, + "bounds": [-20037508.342787, -30291077.06504764, 20037508.342767175, 20037508.342787], + "scaledenominator": 1091957.546930427, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 305.74811314051954, + "tilespanx": 78271.516963973, + "tilespany": 78271.516963973, + }, + { + "zoom": 10, + "bounds": [-20037508.342787, -30251941.306609083, 20037508.342801783, 20037508.342787], + "scaledenominator": 545978.773465685, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 152.8740565703918, + "tilespanx": 39135.7584820203, + "tilespany": 39135.7584820203, + }, + { + "zoom": 11, + "bounds": [-20037508.342787, -30251941.30652203, 20037508.34273241, 20037508.342787], + "scaledenominator": 272989.38673236995, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 76.43702828506358, + "tilespanx": 19567.879240976275, + "tilespany": 19567.879240976275, + }, + { + "zoom": 12, + "bounds": [-20037508.342787, -30242157.366901536, 20037508.34273241, 20037508.342787], + "scaledenominator": 136494.69336618498, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 38.21851414253179, + "tilespanx": 9783.939620488138, + "tilespany": 9783.939620488138, + }, + { + "zoom": 13, + "bounds": [-20037508.342787, -30242157.366901536, 20037508.34273241, 20037508.342787], + "scaledenominator": 68247.34668309249, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 19.109257071265894, + "tilespanx": 4891.969810244069, + "tilespany": 4891.969810244069, + }, + { + "zoom": 14, + "bounds": [-20037508.342787, -30242157.366901536, 20037508.34273241, 20037508.342787], + "scaledenominator": 34123.673341546244, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 9.554628535632947, + "tilespanx": 2445.9849051220344, + "tilespany": 2445.9849051220344, + }, + { + "zoom": 15, + "bounds": [-20037508.342787, -30242157.3682939, 20037508.343842182, 20037508.342787], + "scaledenominator": 17061.836671245605, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 4.777314267948769, + "tilespanx": 1222.9924525948848, + "tilespany": 1222.9924525948848, + }, + { + "zoom": 16, + "bounds": [-20037508.342787, -30241545.8720675, 20037508.3438421, 20037508.342787], + "scaledenominator": 8530.918335622784, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 2.3886571339743794, + "tilespanx": 611.4962262974411, + "tilespany": 611.4962262974411, + }, + { + "zoom": 17, + "bounds": [-20037508.342787, -30241240.11838522, 20037508.339403186, 20037508.342787], + "scaledenominator": 4265.459167338928, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 1.1943285668548997, + "tilespanx": 305.74811311485433, + "tilespany": 305.74811311485433, + }, + { + "zoom": 18, + "bounds": [-20037508.342787, -30241087.25546706, 20037508.34828115, 20037508.342787], + "scaledenominator": 2132.7295841419354, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 0.5971642835597418, + "tilespanx": 152.8740565912939, + "tilespany": 152.8740565912939, + }, + { + "zoom": 19, + "bounds": [-20037508.342787, -30241010.79616208, 20037508.330525283, 20037508.342787], + "scaledenominator": 1066.364791598498, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 0.2985821416475794, + "tilespanx": 76.43702826178033, + "tilespany": 76.43702826178033, + }, + { + "zoom": 20, + "bounds": [-20037508.342787, -30240972.57764789, 20037508.330525238, 20037508.342787], + "scaledenominator": 533.1823957992484, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 0.14929107082378953, + "tilespanx": 38.21851413089012, + "tilespany": 38.21851413089012, + }, + { + "zoom": 21, + "bounds": [-20037508.342787, -30240972.57764789, 20037508.330525238, 20037508.342787], + "scaledenominator": 266.5911978996242, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 0.07464553541189477, + "tilespanx": 19.10925706544506, + "tilespany": 19.10925706544506, + }, + { + "zoom": 22, + "bounds": [-20037508.342787, -30240972.57764789, 20037508.330525238, 20037508.342787], + "scaledenominator": 133.2955989498121, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 0.037322767705947384, + "tilespanx": 9.55462853272253, + "tilespany": 9.55462853272253, + }, + { + "zoom": 23, + "bounds": [-20037508.342787, -30240972.57764789, 20037508.330525238, 20037508.342787], + "scaledenominator": 66.64779947490605, + "tilewidth": 256, + "tileheight": 256, + "pixelspan": 0.018661383852973692, + "tilespanx": 4.777314266361265, + "tilespany": 4.777314266361265, + }, +] + +THUMBNAIL_BACKGROUND = { + "class": "geonode.thumbs.background.GenericWMTSBackground", + "options": { + "url": "myserver.com/WMTS", + "layer": "Hosted_basemap_inforac_3857", + "style": "default", + "tilematrixset": "default028mm", + "minscaledenominator": 272989.38673236995, + }, +} + +EXPECTED_RESULTS_DIR = "geonode/thumbs/tests/expected_results/" + +base_request_url = "https://myserver.com/WMTS?&Service=WMTS&Request=GetTile&Version=1.0.0&Format=image/png&layer=Hosted_basemap_inforac_3857&style=default&tilematrixset=default028mm&" +mocked_requests = { + base_request_url + "TileMatrix=4&TileRow=5&TileCol=7": f"{EXPECTED_RESULTS_DIR}/tiles/wmts_7_5_4.png", + base_request_url + "TileMatrix=4&TileRow=6&TileCol=7": f"{EXPECTED_RESULTS_DIR}/tiles/wmts_7_6_4.png", + base_request_url + "TileMatrix=4&TileRow=5&TileCol=8": f"{EXPECTED_RESULTS_DIR}/tiles/wmts_8_5_4.png", + base_request_url + "TileMatrix=4&TileRow=6&TileCol=8": f"{EXPECTED_RESULTS_DIR}/tiles/wmts_8_6_4.png", + base_request_url + "TileMatrix=4&TileRow=5&TileCol=9": f"{EXPECTED_RESULTS_DIR}/tiles/wmts_9_5_4.png", + base_request_url + "TileMatrix=4&TileRow=6&TileCol=9": f"{EXPECTED_RESULTS_DIR}/tiles/wmts_9_6_4.png", +} + + +class Response: + def __init__(self, status_code=200, content=None) -> None: + self.status_code = status_code + self.content = content + + +def get_mock(*args): + file_path = mocked_requests.get(args[0]) + if file_path and os.path.exists(file_path): + with open(file_path, "rb") as fin: + return Response(200, fin.read()) + + +class GeoNodeThumbnailWMTSBackground(GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + + @override_settings(THUMBNAIL_BACKGROUND=THUMBNAIL_BACKGROUND) + @patch("geonode.thumbs.background.WMTS_TILEMATRIXSET_LEVELS", WMTS_TILEMATRIX_LEVELS) + def test_get_target_pixelspan(self, *args): + bbox = [-757689.8225283397, 4231175.960993547, 3557041.3914652625, 5957068.446590988] + expected_target_pixelspan = 8629.462427987204 + background = GenericWMTSBackground(thumbnail_width=500, thumbnail_height=200) + target_pixelspan = background.get_target_pixelspan(bbox) + self.assertAlmostEqual(expected_target_pixelspan, target_pixelspan) + + @override_settings(THUMBNAIL_BACKGROUND=THUMBNAIL_BACKGROUND) + @patch("geonode.thumbs.background.WMTS_TILEMATRIXSET_LEVELS", WMTS_TILEMATRIX_LEVELS) + def test_get_level_for_targetpixelspan(self, *args): + target_pixelspan = 8629.462427987204 + background = GenericWMTSBackground(thumbnail_width=500, thumbnail_height=200) + level = background.get_level_for_targetpixelspan(target_pixelspan) + self.assertDictEqual(level, WMTS_TILEMATRIX_LEVELS[4]) + + @override_settings(THUMBNAIL_BACKGROUND=THUMBNAIL_BACKGROUND) + @patch("geonode.thumbs.background.WMTS_TILEMATRIXSET_LEVELS", WMTS_TILEMATRIX_LEVELS) + def test_get_tiles_coords(self, *args): + bbox = [-757689.8225283397, 4231175.960993547, 3557041.3914652625, 5957068.446590988] + level = WMTS_TILEMATRIX_LEVELS[4] + expected_tile_rowcols = [[7, 5], [7, 6], [8, 5], [8, 6], [9, 5], [9, 6]] + background = GenericWMTSBackground(thumbnail_width=500, thumbnail_height=200) + tile_rowcols = background.get_tiles_coords(level, bbox) + self.assertListEqual(expected_tile_rowcols, tile_rowcols) + + @override_settings(THUMBNAIL_BACKGROUND=THUMBNAIL_BACKGROUND) + @patch("geonode.thumbs.background.WMTS_TILEMATRIXSET_LEVELS", WMTS_TILEMATRIX_LEVELS) + def test_build_request(self, *args): + expected_imgurl = "https://myserver.com/WMTS?&Service=WMTS&Request=GetTile&Version=1.0.0&Format=image/png& \ + layer=Hosted_basemap_inforac_3857&style=default&tilematrixset=default028mm&TileMatrix=4&TileRow=5&TileCol=7" + background = GenericWMTSBackground(thumbnail_width=500, thumbnail_height=200) + imgurl = background.build_request((7, 5, 4)) + self.assertEqual(expected_imgurl, imgurl) + + @override_settings(THUMBNAIL_BACKGROUND=THUMBNAIL_BACKGROUND) + @patch("geonode.thumbs.background.WMTS_TILEMATRIXSET_LEVELS", WMTS_TILEMATRIX_LEVELS) + @patch("geonode.thumbs.background.requests.get", get_mock) + def test_tile_request(self, *args): + bbox = [-757689.8225283397, 3557041.3914652625, 4231175.960993547, 5957068.446590988, "EPSG:3857"] + background = GenericWMTSBackground(thumbnail_width=500, thumbnail_height=200) + image = background.fetch(bbox) + expected_image = Image.open(f"{EXPECTED_RESULTS_DIR}/tiles/background.png") + diff = Image.new("RGB", image.size) + + mismatch = pixelmatch(image, expected_image, diff) + if mismatch >= expected_image.size[0] * expected_image.size[1] * 0.01: + logger.warn("Mismatch, it was not possible to bump the bg!") + else: + self.assertTrue( + mismatch < expected_image.size[0] * expected_image.size[1] * 0.01, + "Expected test and pre-generated backgrounds to differ up to 1%", + ) diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py index 8f84fec6a92..81cd4a3ea3f 100644 --- a/geonode/upload/api/tests.py +++ b/geonode/upload/api/tests.py @@ -17,7 +17,9 @@ # ######################################################################### +from geonode.base.models import ResourceBase from geonode.resource.models import ExecutionRequest +from geonode.geoserver.helpers import gs_catalog import os import shutil import logging @@ -233,35 +235,67 @@ def test_rest_uploads(self): """ Ensure we can access the Local Server Uploads list. """ - # Try to upload a good raster file and check the session IDs - fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") - resp, data = rest_upload_by_path(fname, self.client) - self.assertEqual(resp.status_code, 201) - - url = reverse("uploads-list") - # Anonymous - self.client.logout() - response = self.client.get(url, format="json") - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 5) - self.assertEqual(response.data["total"], 0) - # Pagination - self.assertEqual(len(response.data["uploads"]), 0) - logger.debug(response.data) + resp = None + layer_name = "relief_san_andres" + try: + self._cleanup_layer(layer_name=layer_name) + # Try to upload a good raster file and check the session IDs + fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") + resp, data = rest_upload_by_path(fname, self.client) + self.assertEqual(resp.status_code, 201) + + url = reverse("uploads-list") + # Anonymous + self.client.logout() + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 5) + self.assertEqual(response.data["total"], 0) + # Pagination + self.assertEqual(len(response.data["uploads"]), 0) + logger.debug(response.data) + except Exception: + if resp.json().get("errors"): + layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + finally: + self._cleanup_layer(layer_name) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_rest_uploads_non_interactive(self): """ Ensure we can access the Local Server Uploads list. """ - # Try to upload a good raster file and check the session IDs - fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") - resp, data = rest_upload_by_path(fname, self.client, non_interactive=True) - self.assertEqual(resp.status_code, 201) - - exec_id = data.get("execution_id", None) - _exec = ExecutionRequest.objects.get(exec_id=exec_id) - self.assertEqual(_exec.status, "finished") + resp = None + layer_name = "relief_san_andres" + try: + self._cleanup_layer(layer_name=layer_name) + # Try to upload a good raster file and check the session IDs + fname = os.path.join(GOOD_DATA, "raster", "relief_san_andres.tif") + resp, data = rest_upload_by_path(fname, self.client, non_interactive=True) + self.assertEqual(resp.status_code, 201) + exec_id = data.get("execution_id", None) + _exec = ExecutionRequest.objects.get(exec_id=exec_id) + self.assertEqual(_exec.status, "finished") + except Exception: + if resp.json().get("errors"): + layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + finally: + self._cleanup_layer(layer_name) + + def _cleanup_layer(self, layer_name): + # removing the layer from geonode + x = ResourceBase.objects.filter(alternate__icontains=layer_name) + if x.exists(): + for el in x.iterator(): + el.delete() + # removing the layer from geoserver + dataset = gs_catalog.get_layer(layer_name) + if dataset: + gs_catalog.delete(dataset, purge="all", recurse=True) + # removing the layer from geoserver + store = gs_catalog.get_store(layer_name, workspace="geonode") + if store: + gs_catalog.delete(store, purge="all", recurse=True) @mock.patch("geonode.upload.uploadhandler.SimpleUploadedFile") def test_rest_uploads_with_size_limit(self, mocked_uploaded_file): diff --git a/requirements.txt b/requirements.txt index cca54c41c83..39864fb3294 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # native dependencies Pillow==10.0.1 lxml==4.9.3 -psycopg2==2.9.7 -Django==3.2.21 +psycopg2==2.9.9 +Django==3.2.22 # Other amqp==5.1.1 @@ -10,7 +10,7 @@ beautifulsoup4==4.12.2 httplib2<0.22.1 hyperlink==21.0.0 idna>=2.5,<3.5 -urllib3==1.26.15 +urllib3==1.26.17 Paver==1.3.4 python-slugify==8.0.1 decorator==5.1.1 @@ -36,7 +36,7 @@ django-filter==23.3 django-imagekit==4.1.0 django-taggit==1.5.1 django-markdownify==0.9.3 -django-mptt==0.14.0 +django-mptt==0.15.0 django-modeltranslation>=0.11,<0.19.0 django-treebeard==4.7 django-guardian<2.4.1 @@ -109,11 +109,11 @@ elasticsearch>=2.0.0,<9.0.0 django-bootstrap3-datetimepicker-2==2.8.3 # storage manager dependencies -django-storages==1.14 +django-storages==1.14.1 dropbox==11.36.2 google-cloud-storage==2.11.0 google-cloud-core==2.3.3 -boto3==1.28.53 +boto3==1.28.56 # Django Caches python-memcached<=1.59 @@ -145,7 +145,7 @@ pycountry # production uWSGI==2.0.22 gunicorn==21.2.0 -ipython==8.15.0 +ipython==8.16.1 docker==6.1.3 invoke==2.2.0 @@ -170,7 +170,7 @@ selenium-requests==2.0.3 webdriver_manager==4.0.1 # Security and audit -mistune==3.0.1 +mistune==3.0.2 protobuf==3.20.3 mako==1.2.4 paramiko==3.3.1 # not directly required, fixes Blowfish deprecation warning diff --git a/setup.cfg b/setup.cfg index c80c1ab9c53..3c2147edf46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,8 +27,8 @@ install_requires = # native dependencies Pillow==10.0.1 lxml==4.9.3 - psycopg2==2.9.7 - Django==3.2.21 + psycopg2==2.9.9 + Django==3.2.22 # Other amqp==5.1.1 @@ -36,7 +36,7 @@ install_requires = httplib2<0.22.1 hyperlink==21.0.0 idna>=2.5,<3.5 - urllib3==1.26.15 + urllib3==1.26.17 Paver==1.3.4 python-slugify==8.0.1 decorator==5.1.1 @@ -62,7 +62,7 @@ install_requires = django-imagekit==4.1.0 django-taggit==1.5.1 django-markdownify==0.9.3 - django-mptt==0.14.0 + django-mptt==0.15.0 django-modeltranslation>=0.11,<0.19.0 django-treebeard==4.7 django-guardian<2.4.1 @@ -134,11 +134,11 @@ install_requires = django-bootstrap3-datetimepicker-2==2.8.3 # storage manager dependencies - django-storages==1.14 + django-storages==1.14.1 dropbox==11.36.2 google-cloud-storage==2.11.0 google-cloud-core==2.3.3 - boto3==1.28.53 + boto3==1.28.56 # Django Caches python-memcached<=1.59 @@ -170,7 +170,7 @@ install_requires = # production uWSGI==2.0.22 gunicorn==21.2.0 - ipython==8.15.0 + ipython==8.16.1 docker==6.1.3 invoke==2.2.0 @@ -195,7 +195,7 @@ install_requires = webdriver_manager==4.0.1 # Security and audit - mistune==3.0.1 + mistune==3.0.2 protobuf==3.20.3 mako==1.2.4 paramiko==3.3.1 # not directly required, fixes Blowfish deprecation warning