diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d37a2b..7a1a5c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: rev: v0.3.7 hooks: - id: ruff - args: ["--fix", "--show-fixes"] + args: ["--fix", "--show-fixes", "--select", "I"] - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.4.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index c25eb74..0c1720d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v1.0.2 +### 2024-04-05 + +This version of HyBIG correctly handles missing/bad input data marked by _FillValue or NoData. +Anytime a bad value occurs in the input raster, the output png image will set to transparent. + ## v1.0.1 ### 2024-04-05 diff --git a/docker/service_version.txt b/docker/service_version.txt index 7dea76e..6d7de6e 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.0.1 +1.0.2 diff --git a/harmony_browse_image_generator/browse.py b/harmony_browse_image_generator/browse.py index 1068673..39dedcd 100644 --- a/harmony_browse_image_generator/browse.py +++ b/harmony_browse_image_generator/browse.py @@ -13,7 +13,7 @@ from harmony.message import Source as HarmonySource from matplotlib.cm import ScalarMappable from matplotlib.colors import Normalize -from numpy import around, concatenate, ndarray +from numpy import ndarray from osgeo_utils.auxiliary.color_palette import ColorPalette from PIL import Image from rasterio.io import DatasetReader @@ -25,9 +25,12 @@ from harmony_browse_image_generator.color_utility import ( NODATA_IDX, NODATA_RGBA, + OPAQUE, + TRANSPARENT, TRANSPARENT_IDX, TRANSPARENT_RGBA, get_color_palette, + remove_alpha, ) from harmony_browse_image_generator.exceptions import HyBIGError from harmony_browse_image_generator.sizes import ( @@ -113,27 +116,39 @@ def create_browse_imagery( def convert_mulitband_to_raster(data_array: DataArray) -> ndarray: """Convert multiband to a raster image. - Reads the three/four bands from the file, then normalizes them to the range + Reads the three or four bands from the file, then normalizes them to the range 0 to 255. This assumes the input image is already in RGB or RGBA format and just ensures that the output is 8bit. """ + if data_array.rio.count not in [3, 4]: + raise HyBIGError( + f'Cannot create image from {data_array.rio.count} band image. ' + 'Expecting 3 or 4 bands.' + ) + bands = data_array.to_numpy() - norm = Normalize() - if data_array.rio.count == 3: - norm.autoscale(bands) - raster = around(norm(bands) * 255.0).astype('uint8') + # Create an alpha layer where input NaN values are transparent. + nan_mask = np.isnan(bands).any(axis=0) + nan_alpha = np.where(nan_mask, TRANSPARENT, OPAQUE) - elif data_array.rio.count == 4: - norm.autoscale(bands[0:3, :, :]) - partial_raster = around(norm(bands[0:3, :, :]) * 255.0).astype('uint8') - raster = concatenate([partial_raster.data, bands[3:4, :, :]]) + # grab any existing alpha layer + bands, image_alpha = remove_alpha(bands) + + norm = Normalize(vmin=np.nanmin(bands), vmax=np.nanmax(bands)) + raster = np.nan_to_num(np.around(norm(bands) * 255.0), copy=False, nan=0.0).astype( + 'uint8' + ) + if image_alpha is not None: + # merge nan alpha with the image alpha prefering transparency to + # opaqueness. + alpha = np.minimum(nan_alpha, image_alpha) else: - raise HyBIGError(f'Cannot create image from {data_array.rio.count} band image') + alpha = nan_alpha - return raster + return np.concatenate((raster, alpha[None, ...]), axis=0) def convert_singleband_to_raster( @@ -153,15 +168,16 @@ def convert_gray_1band_to_raster(data_array: DataArray) -> ndarray: """Convert a 1-band raster without a color association.""" band = data_array[0, :, :] cmap = matplotlib.colormaps['Greys_r'] - norm = Normalize() - norm.autoscale(band) + cmap.set_bad(TRANSPARENT_RGBA) + norm = Normalize(vmin=np.nanmin(band), vmax=np.nanmax(band)) scalar_map = ScalarMappable(cmap=cmap, norm=norm) rgba_image = np.zeros((*band.shape, 4), dtype='uint8') for row_no in range(band.shape[0]): rgba_image_slice = scalar_map.to_rgba(band[row_no, :], bytes=True) rgba_image[row_no, :, :] = rgba_image_slice - return reshape_as_raster(rgba_image[..., 0:3]) + + return reshape_as_raster(rgba_image) def convert_paletted_1band_to_raster( @@ -182,7 +198,7 @@ def convert_paletted_1band_to_raster( levels, scaled_colors, extend='max' ) - # handle no data values + # handle palette no data value if palette.ndv is not None: nodata_colors = palette.color_to_color_entry(palette.ndv, with_alpha=True) cmap.set_bad( @@ -222,13 +238,6 @@ def prepare_raster_for_writing( return palettize_raster(raster) -def remove_alpha(raster: ndarray) -> tuple[ndarray, ndarray | None]: - """Pull raster array off of input if it exists.""" - if raster.shape[0] == 4: - return raster[0:3, :, :], raster[3, :, :] - return raster, None - - def palettize_raster(raster: ndarray) -> tuple[ndarray, dict]: """convert an RGB or RGBA image into a 1band image and palette. @@ -262,10 +271,9 @@ def add_alpha( ) -> tuple[ndarray, dict]: """If the input data had alpha values, manually set the quantized_image index to the transparent index in those places.""" - max_alpha = 255 - if alpha is not None and np.any(alpha != max_alpha): + if alpha is not None and np.any(alpha != OPAQUE): # Set any alpha to the transparent index value - quantized_array = np.where(alpha != max_alpha, TRANSPARENT_IDX, quantized_array) + quantized_array = np.where(alpha != OPAQUE, TRANSPARENT_IDX, quantized_array) color_map[TRANSPARENT_IDX] = TRANSPARENT_RGBA return quantized_array, color_map diff --git a/harmony_browse_image_generator/color_utility.py b/harmony_browse_image_generator/color_utility.py index 8176288..750a836 100644 --- a/harmony_browse_image_generator/color_utility.py +++ b/harmony_browse_image_generator/color_utility.py @@ -5,6 +5,9 @@ """ +from typing import TYPE_CHECKING + +import numpy as np import requests from harmony.message import Source as HarmonySource from osgeo_utils.auxiliary.color_palette import ColorPalette @@ -18,6 +21,8 @@ # Constants for output PNG images # Applied to transparent pixels where alpha < 255 +TRANSPARENT = np.uint8(0) +OPAQUE = np.uint8(255) TRANSPARENT_RGBA = (0, 0, 0, 0) TRANSPARENT_IDX = 254 @@ -26,6 +31,13 @@ NODATA_IDX = 255 +def remove_alpha(raster: np.ndarray) -> tuple[np.ndarray, np.ndarray, None]: + """remove alpha layer when it exists.""" + if raster.shape[0] == 4: + return raster[0:3, :, :], raster[3, :, :] + return raster, None + + def palette_from_remote_colortable(url: str) -> ColorPalette: """Return a gdal ColorPalette from a remote colortable.""" response = requests.get(url, timeout=10) diff --git a/tests/unit/test_browse.py b/tests/unit/test_browse.py index 6f2b86c..f9bf619 100644 --- a/tests/unit/test_browse.py +++ b/tests/unit/test_browse.py @@ -33,6 +33,8 @@ validate_file_type, ) from harmony_browse_image_generator.color_utility import ( + OPAQUE, + TRANSPARENT, convert_colormap_to_palette, get_color_palette, palette_from_remote_colortable, @@ -298,39 +300,46 @@ def test_create_browse_imagery_with_mocks( ) def test_convert_singleband_to_raster_without_colortable(self): - ds = MagicMock(DataArray) - ds.__getitem__.return_value = self.data + """Tests convert_gray_1band_to_raster.""" + + return_data = np.copy(self.data).astype('float64') + return_data[0][1] = np.nan + ds = DataArray(return_data).expand_dims('band') expected_raster = np.array( [ [ - [0, 104, 198, 255], + [0, 0, 198, 255], [0, 104, 198, 255], [0, 104, 198, 255], [0, 104, 198, 255], ], [ - [0, 104, 198, 255], + [0, 0, 198, 255], [0, 104, 198, 255], [0, 104, 198, 255], [0, 104, 198, 255], ], [ + [0, 0, 198, 255], [0, 104, 198, 255], [0, 104, 198, 255], [0, 104, 198, 255], - [0, 104, 198, 255], + ], + [ + [255, 0, 255, 255], + [255, 255, 255, 255], + [255, 255, 255, 255], + [255, 255, 255, 255], ], ], dtype='uint8', ) - actual_raster = convert_singleband_to_raster(ds, None) assert_array_equal(expected_raster, actual_raster) def test_convert_singleband_to_raster_with_colormap(self): - ds = MagicMock(DataArray) - ds.__getitem__.return_value = self.data + ds = DataArray(self.data).expand_dims('band') expected_raster = np.array( [ @@ -367,13 +376,11 @@ def test_convert_singleband_to_raster_with_colormap(self): assert_array_equal(expected_raster, actual_raster) def test_convert_singleband_to_raster_with_colormap_and_bad_data(self): - ds = MagicMock(DataArray) data_array = np.array(self.data, dtype='float') data_array[0, 0] = np.nan + ds = DataArray(data_array).expand_dims('band') nv_color = (10, 20, 30, 40) - ds.__getitem__.return_value = data_array - # Read the image down: red, yellow, green, blue expected_raster = np.array( [ @@ -412,9 +419,14 @@ def test_convert_singleband_to_raster_with_colormap_and_bad_data(self): assert_array_equal(expected_raster, actual_raster) def test_convert_3_multiband_to_raster(self): - ds = Mock(DataArray) - ds.rio.count = 3 - ds.to_numpy.return_value = np.stack([self.data, self.data, self.data]) + + bad_data = np.copy(self.data).astype('float64') + bad_data[1][1] = np.nan + bad_data[1][2] = np.nan + ds = DataArray( + np.stack([self.data, bad_data, self.data]), + dims=('band', 'y', 'x'), + ) expected_raster = np.array( [ @@ -426,7 +438,7 @@ def test_convert_3_multiband_to_raster(self): ], [ [0, 85, 170, 255], - [0, 85, 170, 255], + [0, 0, 0, 255], [0, 85, 170, 255], [0, 85, 170, 255], ], @@ -436,6 +448,12 @@ def test_convert_3_multiband_to_raster(self): [0, 85, 170, 255], [0, 85, 170, 255], ], + [ + [OPAQUE, OPAQUE, OPAQUE, OPAQUE], + [OPAQUE, TRANSPARENT, TRANSPARENT, OPAQUE], + [OPAQUE, OPAQUE, OPAQUE, OPAQUE], + [OPAQUE, OPAQUE, OPAQUE, OPAQUE], + ], ], dtype='uint8', ) @@ -444,16 +462,27 @@ def test_convert_3_multiband_to_raster(self): assert_array_equal(expected_raster, actual_raster.data) def test_convert_4_multiband_to_raster(self): + """Input data has NaN _fillValue match in the red layer at [1,1] + and alpha channel also exists with a single transparent value at [0,0] + + See that the expected output has transformed the missing data [nan] + into fully transparent at [1,1] and retained the transparent value of 1 + at [0,0] + + """ ds = Mock(DataArray) + bad_data = np.copy(self.data).astype('float64') + bad_data[1, 1] = np.nan + alpha = np.ones_like(self.data) * 255 alpha[0, 0] = 1 ds.rio.count = 4 - ds.to_numpy.return_value = np.stack([self.data, self.data, self.data, alpha]) + ds.to_numpy.return_value = np.stack([bad_data, self.data, self.data, alpha]) expected_raster = np.array( [ [ [0, 85, 170, 255], - [0, 85, 170, 255], + [0, 0, 170, 255], [0, 85, 170, 255], [0, 85, 170, 255], ], @@ -471,7 +500,7 @@ def test_convert_4_multiband_to_raster(self): ], [ [1, 255, 255, 255], - [255, 255, 255, 255], + [255, 0, 255, 255], [255, 255, 255, 255], [255, 255, 255, 255], ], @@ -493,7 +522,8 @@ def test_convert_5_multiband_to_raster(self): convert_mulitband_to_raster(ds) self.assertEqual( - excepted.exception.message, 'Cannot create image from 5 band image' + excepted.exception.message, + 'Cannot create image from 5 band image. Expecting 3 or 4 bands.', ) def test_prepare_raster_for_writing_jpeg_3band(self): diff --git a/tests/utilities.py b/tests/utilities.py index 2ccb98a..0c81a58 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -6,7 +6,6 @@ from harmony.util import bbox_to_geometry from pystac import Asset, Catalog, Item - Granule = namedtuple('Granule', ['url', 'media_type', 'roles'])