From c374797fb094bc832dae185bbdfba10c377aeb8b Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Wed, 10 Apr 2024 14:00:34 -0400 Subject: [PATCH 1/4] IP-241 - Implement pre-commit. --- .pre-commit-config.yaml | 20 +++++++++++++++ README.md | 33 +++++++++++++++++++++++++ docker/tests.Dockerfile | 2 +- harmony_regridding_service/utilities.py | 2 +- tests/pip_test_requirements.txt | 1 + tests/test_code_format.py | 4 ++- tests/unit/test_adapter.py | 2 +- tests/unit/test_regridding_service.py | 4 +-- 8 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..147014e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.4 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] +# - repo: https://github.com/psf/black-pre-commit-mirror +# rev: 24.3.0 +# hooks: +# - id: black-jupyter +# args: ["--skip-string-normalization"] +# language_version: python3.11 diff --git a/README.md b/README.md index 7daec9d..7e5a230 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,39 @@ CI/CD pipeline. In future, this project will be migrated from Bitbucket to GitHub, at which point the CI/CD will be migrated to workflows that use GitHub Actions. +## pre-commit hooks: + +This repository uses [pre-commit](https://pre-commit.com/) to enable pre-commit +checking the repository for some coding standard best practices. These include: + +* Removing trailing whitespaces. +* Removing blank lines at the end of a file. +* JSON files have valid formats. +* [ruff](https://github.com/astral-sh/ruff) Python linting checks. +* [black](https://black.readthedocs.io/en/stable/index.html) Python code + formatting checks. + +To enable these checks: + +```bash +# Install pre-commit Python package as part of test requirements: +pip install -r tests/pip_test_requirements.txt + +# Install the git hook scripts: +pre-commit install + +# (Optional) Run against all files: +pre-commit run --all-files +``` + +When you try to make a new commit locally, `pre-commit` will automatically run. +If any of the hooks detect non-compliance (e.g., trailing whitespace), that +hook will state it failed, and also try to fix the issue. You will need to +review and `git add` the changes before you can make a commit. + +It is planned to implement additional hooks, possibly including tools such as +`mypy`. + ## Versioning: Service Docker images for the Harmony Regridding Service adhere to semantic diff --git a/docker/tests.Dockerfile b/docker/tests.Dockerfile index bd1266a..a49d1f5 100644 --- a/docker/tests.Dockerfile +++ b/docker/tests.Dockerfile @@ -16,7 +16,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 COPY tests/pip_test_requirements.txt . RUN pip install --no-input -r pip_test_requirements.txt -# Copy test directory containing Python unittest suite, test data and utilities +# Copy test directory containing Python unittest suite, test data and utilities COPY ./tests tests # Configure a container to be executable via the `docker run` command. diff --git a/harmony_regridding_service/utilities.py b/harmony_regridding_service/utilities.py index 82aa874..c26f0a7 100644 --- a/harmony_regridding_service/utilities.py +++ b/harmony_regridding_service/utilities.py @@ -4,7 +4,7 @@ """ from mimetypes import guess_type as guess_mime_type from os.path import splitext -from typing import Any, Optional +from typing import Optional from harmony.message import Message from harmony.message_utility import rgetattr diff --git a/tests/pip_test_requirements.txt b/tests/pip_test_requirements.txt index 0cf95be..4b2bb55 100644 --- a/tests/pip_test_requirements.txt +++ b/tests/pip_test_requirements.txt @@ -1,4 +1,5 @@ coverage~=7.2.2 +pre-commit~=3.7.0 pycodestyle~=2.10.0 pylint~=2.17.2 unittest-xml-reporting~=3.2.0 diff --git a/tests/test_code_format.py b/tests/test_code_format.py index e1e3b4c..0b4ba6d 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -15,6 +15,8 @@ class TestCodeFormat(TestCase): * W503: Break before binary operator. Have to ignore one of W503 or W504 to allow for breaking of some long lines. PEP8 suggests breaking the line before a binary operatore is more "Pythonic". + * E203, E701: This repository uses black code formatting, which deviates + from PEP8 for these errors. """ @classmethod def setUpClass(cls): @@ -25,6 +27,6 @@ def test_pycodestyle_adherence(self): defined standard. """ - style_guide = StyleGuide(ignore=['E501', 'W503']) + style_guide = StyleGuide(ignore=['E501', 'W503', 'E203', 'E701']) results = style_guide.check_files(self.python_files) self.assertEqual(results.total_errors, 0, 'Found code style issues.') diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index ca6256e..1b5dcc3 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -1,4 +1,4 @@ -from unittest import skip, TestCase +from unittest import TestCase from harmony.message import Message from harmony.util import config, HarmonyException diff --git a/tests/unit/test_regridding_service.py b/tests/unit/test_regridding_service.py index 88ace23..aa288cc 100644 --- a/tests/unit/test_regridding_service.py +++ b/tests/unit/test_regridding_service.py @@ -256,7 +256,7 @@ def test_copy_var_with_attrs(self): target_file = self.test_file() target_area = self.test_area() var_info = self.var_info(self.test_1D_dimensions_ncfile) - expected_metadata = {'units', 'widgets per month'} + expected_metadata = {'units': 'widgets per month'} with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ Dataset(target_file, mode='w') as target_ds: @@ -268,7 +268,7 @@ def test_copy_var_with_attrs(self): attr: validate['/data'].getncattr(attr) for attr in validate['/data'].ncattrs() } - self.assertDictEqual(actual_metadata, actual_metadata) + self.assertDictEqual(actual_metadata, expected_metadata) def test_copy_dimension_variables(self): target_file = self.test_file() From 2967ed10ef6e38d4346735c1693212a56c756902 Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Wed, 10 Apr 2024 14:08:17 -0400 Subject: [PATCH 2/4] IP-241 - Implement black code formatting across repository. --- .pre-commit-config.yaml | 12 +- harmony_regridding_service/__main__.py | 10 +- harmony_regridding_service/adapter.py | 133 +++-- harmony_regridding_service/exceptions.py | 23 +- .../regridding_service.py | 382 ++++++++----- harmony_regridding_service/utilities.py | 46 +- ...egridding_Service_User_Documentation.ipynb | 45 +- tests/test_adapter.py | 329 ++++++----- tests/test_code_format.py | 29 +- tests/unit/test_adapter.py | 198 ++++--- tests/unit/test_regridding_service.py | 521 ++++++++++-------- tests/unit/test_utilities.py | 55 +- tests/utilities.py | 30 +- 13 files changed, 1040 insertions(+), 773 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 147014e..c59f584 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,9 +12,9 @@ repos: hooks: - id: ruff args: ["--fix", "--show-fixes"] -# - repo: https://github.com/psf/black-pre-commit-mirror -# rev: 24.3.0 -# hooks: -# - id: black-jupyter -# args: ["--skip-string-normalization"] -# language_version: python3.11 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.3.0 + hooks: + - id: black-jupyter + args: ["--skip-string-normalization"] + language_version: python3.11 diff --git a/harmony_regridding_service/__main__.py b/harmony_regridding_service/__main__.py index ad93fd5..958ffb2 100644 --- a/harmony_regridding_service/__main__.py +++ b/harmony_regridding_service/__main__.py @@ -1,4 +1,5 @@ """ Run the Harmony Regridding Service Adapter via the Harmony CLI. """ + from argparse import ArgumentParser from sys import argv from typing import List @@ -9,12 +10,13 @@ def main(arguments: List[str]): - """ Parse command line arguments and invoke the appropriate method to - respond to them + """Parse command line arguments and invoke the appropriate method to + respond to them """ - parser = ArgumentParser(prog='harmony-regridding-service', - description='Run Harmony regridding service.') + parser = ArgumentParser( + prog='harmony-regridding-service', description='Run Harmony regridding service.' + ) setup_cli(parser) harmony_arguments, _ = parser.parse_known_args(arguments[1:]) diff --git a/harmony_regridding_service/adapter.py b/harmony_regridding_service/adapter.py index f12975f..6307e3c 100644 --- a/harmony_regridding_service/adapter.py +++ b/harmony_regridding_service/adapter.py @@ -5,6 +5,7 @@ another grid as specified in the input Harmony message. """ + from os.path import basename from shutil import rmtree from tempfile import mkdtemp @@ -13,49 +14,53 @@ from harmony import BaseHarmonyAdapter from harmony.message import Source as HarmonySource from harmony.message_utility import has_self_consistent_grid -from harmony.util import (bbox_to_geometry, download, generate_output_filename, - stage) +from harmony.util import bbox_to_geometry, download, generate_output_filename, stage from pystac import Asset, Catalog, Item -from harmony_regridding_service.exceptions import (InvalidInterpolationMethod, - InvalidTargetCRS, - InvalidTargetGrid) +from harmony_regridding_service.exceptions import ( + InvalidInterpolationMethod, + InvalidTargetCRS, + InvalidTargetGrid, +) from harmony_regridding_service.regridding_service import regrid -from harmony_regridding_service.utilities import (get_file_mime_type, - has_valid_crs, - has_valid_interpolation) +from harmony_regridding_service.utilities import ( + get_file_mime_type, + has_valid_crs, + has_valid_interpolation, +) class RegriddingServiceAdapter(BaseHarmonyAdapter): - """ This class extends the BaseHarmonyAdapter class from the - harmony-service-lib package to implement regridding operations. + """This class extends the BaseHarmonyAdapter class from the + harmony-service-lib package to implement regridding operations. """ + def __init__(self, message, catalog=None, config=None): super().__init__(message, catalog=catalog, config=config) self.cache = {'grids': {}} def invoke(self) -> Catalog: - """ Adds validation to process_item based invocations. """ + """Adds validation to process_item based invocations.""" self.validate_message() return super().invoke() def validate_message(self): - """ Validates that the contents of the Harmony message provides all - necessary parameters. - - For an input Harmony message to be considered valid it must: - - * Contain a valid target grid, with `format.scaleExtent` and either - `format.scaleSize` or both `format.height` and `format.width` - fully populated. - * Not specify an incompatible target CRS. Initially, the target CRS - is limited to geographic. The message should either specify a - geographic CRS, or not specify one at all. - * Not specify an incompatible interpolation method. Initially, the - Harmony Regridding Service will use Elliptical Weighted Averaging - to interpolate when needed. The message should either specify - this interpolation method, or not specify one at all. + """Validates that the contents of the Harmony message provides all + necessary parameters. + + For an input Harmony message to be considered valid it must: + + * Contain a valid target grid, with `format.scaleExtent` and either + `format.scaleSize` or both `format.height` and `format.width` + fully populated. + * Not specify an incompatible target CRS. Initially, the target CRS + is limited to geographic. The message should either specify a + geographic CRS, or not specify one at all. + * Not specify an incompatible interpolation method. Initially, the + Harmony Regridding Service will use Elliptical Weighted Averaging + to interpolate when needed. The message should either specify + this interpolation method, or not specify one at all. """ if not has_valid_crs(self.message): @@ -68,54 +73,72 @@ def validate_message(self): raise InvalidTargetGrid() def process_item(self, item: Item, source: HarmonySource) -> Item: - """ Processes a single input STAC item. """ + """Processes a single input STAC item.""" try: working_directory = mkdtemp() results = item.clone() results.assets = {} - asset = next((item_asset for item_asset in item.assets.values() - if 'data' in (item_asset.roles or []))) + asset = next( + ( + item_asset + for item_asset in item.assets.values() + if 'data' in (item_asset.roles or []) + ) + ) # Download the input: - input_filepath = download(asset.href, working_directory, - logger=self.logger, cfg=self.config, - access_token=self.message.accessToken) + input_filepath = download( + asset.href, + working_directory, + logger=self.logger, + cfg=self.config, + access_token=self.message.accessToken, + ) transformed_file_name = regrid(self, input_filepath, source) # Stage the transformed output: transformed_mime_type = get_file_mime_type(transformed_file_name) - staged_url = self.stage_output(transformed_file_name, asset.href, - transformed_mime_type) + staged_url = self.stage_output( + transformed_file_name, asset.href, transformed_mime_type + ) - return self.create_output_stac_item(item, staged_url, - transformed_mime_type) + return self.create_output_stac_item(item, staged_url, transformed_mime_type) except Exception as exception: self.logger.exception(exception) raise exception finally: rmtree(working_directory) - def stage_output(self, transformed_file: str, input_file: str, - transformed_mime_type: Optional[str]) -> str: - """ Generate an output file name based on the input asset URL and the - operations performed to produce the output. Use this name to stage - the output in the S3 location specified in the input Harmony - message. + def stage_output( + self, + transformed_file: str, + input_file: str, + transformed_mime_type: Optional[str], + ) -> str: + """Generate an output file name based on the input asset URL and the + operations performed to produce the output. Use this name to stage + the output in the S3 location specified in the input Harmony + message. """ - output_file_name = generate_output_filename(input_file, - is_regridded=True) - - return stage(transformed_file, output_file_name, transformed_mime_type, - location=self.message.stagingLocation, - logger=self.logger, cfg=self.config) + output_file_name = generate_output_filename(input_file, is_regridded=True) + + return stage( + transformed_file, + output_file_name, + transformed_mime_type, + location=self.message.stagingLocation, + logger=self.logger, + cfg=self.config, + ) - def create_output_stac_item(self, input_stac_item: Item, staged_url: str, - transformed_mime_type: str) -> Item: - """ Create an output STAC item used to access the transformed and - staged output in S3. + def create_output_stac_item( + self, input_stac_item: Item, staged_url: str, transformed_mime_type: str + ) -> Item: + """Create an output STAC item used to access the transformed and + staged output in S3. """ output_stac_item = input_stac_item.clone() @@ -127,8 +150,10 @@ def create_output_stac_item(self, input_stac_item: Item, staged_url: str, output_stac_item.geometry = bbox_to_geometry(output_stac_item.bbox) output_stac_item.assets['data'] = Asset( - staged_url, title=basename(staged_url), - media_type=transformed_mime_type, roles=['data'] + staged_url, + title=basename(staged_url), + media_type=transformed_mime_type, + roles=['data'], ) return output_stac_item diff --git a/harmony_regridding_service/exceptions.py b/harmony_regridding_service/exceptions.py index edbdfc9..618171f 100644 --- a/harmony_regridding_service/exceptions.py +++ b/harmony_regridding_service/exceptions.py @@ -3,38 +3,45 @@ application. """ + from harmony.util import HarmonyException class RegridderException(HarmonyException): - """ Base service exception. """ + """Base service exception.""" + def __init__(self, message=None): super().__init__(message, 'sds/harmony-regridder') class InvalidTargetCRS(RegridderException): - """ Raised when a request specifies an unsupported target Coordinate - Reference System. + """Raised when a request specifies an unsupported target Coordinate + Reference System. """ + def __init__(self, target_crs: str): super().__init__(f'Target CRS not supported: "{target_crs}"') class InvalidInterpolationMethod(RegridderException): - """ Raised when a user specifies an unsupported interpolation method. """ + """Raised when a user specifies an unsupported interpolation method.""" + def __init__(self, interpolation_method: str): - super().__init__('Interpolation method not supported: ' - f'"{interpolation_method}"') + super().__init__( + 'Interpolation method not supported: ' f'"{interpolation_method}"' + ) class InvalidTargetGrid(RegridderException): - """ Raised when a request specifies an incomplete or invalid grid. """ + """Raised when a request specifies an incomplete or invalid grid.""" + def __init__(self): super().__init__('Insufficient or invalid target grid parameters.') class InvalidSourceDimensions(RegridderException): - """ Raised when a source granule does not meet the expected dimension shapes. """ + """Raised when a source granule does not meet the expected dimension shapes.""" + def __init__(self, message: str): super().__init__(message) diff --git a/harmony_regridding_service/regridding_service.py b/harmony_regridding_service/regridding_service.py index 65db53b..efa1fc7 100644 --- a/harmony_regridding_service/regridding_service.py +++ b/harmony_regridding_service/regridding_service.py @@ -12,6 +12,7 @@ to look at something like check_coor_valid in swot repo. """ + from __future__ import annotations from pathlib import Path, PurePath @@ -26,52 +27,61 @@ from pyresample.geometry import AreaDefinition, SwathDefinition from varinfo import VarInfoFromNetCDF4 -from harmony_regridding_service.exceptions import (InvalidSourceDimensions, - RegridderException) +from harmony_regridding_service.exceptions import ( + InvalidSourceDimensions, + RegridderException, +) if TYPE_CHECKING: from harmony_regridding_service.adapter import RegriddingServiceAdapter HRS_VARINFO_CONFIG_FILENAME = str( - Path(Path(__file__).parent, 'config', 'HRS_varinfo_config.json')) + Path(Path(__file__).parent, 'config', 'HRS_varinfo_config.json') +) -def regrid(adapter: RegriddingServiceAdapter, input_filepath: str, - source: Source) -> str: +def regrid( + adapter: RegriddingServiceAdapter, input_filepath: str, source: Source +) -> str: """Regrid the input data at input_filepath.""" - var_info = VarInfoFromNetCDF4(input_filepath, - short_name=source.shortName, - config_file=HRS_VARINFO_CONFIG_FILENAME) + var_info = VarInfoFromNetCDF4( + input_filepath, + short_name=source.shortName, + config_file=HRS_VARINFO_CONFIG_FILENAME, + ) target_area = _compute_target_area(adapter.message) resampler_cache = _cache_resamplers(input_filepath, var_info, target_area) adapter.logger.info(f'cached resamplers for {resampler_cache.keys()}') - target_filepath = generate_output_filename(input_filepath, - is_regridded=True) + target_filepath = generate_output_filename(input_filepath, is_regridded=True) - with Dataset(input_filepath, mode='r') as source_ds, \ - Dataset(target_filepath, mode='w', format='NETCDF4') as target_ds: + with Dataset(input_filepath, mode='r') as source_ds, Dataset( + target_filepath, mode='w', format='NETCDF4' + ) as target_ds: _transfer_metadata(source_ds, target_ds) _transfer_dimensions(source_ds, target_ds, target_area, var_info) - crs_map = _write_grid_mappings(target_ds, - _resampled_dimension_pairs(var_info), - target_area) + crs_map = _write_grid_mappings( + target_ds, _resampled_dimension_pairs(var_info), target_area + ) vars_to_process = var_info.get_all_variables() - cloned_vars = _clone_variables(source_ds, target_ds, - _unresampled_variables(var_info)) + cloned_vars = _clone_variables( + source_ds, target_ds, _unresampled_variables(var_info) + ) adapter.logger.info(f'cloned variables: {cloned_vars}') vars_to_process -= cloned_vars - dimension_vars = _copy_dimension_variables(source_ds, target_ds, - target_area, var_info) + dimension_vars = _copy_dimension_variables( + source_ds, target_ds, target_area, var_info + ) adapter.logger.info(f'processed dimension variables: {dimension_vars}') vars_to_process -= dimension_vars - resampled_vars = _resample_nD_variables(source_ds, target_ds, var_info, - resampler_cache, set(vars_to_process)) + resampled_vars = _resample_nD_variables( + source_ds, target_ds, var_info, resampler_cache, set(vars_to_process) + ) vars_to_process -= resampled_vars adapter.logger.info(f'resampled variables: {resampled_vars}') @@ -109,19 +119,23 @@ def _transfer_metadata(source_ds: Dataset, target_ds: Dataset) -> None: t_group.setncatts(group_metadata) -def _add_grid_mapping_metadata(target_ds: Dataset, variables: Set[str], - var_info: VarInfoFromNetCDF4, - crs_map: Dict) -> None: +def _add_grid_mapping_metadata( + target_ds: Dataset, variables: Set[str], var_info: VarInfoFromNetCDF4, crs_map: Dict +) -> None: """Link regridded variables to the correct crs variable.""" for var_name in variables: - crs_variable_name = crs_map[_horizontal_dims_for_variable( - var_info, var_name)] + crs_variable_name = crs_map[_horizontal_dims_for_variable(var_info, var_name)] var = _get_variable(target_ds, var_name) var.setncattr('grid_mapping', crs_variable_name) -def _resample_variable_data(s_var: np.ndarray, t_var: np.ndarray, resampler: DaskEWAResampler, - var_info: VarInfoFromNetCDF4, var_name: str) -> None: +def _resample_variable_data( + s_var: np.ndarray, + t_var: np.ndarray, + resampler: DaskEWAResampler, + var_info: VarInfoFromNetCDF4, + var_name: str, +) -> None: """Recursively resample variable data in N-dimensions. A recursive function that will reduce an N-dimensional variable to the base @@ -132,36 +146,47 @@ def _resample_variable_data(s_var: np.ndarray, t_var: np.ndarray, resampler: Das if len(s_var.shape) > 2: for layer_index in range(s_var.shape[0]): t_var[layer_index, ...] = _resample_variable_data( - s_var[layer_index, ...], t_var[layer_index, ...], - resampler, var_info, var_name + s_var[layer_index, ...], + t_var[layer_index, ...], + resampler, + var_info, + var_name, ) return t_var else: return _resample_layer(s_var[:], resampler, var_info, var_name) -def _resample_nD_variables(source_ds: Dataset, target_ds: Dataset, - var_info: VarInfoFromNetCDF4, resampler_cache: Dict, - variables: Set[str]) -> Set[str]: +def _resample_nD_variables( + source_ds: Dataset, + target_ds: Dataset, + var_info: VarInfoFromNetCDF4, + resampler_cache: Dict, + variables: Set[str], +) -> Set[str]: """Function to resample any projected variable.""" for var_name in variables: - resampler = resampler_cache[ - _horizontal_dims_for_variable(var_info, var_name) - ] + resampler = resampler_cache[_horizontal_dims_for_variable(var_info, var_name)] (s_var, t_var) = _copy_var_with_attrs(source_ds, target_ds, var_name) - t_var[:] = _resample_variable_data(s_var[:], t_var[:], resampler, var_info, var_name) + t_var[:] = _resample_variable_data( + s_var[:], t_var[:], resampler, var_info, var_name + ) return variables -def _resample_layer(source_plane: np.ma.array, resampler: DaskEWAResampler, - var_info: VarInfoFromNetCDF4, var_name: str) -> np.ma.array: +def _resample_layer( + source_plane: np.ma.array, + resampler: DaskEWAResampler, + var_info: VarInfoFromNetCDF4, + var_name: str, +) -> np.ma.array: """Prepare the input layer, resample and return the results.""" prepped_source = _prepare_data_plane(source_plane, var_info, var_name) - target_data = resampler.compute(prepped_source, - **_resampler_kwargs(prepped_source)) - return _prepare_data_plane(target_data, var_info, - var_name).astype(source_plane.dtype) + target_data = resampler.compute(prepped_source, **_resampler_kwargs(prepped_source)) + return _prepare_data_plane(target_data, var_info, var_name).astype( + source_plane.dtype + ) def _integer_like(test_type: np.dtype) -> bool: @@ -173,12 +198,16 @@ def _best_cast(integer_type: np.dtype) -> np.dtype: """Return smallest float type to cast an integer type to.""" float_types = [np.float16, np.float32, np.float64] - return next(float_type for float_type in float_types - if np.can_cast(integer_type, float_type)) + return next( + float_type + for float_type in float_types + if np.can_cast(integer_type, float_type) + ) -def _prepare_data_plane(data: np.Array, var_info: VarInfoFromNetCDF4, - var_name: str) -> np.Array: +def _prepare_data_plane( + data: np.Array, var_info: VarInfoFromNetCDF4, var_name: str +) -> np.Array: """Perform Type casting and transpose 2d data array when necessary. If an input data plane is an int, recast to the smallest floating point @@ -228,10 +257,22 @@ def _needs_rotation(var_info: VarInfoFromNetCDF4, variable: str) -> bool: """ needs_rotation = False var_dims = var_info.get_variable(variable).dimensions - xloc = next((index for index, dimension in enumerate(var_dims) - if _is_projection_x_dim(dimension, var_info)), None) - yloc = next((index for index, dimension in enumerate(var_dims) - if _is_projection_y_dim(dimension, var_info)), None) + xloc = next( + ( + index + for index, dimension in enumerate(var_dims) + if _is_projection_x_dim(dimension, var_info) + ), + None, + ) + yloc = next( + ( + index + for index, dimension in enumerate(var_dims) + if _is_projection_y_dim(dimension, var_info) + ), + None, + ) if yloc > xloc: needs_rotation = True @@ -249,7 +290,8 @@ def _validate_remaining_variables(resampled_variables: Dict) -> None: extra_dimensions = variable_dimensions.difference(valid_dimensions) if len(extra_dimensions) != 0: raise RegridderException( - f'Variables with dimensions {extra_dimensions} cannot be handled.') + f'Variables with dimensions {extra_dimensions} cannot be handled.' + ) def _group_by_ndim(var_info: VarInfoFromNetCDF4, variables: Set) -> Dict: @@ -269,10 +311,13 @@ def _group_by_ndim(var_info: VarInfoFromNetCDF4, variables: Set) -> Dict: return grouped_vars -def _copy_resampled_bounds_variable(source_ds: Dataset, target_ds: Dataset, - bounds_var: str, - target_area: AreaDefinition, - var_info: VarInfoFromNetCDF4): +def _copy_resampled_bounds_variable( + source_ds: Dataset, + target_ds: Dataset, + bounds_var: str, + target_area: AreaDefinition, + var_info: VarInfoFromNetCDF4, +): """Copy computed values for dimension variable bounds variables.""" var_dims = var_info.get_variable(bounds_var).dimensions @@ -299,27 +344,34 @@ def _copy_resampled_bounds_variable(source_ds: Dataset, target_ds: Dataset, return {bounds_var} -def _copy_dimension_variables(source_ds: Dataset, target_ds: Dataset, - target_area: AreaDefinition, - var_info: VarInfoFromNetCDF4) -> Set[str]: +def _copy_dimension_variables( + source_ds: Dataset, + target_ds: Dataset, + target_area: AreaDefinition, + var_info: VarInfoFromNetCDF4, +) -> Set[str]: """Copy over dimension variables that are changed in the target file.""" dim_var_names = _resampled_dimension_variable_names(var_info) - processed_vars = _copy_1d_dimension_variables(source_ds, target_ds, - dim_var_names, target_area, - var_info) + processed_vars = _copy_1d_dimension_variables( + source_ds, target_ds, dim_var_names, target_area, var_info + ) bounds_vars = dim_var_names - processed_vars for bounds_var in bounds_vars: processed_vars |= _copy_resampled_bounds_variable( - source_ds, target_ds, bounds_var, target_area, var_info) + source_ds, target_ds, bounds_var, target_area, var_info + ) return processed_vars -def _copy_1d_dimension_variables(source_ds: Dataset, target_ds: Dataset, - dim_var_names: Set[str], - target_area: AreaDefinition, - var_info: VarInfoFromNetCDF4) -> Set[str]: +def _copy_1d_dimension_variables( + source_ds: Dataset, + target_ds: Dataset, + dim_var_names: Set[str], + target_area: AreaDefinition, + var_info: VarInfoFromNetCDF4, +) -> Set[str]: """Copy 1 dimensional dimension variables. These are the variables associated directly with the resampled @@ -327,7 +379,8 @@ def _copy_1d_dimension_variables(source_ds: Dataset, target_ds: Dataset, """ # pylint: disable-msg=too-many-locals one_d_vars = { - dim_var_name for dim_var_name in dim_var_names + dim_var_name + for dim_var_name in dim_var_names if len(var_info.get_variable(dim_var_name).dimensions) == 1 } @@ -340,18 +393,19 @@ def _copy_1d_dimension_variables(source_ds: Dataset, target_ds: Dataset, standard_metadata = { 'long_name': 'longitude', 'standard_name': 'longitude', - 'units': 'degrees_east' + 'units': 'degrees_east', } elif dim_name in ydims: target_coords = target_area.projection_y_coords standard_metadata = { 'long_name': 'latitude', 'standard_name': 'latitude', - 'units': 'degrees_north' + 'units': 'degrees_north', } else: raise RegridderException( - f'dim_name: {dim_name} not found in projection dimensions') + f'dim_name: {dim_name} not found in projection dimensions' + ) (_, t_var) = _copy_var_without_metadata(source_ds, target_ds, dim_name) @@ -367,14 +421,17 @@ def _copy_1d_dimension_variables(source_ds: Dataset, target_ds: Dataset, def _get_bounds_var(var_info: VarInfoFromNetCDF4, dim_name: str) -> str: - return next((var_info.get_variable(f'{dim_name}_{ext}').name - for ext in ['bnds', 'bounds'] - if var_info.get_variable(f'{dim_name}_{ext}') is not None), - None) + return next( + ( + var_info.get_variable(f'{dim_name}_{ext}').name + for ext in ['bnds', 'bounds'] + if var_info.get_variable(f'{dim_name}_{ext}') is not None + ), + None, + ) -def _resampled_dimension_variable_names( - var_info: VarInfoFromNetCDF4) -> Set[str]: +def _resampled_dimension_variable_names(var_info: VarInfoFromNetCDF4) -> Set[str]: """Return the list of dimension variables to resample to target grid. This returns a list of the fully qualified variables that need to use the @@ -391,9 +448,12 @@ def _resampled_dimension_variable_names( return dims_to_transfer -def _transfer_dimensions(source_ds: Dataset, target_ds: Dataset, - target_area: AreaDefinition, - var_info: VarInfoFromNetCDF4) -> None: +def _transfer_dimensions( + source_ds: Dataset, + target_ds: Dataset, + target_area: AreaDefinition, + var_info: VarInfoFromNetCDF4, +) -> None: """Transfer all dimensions from source to target. Horizontal source dimensions that are changed due to resampling, are @@ -406,13 +466,16 @@ def _transfer_dimensions(source_ds: Dataset, target_ds: Dataset, unchanged_dimensions = all_dimensions - resampled_dimensions _copy_dimensions(unchanged_dimensions, source_ds, target_ds) - _create_resampled_dimensions(resampled_dimension_pairs, target_ds, - target_area, var_info) + _create_resampled_dimensions( + resampled_dimension_pairs, target_ds, target_area, var_info + ) -def _write_grid_mappings(target_ds: Dataset, - resampled_dim_pairs: List[Tuple[str, str]], - target_area: AreaDefinition) -> Dict: +def _write_grid_mappings( + target_ds: Dataset, + resampled_dim_pairs: List[Tuple[str, str]], + target_area: AreaDefinition, +) -> Dict: """Add cordinate reference system metadata variables. Add placeholder variables that contain the metadata related the cordinate @@ -436,8 +499,9 @@ def _write_grid_mappings(target_ds: Dataset, return crs_map -def _crs_variable_name(dim_pair: Tuple[str, str], - resampled_dim_pairs: List[Tuple[str, str]]) -> str: +def _crs_variable_name( + dim_pair: Tuple[str, str], resampled_dim_pairs: List[Tuple[str, str]] +) -> str: """Return a crs variable name for this dimension pair. This will be "//crs" unless there are multiple grids in the @@ -457,30 +521,30 @@ def _crs_variable_name(dim_pair: Tuple[str, str], return crs_var_name -def _clone_variables(source_ds: Dataset, target_ds: Dataset, - dimensions: Set[str]) -> Set[str]: +def _clone_variables( + source_ds: Dataset, target_ds: Dataset, dimensions: Set[str] +) -> Set[str]: """Clone variables from source to target. Copy variables and their attributes directly from the source Dataset to the target Dataset. """ for dimension_name in dimensions: - (s_var, t_var) = _copy_var_with_attrs(source_ds, target_ds, - dimension_name) + (s_var, t_var) = _copy_var_with_attrs(source_ds, target_ds, dimension_name) t_var[:] = s_var[:] return dimensions -def _copy_var_with_attrs(source_ds: Dataset, target_ds: Dataset, - variable_name: str) -> (Variable, Variable): +def _copy_var_with_attrs( + source_ds: Dataset, target_ds: Dataset, variable_name: str +) -> (Variable, Variable): """Copy a source variable and metadata to target. Copy both the variable and metadata from a souce variable into a target, return both source and target variables. """ - s_var, t_var = _copy_var_without_metadata(source_ds, target_ds, - variable_name) + s_var, t_var = _copy_var_without_metadata(source_ds, target_ds, variable_name) for att in s_var.ncattrs(): if att != '_FillValue': @@ -489,8 +553,9 @@ def _copy_var_with_attrs(source_ds: Dataset, target_ds: Dataset, return (s_var, t_var) -def _copy_var_without_metadata(source_ds: Dataset, target_ds: Dataset, - variable_name: str) -> (Variable, Variable): +def _copy_var_without_metadata( + source_ds: Dataset, target_ds: Dataset, variable_name: str +) -> (Variable, Variable): """Clones a single variable and returns both source and target variables. This function uses the netCDF4 createGroup('/[optionalgroup/andsubgroup]') @@ -502,10 +567,9 @@ def _copy_var_without_metadata(source_ds: Dataset, target_ds: Dataset, s_var = _get_variable(source_ds, variable_name) t_group = target_ds.createGroup(var.parent) fill_value = getattr(s_var, '_FillValue', None) - t_var = t_group.createVariable(var.name, - s_var.dtype, - s_var.dimensions, - fill_value=fill_value) + t_var = t_group.createVariable( + var.name, s_var.dtype, s_var.dimensions, fill_value=fill_value + ) return (s_var, t_var) @@ -520,30 +584,29 @@ def _get_variable(dataset: Dataset, variable_name: str) -> Variable: return group[var.name] -def _create_dimension(dataset: Dataset, dimension_name: str, - size: int) -> Dimension: +def _create_dimension(dataset: Dataset, dimension_name: str, size: int) -> Dimension: """Create a fully qualified dimension on the dataset.""" dim = PurePath(dimension_name) group = dataset.createGroup(dim.parent) return group.createDimension(dim.name, size) -def _create_resampled_dimensions(resampled_dim_pairs: List[Tuple[str, str]], - dataset: Dataset, target_area: AreaDefinition, - var_info: VarInfoFromNetCDF4): +def _create_resampled_dimensions( + resampled_dim_pairs: List[Tuple[str, str]], + dataset: Dataset, + target_area: AreaDefinition, + var_info: VarInfoFromNetCDF4, +): """Create dimensions for the target resampled grids.""" for dim_pair in resampled_dim_pairs: xdim = _get_projection_x_dims(set(dim_pair), var_info)[0] ydim = _get_projection_y_dims(set(dim_pair), var_info)[0] - _create_dimension(dataset, xdim, - target_area.projection_x_coords.shape[0]) - _create_dimension(dataset, ydim, - target_area.projection_y_coords.shape[0]) + _create_dimension(dataset, xdim, target_area.projection_x_coords.shape[0]) + _create_dimension(dataset, ydim, target_area.projection_y_coords.shape[0]) -def _copy_dimension(dimension_name: str, source_ds: Dataset, - target_ds: Dataset) -> str: +def _copy_dimension(dimension_name: str, source_ds: Dataset, target_ds: Dataset) -> str: """Copy dimension from source to target file.""" source_dimension = _get_dimension(source_ds, dimension_name) @@ -565,8 +628,9 @@ def _get_dimension(dataset: Dataset, dimension_name: str) -> Dimension: return dataset.createGroup(dim.parent).dimensions[dim.name] -def _copy_dimensions(dimensions: Set[str], source_ds: Dataset, - target_ds: Dataset) -> Set[str]: +def _copy_dimensions( + dimensions: Set[str], source_ds: Dataset, target_ds: Dataset +) -> Set[str]: """Copy each dimension from source to target. ensure the first dimensions copied are the UNLIMITED dimensions. @@ -583,12 +647,18 @@ def sort_unlimited_first(dimension_name): _copy_dimension(dim, source_ds, target_ds) -def _horizontal_dims_for_variable(var_info: VarInfoFromNetCDF4, - var_name: str) -> Tuple[str, str]: +def _horizontal_dims_for_variable( + var_info: VarInfoFromNetCDF4, var_name: str +) -> Tuple[str, str]: """Return the horizontal dimensions for desired variable.""" - return next((dims for dims, var_names in - var_info.group_variables_by_horizontal_dimensions().items() - if var_name in var_names), None) + return next( + ( + dims + for dims, var_names in var_info.group_variables_by_horizontal_dimensions().items() + if var_name in var_names + ), + None, + ) def _all_dimensions(var_info: VarInfoFromNetCDF4) -> Set[str]: @@ -612,10 +682,13 @@ def _unresampled_variables(var_info: VarInfoFromNetCDF4) -> Set[str]: vars_by_dim = var_info.group_variables_by_dimensions() resampled_dims = _resampled_dimensions(var_info) - return set.union(*[ - variable_set for dimension_name, variable_set in vars_by_dim.items() - if not resampled_dims.intersection(set(dimension_name)) - ]) + return set.union( + *[ + variable_set + for dimension_name, variable_set in vars_by_dim.items() + if not resampled_dims.intersection(set(dimension_name)) + ] + ) def _all_dimension_variables(var_info: VarInfoFromNetCDF4) -> Set[str]: @@ -623,8 +696,7 @@ def _all_dimension_variables(var_info: VarInfoFromNetCDF4) -> Set[str]: return var_info.get_required_dimensions(var_info.get_all_variables()) -def _resampled_dimension_pairs( - var_info: VarInfoFromNetCDF4) -> List[Tuple[str, str]]: +def _resampled_dimension_pairs(var_info: VarInfoFromNetCDF4) -> List[Tuple[str, str]]: """Return a list of the resampled horizontal spatial dimensions. Gives a list of the 2-element horizontal dimensions that are used in @@ -646,8 +718,9 @@ def _resampled_dimensions(var_info: VarInfoFromNetCDF4) -> Set[str]: return dimensions -def _cache_resamplers(filepath: str, var_info: VarInfoFromNetCDF4, - target_area: AreaDefinition) -> None: +def _cache_resamplers( + filepath: str, var_info: VarInfoFromNetCDF4, target_area: AreaDefinition +) -> None: """Precompute the resampling weights. Determine the desired output Target Area from the Harmony Message. Use @@ -675,17 +748,26 @@ def _cache_resamplers(filepath: str, var_info: VarInfoFromNetCDF4, def _compute_target_area(message: Message) -> AreaDefinition: """Parse the harmony message and build a target AreaDefinition.""" # ScaleExtent is required and validated. - area_extent = (message.format.scaleExtent.x.min, - message.format.scaleExtent.y.min, - message.format.scaleExtent.x.max, - message.format.scaleExtent.y.max) + area_extent = ( + message.format.scaleExtent.x.min, + message.format.scaleExtent.y.min, + message.format.scaleExtent.x.max, + message.format.scaleExtent.y.max, + ) height = _grid_height(message) width = _grid_width(message) projection = message.format.crs or 'EPSG:4326' - return AreaDefinition('target_area_id', 'target area definition', None, - projection, width, height, area_extent) + return AreaDefinition( + 'target_area_id', + 'target area definition', + None, + projection, + width, + height, + area_extent, + ) def _grid_height(message: Message) -> int: @@ -713,8 +795,7 @@ def _compute_num_elements(message: Message, dimension_name: str) -> int: scale_extent = getattr(message.format.scaleExtent, dimension_name) scale_size = getattr(message.format.scaleSize, dimension_name) - num_elements = int( - np.round((scale_extent.max - scale_extent.min) / scale_size)) + num_elements = int(np.round((scale_extent.max - scale_extent.min) / scale_size)) return num_elements @@ -737,30 +818,32 @@ def _is_projection_y_dim(dim: str, var_info: VarInfoFromNetCDF4) -> str: return is_y_dim -def _get_projection_x_dims(dims: Iterable[str], - var_info: VarInfoFromNetCDF4) -> List[str]: +def _get_projection_x_dims( + dims: Iterable[str], var_info: VarInfoFromNetCDF4 +) -> List[str]: """Return name for horizontal grid dimension [column/longitude/x].""" return [dim for dim in dims if _is_projection_x_dim(dim, var_info)] -def _get_projection_y_dims(dims: Iterable[str], - var_info: VarInfoFromNetCDF4) -> str: +def _get_projection_y_dims(dims: Iterable[str], var_info: VarInfoFromNetCDF4) -> str: """Return name for vertical grid dimension [row/latitude/y].""" return [dim for dim in dims if _is_projection_y_dim(dim, var_info)] -def _compute_source_swath(grid_dimensions: Tuple[str, str], filepath: str, - var_info: VarInfoFromNetCDF4) -> SwathDefinition: +def _compute_source_swath( + grid_dimensions: Tuple[str, str], filepath: str, var_info: VarInfoFromNetCDF4 +) -> SwathDefinition: """Return a SwathDefinition for the input gridDimensions.""" longitudes, latitudes = _compute_horizontal_source_grids( - grid_dimensions, filepath, var_info) + grid_dimensions, filepath, var_info + ) return SwathDefinition(lons=longitudes, lats=latitudes) def _compute_horizontal_source_grids( - grid_dimensions: Tuple[str, str], filepath: str, - var_info: VarInfoFromNetCDF4) -> Tuple[np.array, np.array]: + grid_dimensions: Tuple[str, str], filepath: str, var_info: VarInfoFromNetCDF4 +) -> Tuple[np.array, np.array]: """Return 2D np.arrays of longitude and latitude.""" row_dim = _get_projection_y_dims(grid_dimensions, var_info)[0] column_dim = _get_projection_x_dims(grid_dimensions, var_info)[0] @@ -768,20 +851,21 @@ def _compute_horizontal_source_grids( with Dataset(filepath, mode='r') as data_set: row_shape = data_set[row_dim].shape column_shape = data_set[column_dim].shape - if (len(row_shape) == 1 and len(column_shape) == 1): + if len(row_shape) == 1 and len(column_shape) == 1: num_rows = row_shape[0] num_columns = column_shape[0] - longitudes = np.broadcast_to(data_set[column_dim], - (num_rows, num_columns)) + longitudes = np.broadcast_to(data_set[column_dim], (num_rows, num_columns)) latitudes = np.broadcast_to( np.broadcast_to(data_set[row_dim], (1, num_rows)).T, - (num_rows, num_columns)) + (num_rows, num_columns), + ) longitudes = np.ascontiguousarray(longitudes) latitudes = np.ascontiguousarray(latitudes) else: # Only handling the case of 1-Dimensional dimensions on MVP raise InvalidSourceDimensions( f'Incorrect source data dimensions. ' - f'rows:{row_shape}, columns:{column_shape}') + f'rows:{row_shape}, columns:{column_shape}' + ) return (longitudes, latitudes) diff --git a/harmony_regridding_service/utilities.py b/harmony_regridding_service/utilities.py index c26f0a7..e437c8c 100644 --- a/harmony_regridding_service/utilities.py +++ b/harmony_regridding_service/utilities.py @@ -2,6 +2,7 @@ include MIME type determination and basic components of message validation. """ + from mimetypes import guess_type as guess_mime_type from os.path import splitext from typing import Optional @@ -13,34 +14,35 @@ from harmony_regridding_service.exceptions import InvalidTargetCRS -KNOWN_MIME_TYPES = {'.nc4': 'application/x-netcdf4', - '.h5': 'application/x-hdf5', - '.hdf5': 'application/x-hdf5'} -VALID_INTERPOLATION_METHODS = ('Elliptical Weighted Averaging', ) +KNOWN_MIME_TYPES = { + '.nc4': 'application/x-netcdf4', + '.h5': 'application/x-hdf5', + '.hdf5': 'application/x-hdf5', +} +VALID_INTERPOLATION_METHODS = ('Elliptical Weighted Averaging',) def get_file_mime_type(file_name: str) -> Optional[str]: - """ This function tries to infer the MIME type of a file string. If the - `mimetypes.guess_type` function cannot guess the MIME type of the - granule, a dictionary of known file types is checked using the file - extension. That dictionary only contains keys for MIME types that - `mimetypes.guess_type` cannot resolve. + """This function tries to infer the MIME type of a file string. If the + `mimetypes.guess_type` function cannot guess the MIME type of the + granule, a dictionary of known file types is checked using the file + extension. That dictionary only contains keys for MIME types that + `mimetypes.guess_type` cannot resolve. """ mime_type = guess_mime_type(file_name, False) if not mime_type or mime_type[0] is None: - mime_type = (KNOWN_MIME_TYPES.get(splitext(file_name)[1].lower()), - None) + mime_type = (KNOWN_MIME_TYPES.get(splitext(file_name)[1].lower()), None) return mime_type[0] def has_valid_crs(message: Message) -> bool: - """ Check the target Coordinate Reference System (CRS) in the Harmony - message is compatible with the CRS types supported by the regridder. In - the MVP, only a geographic CRS is supported, so `Message.format.crs` - should either be undefined or specify a geographic CRS. + """Check the target Coordinate Reference System (CRS) in the Harmony + message is compatible with the CRS types supported by the regridder. In + the MVP, only a geographic CRS is supported, so `Message.format.crs` + should either be undefined or specify a geographic CRS. """ target_crs = rgetattr(message, 'format.crs') @@ -48,8 +50,8 @@ def has_valid_crs(message: Message) -> bool: def _is_geographic_crs(crs_string: str) -> bool: - """ Use pyproj to ascertain if the supplied Coordinate Reference System - (CRS) is geographic. + """Use pyproj to ascertain if the supplied Coordinate Reference System + (CRS) is geographic. """ try: @@ -62,11 +64,11 @@ def _is_geographic_crs(crs_string: str) -> bool: def has_valid_interpolation(message: Message) -> bool: - """ Check the interpolation method in the input Harmony message is - compatible with the methods supported by the regridder. In the MVP, - only the EWA algorithm is used, so either the interpolation should be - unspecified in the message, or it should be the string - "Elliptical Weighted Averaging". + """Check the interpolation method in the input Harmony message is + compatible with the methods supported by the regridder. In the MVP, + only the EWA algorithm is used, so either the interpolation should be + unspecified in the message, or it should be the string + "Elliptical Weighted Averaging". """ interpolation_method = rgetattr(message, 'format.interpolation') diff --git a/notebooks/Harmony_Regridding_Service_User_Documentation.ipynb b/notebooks/Harmony_Regridding_Service_User_Documentation.ipynb index c25af5a..9474dc2 100644 --- a/notebooks/Harmony_Regridding_Service_User_Documentation.ipynb +++ b/notebooks/Harmony_Regridding_Service_User_Documentation.ipynb @@ -121,16 +121,18 @@ "metadata": {}, "outputs": [], "source": [ - "grid_request = Request(collection=merra_collection, granule_id=merra_granules[0],\n", - " grid='GEOS1x1test')\n", + "grid_request = Request(\n", + " collection=merra_collection, granule_id=merra_granules[0], grid='GEOS1x1test'\n", + ")\n", "\n", "grid_job_id = harmony_client.submit(grid_request)\n", "harmony_client.wait_for_processing(grid_job_id, show_progress=True)\n", "\n", "downloaded_grid_output = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(grid_job_id, overwrite=True, directory=demo_directory)\n", + " for file_future in harmony_client.download_all(\n", + " grid_job_id, overwrite=True, directory=demo_directory\n", + " )\n", "]\n", "print(f'Downloaded: {\", \".join(downloaded_grid_output)}')" ] @@ -172,10 +174,20 @@ "metadata": {}, "outputs": [], "source": [ - "player_grid = pnw.Player(name='time', start=0, end=23, loop_policy='once', interval=200, width=900)\n", - "ax = ds_grid.Q850.interactive(loc='bottom', width=900, height=600).isel(time=player_grid).plot(\n", - " cmap=plt.cm.turbo, vmin=0.0, vmax=0.02, cbar_kwargs={'format': '{x:.3}', 'fraction': 0.0235},\n", - " xlim=[-180, 180], ylim=[-90, 90]\n", + "player_grid = pnw.Player(\n", + " name='time', start=0, end=23, loop_policy='once', interval=200, width=900\n", + ")\n", + "ax = (\n", + " ds_grid.Q850.interactive(loc='bottom', width=900, height=600)\n", + " .isel(time=player_grid)\n", + " .plot(\n", + " cmap=plt.cm.turbo,\n", + " vmin=0.0,\n", + " vmax=0.02,\n", + " cbar_kwargs={'format': '{x:.3}', 'fraction': 0.0235},\n", + " xlim=[-180, 180],\n", + " ylim=[-90, 90],\n", + " )\n", ")\n", "ax.axes.set_aspect('equal')" ] @@ -209,17 +221,24 @@ "metadata": {}, "outputs": [], "source": [ - "detailed_request = Request(collection=merra_collection, granule_id=merra_granules[1],\n", - " scale_size=(1.0, 1.0), scale_extent=(-180, -90, 180, 90),\n", - " crs='EPSG:4326', height=180, width=360)\n", + "detailed_request = Request(\n", + " collection=merra_collection,\n", + " granule_id=merra_granules[1],\n", + " scale_size=(1.0, 1.0),\n", + " scale_extent=(-180, -90, 180, 90),\n", + " crs='EPSG:4326',\n", + " height=180,\n", + " width=360,\n", + ")\n", "\n", "detailed_job_id = harmony_client.submit(detailed_request)\n", "harmony_client.wait_for_processing(detailed_job_id, show_progress=True)\n", "\n", "downloaded_detailed_output = [\n", " file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(detailed_job_id, overwrite=True, directory=demo_directory)\n", + " for file_future in harmony_client.download_all(\n", + " detailed_job_id, overwrite=True, directory=demo_directory\n", + " )\n", "]\n", "print(f'Downloaded: {\", \".join(downloaded_detailed_output)}')" ] diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 3c19583..8701b19 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -1,4 +1,5 @@ """ End-to-end tests of the Harmony Regridding service. """ + from os.path import exists, join as path_join from shutil import rmtree from tempfile import mkdtemp @@ -10,28 +11,31 @@ from pystac import Catalog from harmony_regridding_service.adapter import RegriddingServiceAdapter -from harmony_regridding_service.exceptions import (InvalidInterpolationMethod, - InvalidTargetCRS, - InvalidTargetGrid) +from harmony_regridding_service.exceptions import ( + InvalidInterpolationMethod, + InvalidTargetCRS, + InvalidTargetGrid, +) from tests.utilities import create_stac, Granule class TestAdapter(TestCase): - """ A class testing the harmony_regridding_service.utilities module. """ + """A class testing the harmony_regridding_service.utilities module.""" + @classmethod def setUpClass(cls): - """ Define test fixtures that can be shared between tests. """ + """Define test fixtures that can be shared between tests.""" cls.access_token = 'fake-token' cls.granule_url = 'https://www.example.com/input.nc4' - cls.input_stac = create_stac(Granule(cls.granule_url, - 'application/x-netcdf4', - ['data'])) + cls.input_stac = create_stac( + Granule(cls.granule_url, 'application/x-netcdf4', ['data']) + ) cls.staging_location = 's3://example-bucket' cls.user = 'blightyear' def setUp(self): - """ Define test fixtures that are not shared between tests. """ + """Define test fixtures that are not shared between tests.""" self.temp_dir = mkdtemp() self.config = config(validate=False) @@ -39,13 +43,16 @@ def tearDown(self): if exists(self.temp_dir): rmtree(self.temp_dir) - def assert_expected_output_catalog(self, catalog: Catalog, - expected_href: str, - expected_title: str, - expected_media_type: str): - """ Check the contents of the Harmony output STAC. It should have a - single data item. The URL, title and media type for this asset will - be compared to supplied values. + def assert_expected_output_catalog( + self, + catalog: Catalog, + expected_href: str, + expected_title: str, + expected_media_type: str, + ): + """Check the contents of the Harmony output STAC. It should have a + single data item. The URL, title and media type for this asset will + be compared to supplied values. """ items = list(catalog.get_items()) @@ -53,10 +60,12 @@ def assert_expected_output_catalog(self, catalog: Catalog, self.assertListEqual(list(items[0].assets.keys()), ['data']) self.assertDictEqual( items[0].assets['data'].to_dict(), - {'href': expected_href, - 'title': expected_title, - 'type': expected_media_type, - 'roles': ['data']} + { + 'href': expected_href, + 'title': expected_title, + 'type': expected_media_type, + 'roles': ['data'], + }, ) @patch('harmony_regridding_service.adapter.rmtree') @@ -64,64 +73,76 @@ def assert_expected_output_catalog(self, catalog: Catalog, @patch('harmony_regridding_service.adapter.regrid') @patch('harmony_regridding_service.adapter.download') @patch('harmony_regridding_service.adapter.stage') - def test_valid_request(self, mock_stage, mock_download, mock_regrid, - mock_mkdtemp, mock_rmtree): - """ Ensure a request with a correctly formatted message is fully - processed. + def test_valid_request( + self, mock_stage, mock_download, mock_regrid, mock_mkdtemp, mock_rmtree + ): + """Ensure a request with a correctly formatted message is fully + processed. - This test will need updating when the service functions fully. + This test will need updating when the service functions fully. """ expected_downloaded_file = f'{self.temp_dir}/input.nc4' expected_output_basename = 'input_regridded.nc4' - expected_staged_url = path_join(self.staging_location, - expected_output_basename) + expected_staged_url = path_join(self.staging_location, expected_output_basename) mock_regrid.return_value = expected_downloaded_file mock_mkdtemp.return_value = self.temp_dir mock_download.return_value = expected_downloaded_file mock_stage.return_value = expected_staged_url - message = Message({ - 'accessToken': self.access_token, - 'callback': 'https://example.com/', - 'format': { - 'height': 181, - 'scaleExtent': {'x': {'min': -180, 'max': 180}, - 'y': {'min': -90, 'max': 90}}, - 'width': 361 - }, - 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], - 'stagingLocation': self.staging_location, - 'user': self.user, - }) + message = Message( + { + 'accessToken': self.access_token, + 'callback': 'https://example.com/', + 'format': { + 'height': 181, + 'scaleExtent': { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + }, + 'width': 361, + }, + 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], + 'stagingLocation': self.staging_location, + 'user': self.user, + } + ) - regridder = RegriddingServiceAdapter(message, config=self.config, - catalog=self.input_stac) + regridder = RegriddingServiceAdapter( + message, config=self.config, catalog=self.input_stac + ) _, output_catalog = regridder.invoke() # Ensure the output catalog contains the single, expected item: - self.assert_expected_output_catalog(output_catalog, - expected_staged_url, - expected_output_basename, - 'application/x-netcdf4') + self.assert_expected_output_catalog( + output_catalog, + expected_staged_url, + expected_output_basename, + 'application/x-netcdf4', + ) # Ensure a download was requested via harmony-service-lib: - mock_download.assert_called_once_with(self.granule_url, self.temp_dir, - logger=regridder.logger, - cfg=regridder.config, - access_token=self.access_token) + mock_download.assert_called_once_with( + self.granule_url, + self.temp_dir, + logger=regridder.logger, + cfg=regridder.config, + access_token=self.access_token, + ) # Ensure regrid was called with the input filepath. mock_regrid.assert_called_once_with(ANY, expected_downloaded_file, ANY) # Ensure the file was staged as expected: - mock_stage.assert_called_once_with(expected_downloaded_file, - expected_output_basename, - 'application/x-netcdf4', - logger=regridder.logger, - location=self.staging_location, - cfg=self.config) + mock_stage.assert_called_once_with( + expected_downloaded_file, + expected_output_basename, + 'application/x-netcdf4', + logger=regridder.logger, + location=self.staging_location, + cfg=self.config, + ) # Ensure container clean-up was requested: mock_rmtree.assert_called_once_with(self.temp_dir) @@ -130,28 +151,30 @@ def test_valid_request(self, mock_stage, mock_download, mock_regrid, @patch('harmony_regridding_service.adapter.mkdtemp') @patch('harmony_regridding_service.adapter.download') @patch('harmony_regridding_service.adapter.stage') - def test_missing_grid(self, mock_stage, mock_download, mock_mkdtemp, - mock_rmtree): - """ Ensure a request that fails message validation correctly raises an - exception that is reported at the top level of invocation. Message - validation occurs prior to the `RegriddingServiceAdapter.process_item` - method, so none of the functions or methods within that method - should be called. In this test there are no target grid parameters, - so the validation should raise an `InvalidTargetGrid` exception. + def test_missing_grid(self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree): + """Ensure a request that fails message validation correctly raises an + exception that is reported at the top level of invocation. Message + validation occurs prior to the `RegriddingServiceAdapter.process_item` + method, so none of the functions or methods within that method + should be called. In this test there are no target grid parameters, + so the validation should raise an `InvalidTargetGrid` exception. """ error_message = 'Insufficient or invalid target grid parameters.' - harmony_message = Message({ - 'accessToken': self.access_token, - 'callback': 'https://example.com/', - 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], - 'stagingLocation': self.staging_location, - 'user': self.user, - }) + harmony_message = Message( + { + 'accessToken': self.access_token, + 'callback': 'https://example.com/', + 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], + 'stagingLocation': self.staging_location, + 'user': self.user, + } + ) - regridder = RegriddingServiceAdapter(harmony_message, config=self.config, - catalog=self.input_stac) + regridder = RegriddingServiceAdapter( + harmony_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidTargetGrid) as context_manager: regridder.invoke() @@ -169,36 +192,40 @@ def test_missing_grid(self, mock_stage, mock_download, mock_mkdtemp, @patch('harmony_regridding_service.adapter.mkdtemp') @patch('harmony_regridding_service.adapter.download') @patch('harmony_regridding_service.adapter.stage') - def test_invalid_grid(self, mock_stage, mock_download, mock_mkdtemp, - mock_rmtree): - """ Ensure a request that fails message validation correctly raises an - exception that is reported at the top level of invocation. Message - validation occurs prior to the `RegriddingServiceAdapter.process_item` - method, so none of the functions or methods within that method - should be called. In this test there ae target grid parameters that - are inconsistent with one another, so the validation should raise - an `InvalidTargetGrid` exception. + def test_invalid_grid(self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree): + """Ensure a request that fails message validation correctly raises an + exception that is reported at the top level of invocation. Message + validation occurs prior to the `RegriddingServiceAdapter.process_item` + method, so none of the functions or methods within that method + should be called. In this test there ae target grid parameters that + are inconsistent with one another, so the validation should raise + an `InvalidTargetGrid` exception. """ error_message = 'Insufficient or invalid target grid parameters.' - harmony_message = Message({ - 'accessToken': self.access_token, - 'callback': 'https://example.com/', - 'format': { - 'height': 234, - 'scaleExtent': {'x': {'min': -180, 'max': 180}, - 'y': {'min': -90, 'max': 90}}, - 'scaleSize': {'x': 0.5, 'y': 0.5}, - 'width': 123 - }, - 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], - 'stagingLocation': self.staging_location, - 'user': self.user, - }) + harmony_message = Message( + { + 'accessToken': self.access_token, + 'callback': 'https://example.com/', + 'format': { + 'height': 234, + 'scaleExtent': { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + }, + 'scaleSize': {'x': 0.5, 'y': 0.5}, + 'width': 123, + }, + 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], + 'stagingLocation': self.staging_location, + 'user': self.user, + } + ) - regridder = RegriddingServiceAdapter(harmony_message, config=self.config, - catalog=self.input_stac) + regridder = RegriddingServiceAdapter( + harmony_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidTargetGrid) as context_manager: regridder.invoke() @@ -216,35 +243,41 @@ def test_invalid_grid(self, mock_stage, mock_download, mock_mkdtemp, @patch('harmony_regridding_service.adapter.mkdtemp') @patch('harmony_regridding_service.adapter.download') @patch('harmony_regridding_service.adapter.stage') - def test_invalid_interpolation(self, mock_stage, mock_download, - mock_mkdtemp, mock_rmtree): - """ Ensure a request that fails message validation correctly raises an - exception that is reported at the top level of invocation. Message - validation occurs prior to the `RegriddingServiceAdapter.process_item` - method, so none of the functions or methods within that method - should be called. In this test there is an invalid interpolation in - the Harmony message, so the validation should raise an - `InvalidInterpolationMethod` exception. + def test_invalid_interpolation( + self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree + ): + """Ensure a request that fails message validation correctly raises an + exception that is reported at the top level of invocation. Message + validation occurs prior to the `RegriddingServiceAdapter.process_item` + method, so none of the functions or methods within that method + should be called. In this test there is an invalid interpolation in + the Harmony message, so the validation should raise an + `InvalidInterpolationMethod` exception. """ error_message = 'Interpolation method not supported: "Bilinear"' - harmony_message = Message({ - 'accessToken': self.access_token, - 'callback': 'https://example.com/', - 'format': { - 'interpolation': 'Bilinear', - 'scaleExtent': {'x': {'min': -180, 'max': 180}, - 'y': {'min': -90, 'max': 90}}, - 'scaleSize': {'x': 0.5, 'y': 0.5}, - }, - 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], - 'stagingLocation': self.staging_location, - 'user': self.user, - }) + harmony_message = Message( + { + 'accessToken': self.access_token, + 'callback': 'https://example.com/', + 'format': { + 'interpolation': 'Bilinear', + 'scaleExtent': { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + }, + 'scaleSize': {'x': 0.5, 'y': 0.5}, + }, + 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], + 'stagingLocation': self.staging_location, + 'user': self.user, + } + ) - regridder = RegriddingServiceAdapter(harmony_message, config=self.config, - catalog=self.input_stac) + regridder = RegriddingServiceAdapter( + harmony_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidInterpolationMethod) as context_manager: regridder.invoke() @@ -262,35 +295,39 @@ def test_invalid_interpolation(self, mock_stage, mock_download, @patch('harmony_regridding_service.adapter.mkdtemp') @patch('harmony_regridding_service.adapter.download') @patch('harmony_regridding_service.adapter.stage') - def test_invalid_crs(self, mock_stage, mock_download, mock_mkdtemp, - mock_rmtree): - """ Ensure a request that fails message validation correctly raises an - exception that is reported at the top level of invocation. Message - validation occurs prior to the `RegriddingServiceAdapter.process_item` - method, so none of the functions or methods within that method - should be called. In this test there is an invalid target CRS - specified in the Harmony message, so the validation should raise an - `InvalidTargetCRS` exception. + def test_invalid_crs(self, mock_stage, mock_download, mock_mkdtemp, mock_rmtree): + """Ensure a request that fails message validation correctly raises an + exception that is reported at the top level of invocation. Message + validation occurs prior to the `RegriddingServiceAdapter.process_item` + method, so none of the functions or methods within that method + should be called. In this test there is an invalid target CRS + specified in the Harmony message, so the validation should raise an + `InvalidTargetCRS` exception. """ error_message = 'Target CRS not supported: "invalid CRS"' - harmony_message = Message({ - 'accessToken': self.access_token, - 'callback': 'https://example.com/', - 'format': { - 'crs': 'invalid CRS', - 'scaleExtent': {'x': {'min': -180, 'max': 180}, - 'y': {'min': -90, 'max': 90}}, - 'scaleSize': {'x': 0.5, 'y': 0.5}, - }, - 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], - 'stagingLocation': self.staging_location, - 'user': self.user, - }) + harmony_message = Message( + { + 'accessToken': self.access_token, + 'callback': 'https://example.com/', + 'format': { + 'crs': 'invalid CRS', + 'scaleExtent': { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + }, + 'scaleSize': {'x': 0.5, 'y': 0.5}, + }, + 'sources': [{'collection': 'C1234-EEDTEST', 'shortName': 'test'}], + 'stagingLocation': self.staging_location, + 'user': self.user, + } + ) - regridder = RegriddingServiceAdapter(harmony_message, config=self.config, - catalog=self.input_stac) + regridder = RegriddingServiceAdapter( + harmony_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidTargetCRS) as context_manager: regridder.invoke() diff --git a/tests/test_code_format.py b/tests/test_code_format.py index 0b4ba6d..119ed4a 100644 --- a/tests/test_code_format.py +++ b/tests/test_code_format.py @@ -5,26 +5,27 @@ class TestCodeFormat(TestCase): - """ This test class should ensure all Harmony service Python code adheres - to standard Python code styling. - - Ignored errors and warning: - - * E501: Line length, which defaults to 80 characters. This is a - preferred feature of the code, but not always easily achieved. - * W503: Break before binary operator. Have to ignore one of W503 or - W504 to allow for breaking of some long lines. PEP8 suggests - breaking the line before a binary operatore is more "Pythonic". - * E203, E701: This repository uses black code formatting, which deviates - from PEP8 for these errors. + """This test class should ensure all Harmony service Python code adheres + to standard Python code styling. + + Ignored errors and warning: + + * E501: Line length, which defaults to 80 characters. This is a + preferred feature of the code, but not always easily achieved. + * W503: Break before binary operator. Have to ignore one of W503 or + W504 to allow for breaking of some long lines. PEP8 suggests + breaking the line before a binary operatore is more "Pythonic". + * E203, E701: This repository uses black code formatting, which deviates + from PEP8 for these errors. """ + @classmethod def setUpClass(cls): cls.python_files = Path('harmony_regridding_service').rglob('*.py') def test_pycodestyle_adherence(self): - """ Ensure all code in the `pymods` directory adheres to PEP8 - defined standard. + """Ensure all code in the `pymods` directory adheres to PEP8 + defined standard. """ style_guide = StyleGuide(ignore=['E501', 'W503', 'E203', 'E701']) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index 1b5dcc3..e22919f 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -4,37 +4,47 @@ from harmony.util import config, HarmonyException from harmony_regridding_service.adapter import RegriddingServiceAdapter -from harmony_regridding_service.exceptions import (InvalidInterpolationMethod, - InvalidTargetCRS, - InvalidTargetGrid) +from harmony_regridding_service.exceptions import ( + InvalidInterpolationMethod, + InvalidTargetCRS, + InvalidTargetGrid, +) from tests.utilities import create_stac, Granule class TestAdapter(TestCase): - """ A class testing the harmony_regridding_service.utilities module. """ + """A class testing the harmony_regridding_service.utilities module.""" + @classmethod def setUpClass(cls): - """ Define test fixtures that can be shared between tests. """ + """Define test fixtures that can be shared between tests.""" cls.config = config(validate=False) - cls.input_stac = create_stac(Granule('www.example.com/file.nc4', - 'application/x-netcdf4', - ['data'])) + cls.input_stac = create_stac( + Granule('www.example.com/file.nc4', 'application/x-netcdf4', ['data']) + ) def test_validate_message(self): - """ Ensure only messages with expected content will be processed. """ - valid_scale_extents = {'x': {'min': -180, 'max': 180}, - 'y': {'min': -90, 'max': 90}} + """Ensure only messages with expected content will be processed.""" + valid_scale_extents = { + 'x': {'min': -180, 'max': 180}, + 'y': {'min': -90, 'max': 90}, + } valid_scale_sizes = {'x': 0.5, 'y': 1.0} valid_height = 181 valid_width = 721 with self.subTest('Valid grid, no CRS or interpolation is valid'): - test_message = Message({ - 'format': {'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) try: harmony_adapter.validate_message() @@ -42,15 +52,20 @@ def test_validate_message(self): self.fail(f'Unexpected exception: {exception.message}') with self.subTest('Valid grid (scaleExtent and height/width) is valid'): - test_message = Message({ - 'format': {'crs': 'EPSG:4326', - 'height': valid_height, - 'interpolation': 'Elliptical Weighted Averaging', - 'scaleExtent': valid_scale_extents, - 'width': valid_width} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'crs': 'EPSG:4326', + 'height': valid_height, + 'interpolation': 'Elliptical Weighted Averaging', + 'scaleExtent': valid_scale_extents, + 'width': valid_width, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) try: harmony_adapter.validate_message() @@ -58,13 +73,18 @@ def test_validate_message(self): self.fail(f'Unexpected exception: {exception.message}') with self.subTest('Valid grid and CRS, no interpolation is valid'): - test_message = Message({ - 'format': {'crs': 'EPSG:4326', - 'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'crs': 'EPSG:4326', + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) try: harmony_adapter.validate_message() @@ -72,13 +92,18 @@ def test_validate_message(self): self.fail(f'Unexpected exception: {exception.message}') with self.subTest('Valid grid and interpolation, no CRS is valid'): - test_message = Message({ - 'format': {'interpolation': 'Elliptical Weighted Averaging', - 'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'interpolation': 'Elliptical Weighted Averaging', + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) try: harmony_adapter.validate_message() @@ -86,14 +111,19 @@ def test_validate_message(self): self.fail(f'Unexpected exception: {exception.message}') with self.subTest('Valid grid, CRS and interpolation is valid'): - test_message = Message({ - 'format': {'crs': 'EPSG:4326', - 'interpolation': 'Elliptical Weighted Averaging', - 'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'crs': 'EPSG:4326', + 'interpolation': 'Elliptical Weighted Averaging', + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) try: harmony_adapter.validate_message() @@ -101,49 +131,67 @@ def test_validate_message(self): self.fail(f'Unexpected exception: {exception.message}') with self.subTest('Inconsistent grid is not valid'): - test_message = Message({ - 'format': {'height': valid_height + 100, - 'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes, - 'width': valid_width - 150} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'height': valid_height + 100, + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + 'width': valid_width - 150, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidTargetGrid) as context: harmony_adapter.validate_message() self.assertEqual( context.exception.message, - 'Insufficient or invalid target grid parameters.' + 'Insufficient or invalid target grid parameters.', ) with self.subTest('Non-geographic CRS is not valid'): - test_message = Message({ - 'format': {'crs': 'EPSG:6933', - 'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'crs': 'EPSG:6933', + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidTargetCRS) as context: harmony_adapter.validate_message() - self.assertEqual(context.exception.message, - 'Target CRS not supported: "EPSG:6933"') + self.assertEqual( + context.exception.message, 'Target CRS not supported: "EPSG:6933"' + ) with self.subTest('Non-EWA interpolation method is not valid'): - test_message = Message({ - 'format': {'interpolation': 'Bilinear', - 'scaleExtent': valid_scale_extents, - 'scaleSize': valid_scale_sizes} - }) - harmony_adapter = RegriddingServiceAdapter(test_message, config=self.config, - catalog=self.input_stac) + test_message = Message( + { + 'format': { + 'interpolation': 'Bilinear', + 'scaleExtent': valid_scale_extents, + 'scaleSize': valid_scale_sizes, + } + } + ) + harmony_adapter = RegriddingServiceAdapter( + test_message, config=self.config, catalog=self.input_stac + ) with self.assertRaises(InvalidInterpolationMethod) as context: harmony_adapter.validate_message() - self.assertEqual(context.exception.message, - 'Interpolation method not supported: "Bilinear"') + self.assertEqual( + context.exception.message, + 'Interpolation method not supported: "Bilinear"', + ) diff --git a/tests/unit/test_regridding_service.py b/tests/unit/test_regridding_service.py index aa288cc..139bd53 100644 --- a/tests/unit/test_regridding_service.py +++ b/tests/unit/test_regridding_service.py @@ -15,8 +15,10 @@ from varinfo import VarInfoFromNetCDF4 import harmony_regridding_service.regridding_service as rs -from harmony_regridding_service.exceptions import (InvalidSourceDimensions, - RegridderException) +from harmony_regridding_service.exceptions import ( + InvalidSourceDimensions, + RegridderException, +) class TestRegriddingService(TestCase): @@ -31,16 +33,18 @@ def setUpClass(cls): # Test fixtures representing typical input data # ATL14 data includes two X,Y input grids. cls.test_ATL14_ncfile = Path( - Path(__file__).parent, 'fixtures', 'empty-ATL14.nc') + Path(__file__).parent, 'fixtures', 'empty-ATL14.nc' + ) # MERRA2 has 4 dim variables that are flat at the root cls.test_MERRA2_ncfile = Path( - Path(__file__).parent, 'fixtures', 'empty-MERRA2.nc') + Path(__file__).parent, 'fixtures', 'empty-MERRA2.nc' + ) # IMERG data variables are contained in netcdf groups cls.test_IMERG_ncfile = Path( - Path(__file__).parent, 'fixtures', 'empty-IMERG.nc') + Path(__file__).parent, 'fixtures', 'empty-IMERG.nc' + ) - cls.longitudes = np.array([-180, -80, -45, 45, 80, 180], - dtype=np.dtype('f8')) + cls.longitudes = np.array([-180, -80, -45, 45, 80, 180], dtype=np.dtype('f8')) cls.latitudes = np.array([90, 45, 0, -46, -89], dtype=np.dtype('f8')) cls.test_1D_dimensions_ncfile = Path(cls.tmp_dir, '1D_test.nc') @@ -48,10 +52,7 @@ def setUpClass(cls): # Set up file with one dimensional /lon and /lat root variables dataset = Dataset(cls.test_1D_dimensions_ncfile, 'w') - dataset.setncatts({ - 'root-attribute1': 'value1', - 'root-attribute2': 'value2' - }) + dataset.setncatts({'root-attribute1': 'value1', 'root-attribute2': 'value2'}) # Set up some groups and metadata group1 = dataset.createGroup('/level1-nested1') @@ -68,22 +69,21 @@ def setUpClass(cls): dataset.createVariable('/lon', cls.longitudes.dtype, dimensions=('lon')) dataset.createVariable('/lat', cls.latitudes.dtype, dimensions=('lat')) - dataset.createVariable('/data', - np.dtype('f8'), - dimensions=('lon', 'lat')) + dataset.createVariable('/data', np.dtype('f8'), dimensions=('lon', 'lat')) dataset.createVariable('/time', np.dtype('f8'), dimensions=('time')) - dataset.createVariable('/time_bnds', - np.dtype('u2'), - dimensions=('time', 'bnds')) + dataset.createVariable( + '/time_bnds', np.dtype('u2'), dimensions=('time', 'bnds') + ) dataset['lat'][:] = cls.latitudes dataset['lon'][:] = cls.longitudes - dataset['time'][:] = [1., 2., 3., 4.] + dataset['time'][:] = [1.0, 2.0, 3.0, 4.0] dataset['data'][:] = np.arange( - len(cls.longitudes) * len(cls.latitudes)).reshape( - (len(cls.longitudes), len(cls.latitudes))) - dataset['time_bnds'][:] = np.array([[.5, 1.5, 2.5, 3.5], - [1.5, 2.5, 3.5, 4.5]]).T + len(cls.longitudes) * len(cls.latitudes) + ).reshape((len(cls.longitudes), len(cls.latitudes))) + dataset['time_bnds'][:] = np.array( + [[0.5, 1.5, 2.5, 3.5], [1.5, 2.5, 3.5, 4.5]] + ).T dataset['lon'].setncattr('units', 'degrees_east') dataset['lat'].setncattr('units', 'degrees_north') @@ -95,12 +95,8 @@ def setUpClass(cls): dataset = Dataset(cls.test_2D_dimensions_ncfile, 'w') dataset.createDimension('lon', size=(len(cls.longitudes))) dataset.createDimension('lat', size=(len(cls.latitudes))) - dataset.createVariable('/lon', - cls.longitudes.dtype, - dimensions=('lon', 'lat')) - dataset.createVariable('/lat', - cls.latitudes.dtype, - dimensions=('lon', 'lat')) + dataset.createVariable('/lon', cls.longitudes.dtype, dimensions=('lon', 'lat')) + dataset.createVariable('/lat', cls.latitudes.dtype, dimensions=('lon', 'lat')) dataset['lon'].setncattr('units', 'degrees_east') dataset['lat'].setncattr('units', 'degrees_north') dataset['lat'][:] = np.broadcast_to(cls.latitudes, (6, 5)) @@ -108,41 +104,30 @@ def setUpClass(cls): dataset.close() # Set up test Harmony messages - cls.test_message_with_scale_size = Message({ - 'format': { - 'scaleSize': { - 'x': 10, - 'y': 10 - }, - 'scaleExtent': { - 'x': { - 'min': 0, - 'max': 1000 + cls.test_message_with_scale_size = Message( + { + 'format': { + 'scaleSize': {'x': 10, 'y': 10}, + 'scaleExtent': { + 'x': {'min': 0, 'max': 1000}, + 'y': {'min': 0, 'max': 500}, }, - 'y': { - 'min': 0, - 'max': 500 - } } } - }) - - cls.test_message_with_height_width = Message({ - 'format': { - 'height': 80, - 'width': 40, - 'scaleExtent': { - 'x': { - 'min': 0, - 'max': 1000 + ) + + cls.test_message_with_height_width = Message( + { + 'format': { + 'height': 80, + 'width': 40, + 'scaleExtent': { + 'x': {'min': 0, 'max': 1000}, + 'y': {'min': 0, 'max': 500}, }, - 'y': { - 'min': 0, - 'max': 500 - } } } - }) + ) @classmethod def setUp(self): @@ -163,14 +148,22 @@ def test_file(self): @classmethod def var_info(cls, source_filename): - return VarInfoFromNetCDF4(source_filename, - config_file=rs.HRS_VARINFO_CONFIG_FILENAME) + return VarInfoFromNetCDF4( + source_filename, config_file=rs.HRS_VARINFO_CONFIG_FILENAME + ) @classmethod def test_area(cls, width=360, height=180, area_extent=(-180, -90, 180, 90)): projection = '+proj=longlat +datum=WGS84 +no_defs +type=crs' - return AreaDefinition('test_id', 'test area definition', None, - projection, width, height, area_extent) + return AreaDefinition( + 'test_id', + 'test area definition', + None, + projection, + width, + height, + area_extent, + ) def test_group_by_ndim(self): with self.subTest('one var'): @@ -183,11 +176,7 @@ def test_group_by_ndim(self): with self.subTest('MERRA2'): var_info = self.var_info(self.test_MERRA2_ncfile) variables = {'/OMEGA', '/RH', '/PHIS', '/PS', '/lat'} - expected_sorted = { - 4: {'/OMEGA', '/RH'}, - 3: {'/PHIS', '/PS'}, - 1: {'/lat'} - } + expected_sorted = {4: {'/OMEGA', '/RH'}, 3: {'/PHIS', '/PS'}, 1: {'/lat'}} actual_sorted = rs._group_by_ndim(var_info, variables) self.assertDictEqual(expected_sorted, actual_sorted) @@ -195,9 +184,7 @@ def test_walk_groups(self): """Demonstrate traversing all groups.""" target_path = self.test_file() groups = ['/a/nested/group', '/b/another/deeper/group2'] - expected_visited = { - 'a', 'nested', 'group', 'b', 'another', 'deeper', 'group2' - } + expected_visited = {'a', 'nested', 'group', 'b', 'another', 'deeper', 'group2'} with Dataset(target_path, mode='w') as target_ds: for group in groups: @@ -218,20 +205,19 @@ def test_copy_1d_dimension_variables(self): dim_var_names = {'/lon', '/lat'} expected_attributes = {'long_name', 'standard_name', 'units'} vars_copied = [] - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, target_area, var_info) vars_copied = rs._copy_1d_dimension_variables( - source_ds, target_ds, dim_var_names, target_area, var_info) + source_ds, target_ds, dim_var_names, target_area, var_info + ) self.assertEqual(dim_var_names, vars_copied) with Dataset(target_file, mode='r') as validate: - assert_array_equal(validate['/lon'][:], - target_area.projection_x_coords) - assert_array_equal(validate['/lat'][:], - target_area.projection_y_coords) - self.assertSetEqual(expected_attributes, - set(validate['/lat'].ncattrs())) + assert_array_equal(validate['/lon'][:], target_area.projection_x_coords) + assert_array_equal(validate['/lat'][:], target_area.projection_y_coords) + self.assertSetEqual(expected_attributes, set(validate['/lat'].ncattrs())) with self.assertRaises(AttributeError): validate['/lat'].getncattr('non-standard-attribute') @@ -239,8 +225,9 @@ def test_copy_vars_without_metadata(self): target_file = self.test_file() target_area = self.test_area() var_info = self.var_info(self.test_1D_dimensions_ncfile) - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, target_area, var_info) rs._copy_var_without_metadata(source_ds, target_ds, '/data') @@ -257,8 +244,9 @@ def test_copy_var_with_attrs(self): target_area = self.test_area() var_info = self.var_info(self.test_1D_dimensions_ncfile) expected_metadata = {'units': 'widgets per month'} - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, target_area, var_info) rs._copy_var_with_attrs(source_ds, target_ds, '/data') @@ -278,12 +266,14 @@ def test_copy_dimension_variables(self): var_info = self.var_info(self.test_MERRA2_ncfile) expected_vars_copied = {'/lon', '/lat'} - with Dataset(self.test_MERRA2_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_MERRA2_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, target_area, var_info) - vars_copied = rs._copy_dimension_variables(source_ds, target_ds, - target_area, var_info) + vars_copied = rs._copy_dimension_variables( + source_ds, target_ds, target_area, var_info + ) self.assertSetEqual(expected_vars_copied, vars_copied) @@ -296,9 +286,9 @@ def test_prepare_data_plane(self): with self.subTest('floating point data without rotation'): var_info = self.var_info(self.test_MERRA2_ncfile) - test_data = np.ma.array(np.arange(12).reshape(4, 3), - fill_value=-9999.9, - dtype=np.float32) + test_data = np.ma.array( + np.arange(12).reshape(4, 3), fill_value=-9999.9, dtype=np.float32 + ) var_name = '/T' expected_data = np.ma.copy(test_data) actual_data = rs._prepare_data_plane(test_data, var_info, var_name) @@ -308,9 +298,9 @@ def test_prepare_data_plane(self): with self.subTest('floating point data with rotation'): var_info = self.var_info(self.test_IMERG_ncfile) - test_data = np.ma.array(np.arange(12).reshape(4, 3), - fill_value=-9999.9, - dtype=np.float16) + test_data = np.ma.array( + np.arange(12).reshape(4, 3), fill_value=-9999.9, dtype=np.float16 + ) var_name = '/Grid/HQprecipitation' expected_data = np.ma.copy(test_data.T) actual_data = rs._prepare_data_plane(test_data, var_info, var_name) @@ -320,9 +310,9 @@ def test_prepare_data_plane(self): with self.subTest('integer data without rotation'): var_info = self.var_info(self.test_MERRA2_ncfile) - test_data = np.ma.array(np.arange(12).reshape(4, 3), - fill_value=-9, - dtype=np.int8) + test_data = np.ma.array( + np.arange(12).reshape(4, 3), fill_value=-9, dtype=np.int8 + ) var_name = '/T' expected_data = np.ma.copy(test_data) actual_data = rs._prepare_data_plane(test_data, var_info, var_name) @@ -332,9 +322,9 @@ def test_prepare_data_plane(self): with self.subTest('integer data with rotation'): var_info = self.var_info(self.test_IMERG_ncfile) - test_data = np.ma.array(np.arange(12).reshape(4, 3), - fill_value=-99999999, - dtype=np.int64) + test_data = np.ma.array( + np.arange(12).reshape(4, 3), fill_value=-99999999, dtype=np.int64 + ) var_name = '/Grid/HQprecipitation' expected_data = np.ma.copy(test_data.T).astype(np.float64) @@ -357,22 +347,25 @@ def test_copy_resampled_bounds_variable(self): bnds_var = '/Grid/lat_bnds' var_copied = None - expected_lat_bnds = np.array([ - target_area.projection_y_coords + .5, - target_area.projection_y_coords - .5 - ]).T + expected_lat_bnds = np.array( + [ + target_area.projection_y_coords + 0.5, + target_area.projection_y_coords - 0.5, + ] + ).T - with Dataset(self.test_IMERG_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_IMERG_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, target_area, var_info) var_copied = rs._copy_resampled_bounds_variable( - source_ds, target_ds, bnds_var, target_area, var_info) + source_ds, target_ds, bnds_var, target_area, var_info + ) self.assertEqual({bnds_var}, var_copied) with Dataset(target_file, mode='r') as validate: - assert_array_equal(expected_lat_bnds, - validate['Grid']['lat_bnds'][:]) + assert_array_equal(expected_lat_bnds, validate['Grid']['lat_bnds'][:]) def test_resampled_dimension_variable_names(self): with self.subTest('root level dimensions'): @@ -385,7 +378,10 @@ def test_resampled_dimension_variable_names(self): with self.subTest('grouped dimensions'): var_info = self.var_info(self.test_IMERG_ncfile) expected_resampled = { - '/Grid/lon', '/Grid/lat', '/Grid/lon_bnds', '/Grid/lat_bnds' + '/Grid/lon', + '/Grid/lat', + '/Grid/lon_bnds', + '/Grid/lat_bnds', } actual_resampled = rs._resampled_dimension_variable_names(var_info) @@ -402,9 +398,11 @@ def test_crs_variable_name(self): with self.subTest('multiple grids, separate groups'): dim_pair = ('/Grid/lat', '/Grid/lon') - dim_pairs = [('/Grid/lat', '/Grid/lon'), - ('/Grid2/lat', '/Grid2/lon'), - ('/Grid3/lat', '/Grid3/lon')] + dim_pairs = [ + ('/Grid/lat', '/Grid/lon'), + ('/Grid2/lat', '/Grid2/lon'), + ('/Grid3/lat', '/Grid3/lon'), + ] expected_crs_name = '/Grid/crs' actual_crs_name = rs._crs_variable_name(dim_pair, dim_pairs) @@ -420,9 +418,11 @@ def test_crs_variable_name(self): with self.subTest('multiple grids share group'): dim_pair = ('/global_grid_lat', '/global_grid_lon') - dim_pairs = [('/npolar_grid_lat', '/npolar_grid_lon'), - ('/global_grid_lat', '/global_grid_lon'), - ('/spolar_grid_lat', '/spolar_grid_lon')] + dim_pairs = [ + ('/npolar_grid_lat', '/npolar_grid_lon'), + ('/global_grid_lat', '/global_grid_lon'), + ('/spolar_grid_lat', '/spolar_grid_lon'), + ] expected_crs_name = '/crs_global_grid_lat_global_grid_lon' actual_crs_name = rs._crs_variable_name(dim_pair, dim_pairs) @@ -435,14 +435,14 @@ def test_transfer_metadata(self): # metadata Set in the test 1D file expected_root_metadata = { 'root-attribute1': 'value1', - 'root-attribute2': 'value2' + 'root-attribute2': 'value2', } expected_root_groups = {'level1-nested1', 'level1-nested2'} expected_nested_metadata = {'level2-nested1': 'level2-nested1-value1'} - - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(test_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + test_file, mode='w' + ) as target_ds: rs._transfer_metadata(source_ds, target_ds) with Dataset(test_file, mode='r') as validate: @@ -452,8 +452,7 @@ def test_transfer_metadata(self): root_groups = set(validate.groups.keys()) nested_group = validate['/level1-nested1/level2-nested1'] nested_metadata = { - attr: nested_group.getncattr(attr) - for attr in nested_group.ncattrs() + attr: nested_group.getncattr(attr) for attr in nested_group.ncattrs() } self.assertSetEqual(expected_root_groups, root_groups) @@ -472,13 +471,21 @@ def test_transfer_dimensions(self): width = 36 height = 18 area_extent = (-180, -90, 180, 90) - test_area = AreaDefinition('test_id', 'test area definition', None, - projection, width, height, area_extent) + test_area = AreaDefinition( + 'test_id', + 'test area definition', + None, + projection, + width, + height, + area_extent, + ) var_info = self.var_info(self.test_1D_dimensions_ncfile) target_file = self.test_file() - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, test_area, var_info) with Dataset(target_file, mode='r') as validate: @@ -495,11 +502,19 @@ def test_clone_variables(self): width = 36 height = 18 area_extent = (-180, -90, 180, 90) - test_area = AreaDefinition('test_id', 'test area definition', None, - projection, width, height, area_extent) + test_area = AreaDefinition( + 'test_id', + 'test area definition', + None, + projection, + width, + height, + area_extent, + ) copy_vars = {'/time', '/time_bnds'} - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_dimensions(source_ds, target_ds, test_area, var_info) @@ -508,8 +523,7 @@ def test_clone_variables(self): self.assertEqual(copy_vars, copied) with Dataset(target_file, mode='r') as validate: - assert_array_equal(validate['time_bnds'], - source_ds['time_bnds']) + assert_array_equal(validate['time_bnds'], source_ds['time_bnds']) assert_array_equal(validate['time'], source_ds['time']) def test_create_resampled_dimensions(self): @@ -519,13 +533,21 @@ def test_create_resampled_dimensions(self): width = 36 height = 18 area_extent = (-180, -90, 180, 90) - test_area = AreaDefinition('test_id', 'test area definition', None, - projection, width, height, area_extent) + test_area = AreaDefinition( + 'test_id', + 'test area definition', + None, + projection, + width, + height, + area_extent, + ) target_file = self.test_file() with Dataset(target_file, mode='w') as target_ds: - rs._create_resampled_dimensions([('/lat', '/lon')], target_ds, - test_area, var_info) + rs._create_resampled_dimensions( + [('/lat', '/lon')], target_ds, test_area, var_info + ) with Dataset(target_file, mode='r') as validate: self.assertEqual(validate.dimensions['lat'].size, 18) @@ -537,12 +559,20 @@ def test_create_resampled_dimensions(self): width = 360 height = 180 area_extent = (-180, -90, 180, 90) - test_area = AreaDefinition('test_id', 'test area definition', None, - projection, width, height, area_extent) + test_area = AreaDefinition( + 'test_id', + 'test area definition', + None, + projection, + width, + height, + area_extent, + ) target_file = self.test_file() with Dataset(target_file, mode='w') as target_ds: - rs._create_resampled_dimensions([('/Grid/lon', '/Grid/lat')], - target_ds, test_area, var_info) + rs._create_resampled_dimensions( + [('/Grid/lon', '/Grid/lat')], target_ds, test_area, var_info + ) with Dataset(target_file, mode='r') as validate: self.assertEqual(validate['Grid'].dimensions['lat'].size, 180) @@ -550,29 +580,25 @@ def test_create_resampled_dimensions(self): def test_resampler_kwards(self): with self.subTest('floating data'): - data = np.array([1., 2., 3.], dtype='float') + data = np.array([1.0, 2.0, 3.0], dtype='float') expected_args = {'rows_per_scan': 0} actual_args = rs._resampler_kwargs(data) self.assertDictEqual(expected_args, actual_args) with self.subTest('with fill'): - data = np.ma.array([1., 2., 3.], - mask=[0, 0, 0], - fill_value=-9999.9, - dtype='float') + data = np.ma.array( + [1.0, 2.0, 3.0], mask=[0, 0, 0], fill_value=-9999.9, dtype='float' + ) expected_args = {'rows_per_scan': 0, 'fill_value': -9999.9} actual_args = rs._resampler_kwargs(data) self.assertDictEqual(expected_args, actual_args) with self.subTest('integer data'): - data = np.ma.array([1, 2, 3], - mask=[0, 0, 0], - fill_value=-8, - dtype='int16') + data = np.ma.array([1, 2, 3], mask=[0, 0, 0], fill_value=-8, dtype='int16') expected_args = { 'rows_per_scan': 0, 'fill_value': -8, - 'maximum_weight_mode': True + 'maximum_weight_mode': True, } actual_args = rs._resampler_kwargs(data) self.assertDictEqual(expected_args, actual_args) @@ -583,22 +609,22 @@ def test_write_grid_mappings(self): test_area = self.test_area() expected_crs_map = {('/lon', '/lat'): '/crs'} - with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, \ - Dataset(target_file, mode='w') as target_ds: + with Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds, Dataset( + target_file, mode='w' + ) as target_ds: rs._transfer_metadata(source_ds, target_ds) rs._transfer_dimensions(source_ds, target_ds, test_area, var_info) actual_crs_map = rs._write_grid_mappings( - target_ds, rs._resampled_dimension_pairs(var_info), test_area) + target_ds, rs._resampled_dimension_pairs(var_info), test_area + ) self.assertDictEqual(expected_crs_map, actual_crs_map) with Dataset(target_file, mode='r') as validate: crs = rs._get_variable(validate, '/crs') expected_crs_metadata = test_area.crs.to_cf() - actual_crs_metadata = { - attr: crs.getncattr(attr) for attr in crs.ncattrs() - } + actual_crs_metadata = {attr: crs.getncattr(attr) for attr in crs.ncattrs()} self.assertDictEqual(expected_crs_metadata, actual_crs_metadata) @@ -645,8 +671,9 @@ def test_get_nested_dimension(self): self.assertTrue(lat_dim.size, 1800) def test_copy_dimension(self): - with Dataset(self.test_file(), mode='w' ) as target_ds, \ - Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds: + with Dataset(self.test_file(), mode='w') as target_ds, Dataset( + self.test_1D_dimensions_ncfile, mode='r' + ) as source_ds: time_dimension = rs._copy_dimension('/time', source_ds, target_ds) self.assertTrue(time_dimension.isunlimited()) self.assertEqual(time_dimension.size, 0) @@ -657,27 +684,30 @@ def test_copy_dimension(self): def test_copy_dimensions(self): test_target = self.test_file() - with Dataset(test_target, mode='w' ) as target_ds, \ - Dataset(self.test_1D_dimensions_ncfile, mode='r') as source_ds: - rs._copy_dimensions({'/lat', '/lon', '/time', '/bnds'}, source_ds, - target_ds) + with Dataset(test_target, mode='w') as target_ds, Dataset( + self.test_1D_dimensions_ncfile, mode='r' + ) as source_ds: + rs._copy_dimensions( + {'/lat', '/lon', '/time', '/bnds'}, source_ds, target_ds + ) with Dataset(test_target, mode='r') as validate: self.assertTrue(validate.dimensions['time'].isunlimited()) self.assertEqual(validate.dimensions['time'].size, 0) - self.assertEqual(validate.dimensions['lat'].size, - len(self.latitudes)) - self.assertEqual(validate.dimensions['lon'].size, - len(self.longitudes)) + self.assertEqual(validate.dimensions['lat'].size, len(self.latitudes)) + self.assertEqual(validate.dimensions['lon'].size, len(self.longitudes)) self.assertEqual(validate.dimensions['bnds'].size, 2) def test_copy_dimensions_with_groups(self): test_target = self.test_file() - with Dataset(test_target, mode='w' ) as target_ds, \ - Dataset(self.test_IMERG_ncfile, mode='r') as source_ds: + with Dataset(test_target, mode='w') as target_ds, Dataset( + self.test_IMERG_ncfile, mode='r' + ) as source_ds: rs._copy_dimensions( {'/Grid/latv', '/Grid/lonv', '/Grid/nv', '/Grid/time'}, - source_ds, target_ds) + source_ds, + target_ds, + ) with Dataset(test_target, mode='r') as validate: self.assertTrue(validate['Grid'].dimensions['time'].isunlimited()) @@ -690,7 +720,8 @@ def test_horizontal_dims_for_variable_grouped(self): var_info = self.var_info(self.test_IMERG_ncfile) expected_dims = ('/Grid/lon', '/Grid/lat') actual_dims = rs._horizontal_dims_for_variable( - var_info, '/Grid/IRkalmanFilterWeight') + var_info, '/Grid/IRkalmanFilterWeight' + ) self.assertEqual(expected_dims, actual_dims) def test_horizontal_dims_for_variable(self): @@ -749,11 +780,12 @@ def test_unresampled_variables(self): var_info = self.var_info(self.test_ATL14_ncfile) expected_vars = { - '/Polar_Stereographic', '/orbit_info/bounding_polygon_dim1', + '/Polar_Stereographic', + '/orbit_info/bounding_polygon_dim1', '/orbit_info/bounding_polygon_lat1', '/orbit_info/bounding_polygon_lon1', '/quality_assessment/qa_granule_fail_reason', - '/quality_assessment/qa_granule_pass_fail' + '/quality_assessment/qa_granule_pass_fail', } actual_vars = rs._unresampled_variables(var_info) self.assertEqual(expected_vars, actual_vars) @@ -801,7 +833,7 @@ def test_validate_remaining_variables(self): test_vars = { 2: {'some', '2d', 'vars'}, 3: {'more', 'cubes'}, - 4: {'hypercube', 'data'} + 4: {'hypercube', 'data'}, } self.assertEqual(rs._validate_remaining_variables(test_vars), None) @@ -810,17 +842,25 @@ def test_validate_remaining_variables(self): 1: {'1d', 'should', 'have been', 'processed'}, 2: {'some', '2d', 'vars'}, 3: {'more', 'cubes'}, - 4: {'hypercube', 'data'} + 4: {'hypercube', 'data'}, } with self.assertRaisesRegex( - RegridderException, - 'Variables with dimensions.*cannot be handled.'): + RegridderException, 'Variables with dimensions.*cannot be handled.' + ): rs._validate_remaining_variables(test_vars) def test_integer_like(self): int_types = [ - np.byte, np.ubyte, np.short, np.ushort, np.intc, np.uintc, np.int_, - np.uint, np.longlong, np.ulonglong + np.byte, + np.ubyte, + np.short, + np.ushort, + np.intc, + np.uintc, + np.int_, + np.uint, + np.longlong, + np.ulonglong, ] other_types = [np.float16, np.float32, np.float64] @@ -836,8 +876,10 @@ def test_integer_like(self): with self.subTest('string'): self.assertFalse(rs._integer_like(str)) - @patch('harmony_regridding_service.regridding_service.AreaDefinition', - wraps=AreaDefinition) + @patch( + 'harmony_regridding_service.regridding_service.AreaDefinition', + wraps=AreaDefinition, + ) def test_compute_target_area(self, mock_area): """Ensure Area Definition correctly generated""" crs = '+proj=longlat +datum=WGS84 +no_defs +type=crs' @@ -846,25 +888,18 @@ def test_compute_target_area(self, mock_area): ymin = -90 ymax = 90 - message = Message({ - 'format': { - 'crs': crs, - 'scaleSize': { - 'x': 1.0, - 'y': 2.0 - }, - 'scaleExtent': { - 'x': { - 'min': xmin, - 'max': xmax + message = Message( + { + 'format': { + 'crs': crs, + 'scaleSize': {'x': 1.0, 'y': 2.0}, + 'scaleExtent': { + 'x': {'min': xmin, 'max': xmax}, + 'y': {'min': ymin, 'max': ymax}, }, - 'y': { - 'min': ymin, - 'max': ymax - } } } - }) + ) expected_height = 90 expected_width = 360 @@ -875,35 +910,36 @@ def test_compute_target_area(self, mock_area): self.assertEqual(actual_area.shape, (expected_height, expected_width)) self.assertEqual(actual_area.area_extent, (xmin, ymin, xmax, ymax)) self.assertEqual(actual_area.proj4_string, crs) - mock_area.assert_called_once_with('target_area_id', - 'target area definition', None, crs, - expected_width, expected_height, - (xmin, ymin, xmax, ymax)) + mock_area.assert_called_once_with( + 'target_area_id', + 'target area definition', + None, + crs, + expected_width, + expected_height, + (xmin, ymin, xmax, ymax), + ) def test_grid_height(self): with self.subTest('message with scale size'): expected_grid_height = 50 - actual_grid_height = rs._grid_height( - self.test_message_with_scale_size) + actual_grid_height = rs._grid_height(self.test_message_with_scale_size) self.assertEqual(expected_grid_height, actual_grid_height) with self.subTest('mesage includes height'): expected_grid_height = 80 - actual_grid_height = rs._grid_height( - self.test_message_with_height_width) + actual_grid_height = rs._grid_height(self.test_message_with_height_width) self.assertEqual(expected_grid_height, actual_grid_height) def test_grid_width(self): with self.subTest('message with scale size'): expected_grid_width = 100 - actual_grid_width = rs._grid_width( - self.test_message_with_scale_size) + actual_grid_width = rs._grid_width(self.test_message_with_scale_size) self.assertEqual(expected_grid_width, actual_grid_width) with self.subTest('message with width'): expected_grid_width = 40 - actual_grid_width = rs._grid_width( - self.test_message_with_height_width) + actual_grid_width = rs._grid_width(self.test_message_with_height_width) self.assertEqual(expected_grid_width, actual_grid_width) def test_compute_num_elements(self): @@ -912,24 +948,17 @@ def test_compute_num_elements(self): ymin = 0 ymax = 500 - message = Message({ - 'format': { - 'scaleSize': { - 'x': 10, - 'y': 10 - }, - 'scaleExtent': { - 'x': { - 'min': xmin, - 'max': xmax + message = Message( + { + 'format': { + 'scaleSize': {'x': 10, 'y': 10}, + 'scaleExtent': { + 'x': {'min': xmin, 'max': xmax}, + 'y': {'min': ymin, 'max': ymax}, }, - 'y': { - 'min': ymin, - 'max': ymax - } } } - }) + ) expected_x_elements = 100 expected_y_elements = 50 @@ -1021,8 +1050,7 @@ def test_compute_source_swath(self, mock_horiz_source_grids, mock_swath): rs._compute_source_swath(grid_dims, filepath, var_info) # horizontal grids were called successfully - mock_horiz_source_grids.assert_called_with(grid_dims, filepath, - var_info) + mock_horiz_source_grids.assert_called_with(grid_dims, filepath, var_info) # swath was called with the horizontal 2d grids. mock_swath.assert_called_with(lons=lons, lats=lats) @@ -1030,25 +1058,33 @@ def test_expected_result_compute_horizontal_source_grids(self): """Exercises the single function for computing horizontal grids.""" var_info = self.var_info(self.test_1D_dimensions_ncfile) - expected_longitudes = np.array([[-180, -80, -45, 45, 80, 180], - [-180, -80, -45, 45, 80, 180], - [-180, -80, -45, 45, 80, 180], - [-180, -80, -45, 45, 80, 180], - [-180, -80, -45, 45, 80, 180]]) - - expected_latitudes = np.array([[90, 90, 90, 90, 90, 90], - [45, 45, 45, 45, 45, 45], - [0, 0, 0, 0, 0, 0], - [-46, -46, -46, -46, -46, -46], - [-89, -89, -89, -89, -89, -89]]) + expected_longitudes = np.array( + [ + [-180, -80, -45, 45, 80, 180], + [-180, -80, -45, 45, 80, 180], + [-180, -80, -45, 45, 80, 180], + [-180, -80, -45, 45, 80, 180], + [-180, -80, -45, 45, 80, 180], + ] + ) + + expected_latitudes = np.array( + [ + [90, 90, 90, 90, 90, 90], + [45, 45, 45, 45, 45, 45], + [0, 0, 0, 0, 0, 0], + [-46, -46, -46, -46, -46, -46], + [-89, -89, -89, -89, -89, -89], + ] + ) test_args = [('/lon', '/lat'), ('/lat', '/lon')] for grid_dimensions in test_args: - with self.subTest( - f'independent grid_dimension order {grid_dimensions}'): + with self.subTest(f'independent grid_dimension order {grid_dimensions}'): longitudes, latitudes = rs._compute_horizontal_source_grids( - grid_dimensions, self.test_1D_dimensions_ncfile, var_info) + grid_dimensions, self.test_1D_dimensions_ncfile, var_info + ) np.testing.assert_array_equal(expected_latitudes, latitudes) np.testing.assert_array_equal(expected_longitudes, longitudes) @@ -1057,9 +1093,10 @@ def test_2D_lat_lon_input_compute_horizontal_source_grids(self): var_info = self.var_info(self.test_2D_dimensions_ncfile) grid_dimensions = ('/lat', '/lon') - expected_regex = re.escape('Incorrect source data dimensions. ' - 'rows:(6, 5), columns:(6, 5)') + expected_regex = re.escape( + 'Incorrect source data dimensions. ' 'rows:(6, 5), columns:(6, 5)' + ) with self.assertRaisesRegex(InvalidSourceDimensions, expected_regex): - rs._compute_horizontal_source_grids(grid_dimensions, - self.test_2D_dimensions_ncfile, - var_info) + rs._compute_horizontal_source_grids( + grid_dimensions, self.test_2D_dimensions_ncfile, var_info + ) diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 9334aea..34feb2a 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -12,31 +12,28 @@ class TestUtilities(TestCase): - """ A class testing the harmony_regridding_service.utilities module. """ + """A class testing the harmony_regridding_service.utilities module.""" def test_get_file_mime_type(self): - """ Ensure a MIME type can be retrieved from an input file path. """ + """Ensure a MIME type can be retrieved from an input file path.""" with self.subTest('File with MIME type known by Python.'): - self.assertEqual(get_file_mime_type('file.nc'), - 'application/x-netcdf') + self.assertEqual(get_file_mime_type('file.nc'), 'application/x-netcdf') with self.subTest('File with MIME type retrieved from dictionary.'): - self.assertEqual(get_file_mime_type('file.nc4'), - 'application/x-netcdf4') + self.assertEqual(get_file_mime_type('file.nc4'), 'application/x-netcdf4') with self.subTest('File with entirely unknown MIME type.'): self.assertIsNone(get_file_mime_type('file.xyzzyx')) with self.subTest('Upper case letters handled.'): - self.assertEqual(get_file_mime_type('file.HDF5'), - 'application/x-hdf5') + self.assertEqual(get_file_mime_type('file.HDF5'), 'application/x-hdf5') def test_has_valid_crs(self): - """ Ensure the function correctly determines if the input Harmony - message has a target Coordinate Reference System (CRS) that is - compatible with the service. Currently this is either to not - define the target CRS (assuming it to be geographic), or explicitly - requesting geographic CRS via EPSG code or proj4 string. + """Ensure the function correctly determines if the input Harmony + message has a target Coordinate Reference System (CRS) that is + compatible with the service. Currently this is either to not + define the target CRS (assuming it to be geographic), or explicitly + requesting geographic CRS via EPSG code or proj4 string. """ with self.subTest('format = None returns True'): @@ -73,14 +70,15 @@ def test_has_valid_crs(self): with self.assertRaises(InvalidTargetCRS) as context: has_valid_crs(test_message) - self.assertEqual(context.exception.message, - 'Target CRS not supported: "invalid CRS"') + self.assertEqual( + context.exception.message, 'Target CRS not supported: "invalid CRS"' + ) def test_is_geographic_crs(self): - """ Ensure function correctly determines if a supplied string resolves - to a `pyproj.CRS` object with a geographic Coordinate Reference - System (CRS). Exceptions arising from invalid CRS strings should - also be handled. + """Ensure function correctly determines if a supplied string resolves + to a `pyproj.CRS` object with a geographic Coordinate Reference + System (CRS). Exceptions arising from invalid CRS strings should + also be handled. """ with self.subTest('"EPSG:4326" returns True'): @@ -102,14 +100,15 @@ def test_is_geographic_crs(self): with self.assertRaises(InvalidTargetCRS) as context: _is_geographic_crs('invalid CRS') - self.assertEqual(context.exception.message, - 'Target CRS not supported: "invalid CRS"') + self.assertEqual( + context.exception.message, 'Target CRS not supported: "invalid CRS"' + ) def test_has_valid_interpolation(self): - """ Ensure that the function correctly determines if the supplied - Harmony message either omits the `format.interpolation` attribute, - or specifies EWA via a fully spelled-out string. The TRT-210 MVP - only allows for interpolation using EWA. + """Ensure that the function correctly determines if the supplied + Harmony message either omits the `format.interpolation` attribute, + or specifies EWA via a fully spelled-out string. The TRT-210 MVP + only allows for interpolation using EWA. """ with self.subTest('format = None returns True'): @@ -121,9 +120,9 @@ def test_has_valid_interpolation(self): self.assertTrue(has_valid_interpolation(test_message)) with self.subTest('EWA (spelled fully) returns True'): - test_message = Message({ - 'format': {'interpolation': 'Elliptical Weighted Averaging'} - }) + test_message = Message( + {'format': {'interpolation': 'Elliptical Weighted Averaging'}} + ) self.assertTrue(has_valid_interpolation(test_message)) with self.subTest('Unexpected interpolation returns False'): diff --git a/tests/utilities.py b/tests/utilities.py index a5e1da1..f4e3cf1 100644 --- a/tests/utilities.py +++ b/tests/utilities.py @@ -1,4 +1,5 @@ """ Utilities used to extend unittest capabilities. """ + from collections import namedtuple from datetime import datetime @@ -10,23 +11,28 @@ def create_stac(granule: Granule) -> Catalog: - """ Create a SpatioTemporal Asset Catalog (STAC). These are used as inputs - for Harmony requests, containing the URL and other information for - input granules. + """Create a SpatioTemporal Asset Catalog (STAC). These are used as inputs + for Harmony requests, containing the URL and other information for + input granules. - For simplicity the geometric and temporal properties of each item are - set to default values. + For simplicity the geometric and temporal properties of each item are + set to default values. """ catalog = Catalog(id='input catalog', description='test input') - item = Item(id='input granule', bbox=[-180, -90, 180, 90], - geometry=bbox_to_geometry([-180, -90, 180, 90]), - datetime=datetime(2020, 1, 1), properties=None) - - item.add_asset('input data', - Asset(granule.url, media_type=granule.media_type, - roles=granule.roles)) + item = Item( + id='input granule', + bbox=[-180, -90, 180, 90], + geometry=bbox_to_geometry([-180, -90, 180, 90]), + datetime=datetime(2020, 1, 1), + properties=None, + ) + + item.add_asset( + 'input data', + Asset(granule.url, media_type=granule.media_type, roles=granule.roles), + ) catalog.add_item(item) return catalog From 32dbee865c0b96c83fcf949cc4e6b13a0425a3ae Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Wed, 10 Apr 2024 14:09:06 -0400 Subject: [PATCH 3/4] IP-241 - Add .git-blame-ignore-revs. --- .git-blame-ignore-revs | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000..15ea35b --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,5 @@ +# For more information, see: +# https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view + +# Black code formatting of entire repository +2967ed10ef6e38d4346735c1693212a56c756902 From 48d218cb9592f5ee844d6760d3657956f6123dad Mon Sep 17 00:00:00 2001 From: Owen Littlejohns Date: Wed, 10 Apr 2024 14:10:14 -0400 Subject: [PATCH 4/4] IP-241 - Increment service for black formatting changes. --- CHANGELOG.md | 7 +++++++ docker/service_version.txt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4664eb..82f74b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.0.4 +### 2024-04-10 + +This version of the Harmony Regridding service implements `black` code +formatting across the entire repository. There should be no functional changes +to the service. + ## v0.0.3 ### 2023-12-18 diff --git a/docker/service_version.txt b/docker/service_version.txt index bcab45a..81340c7 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -0.0.3 +0.0.4