From 42b32400217ac794424d72422bb136f2c0667a36 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Thu, 20 Jun 2024 14:41:57 -0600 Subject: [PATCH] DAS-2180: Remove Conda dependency on docker images (#20) --- CHANGELOG.md | 8 ++++++- README.md | 16 +++++++------ conda_requirements.txt | 1 - docker/service.Dockerfile | 41 +++++++++------------------------- docker/service_version.txt | 2 +- docker/tests.Dockerfile | 4 +++- pip_requirements_skip_snyk.txt | 3 +++ 7 files changed, 34 insertions(+), 41 deletions(-) delete mode 100644 conda_requirements.txt create mode 100644 pip_requirements_skip_snyk.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index cd91741..baf9a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ HyBIG follows semantic versioning. All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## [v1.2.2] - 2024-06-18 + +### Changed +Removes internal dependency on conda. + ## [v1.2.1] - 2024-06-10 ### Changed @@ -47,7 +52,8 @@ outlined by the NASA open-source guidelines. For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/1.2.1..HEAD +[unreleased]:https://github.com/nasa/harmony-browse-image-generator/compare/1.2.2..HEAD +[v1.2.2]: https://github.com/nasa/harmony-browse-image-generator/compare/1.2.1..1.2.2 [v1.2.1]: https://github.com/nasa/harmony-browse-image-generator/compare/1.2.0..1.2.1 [v1.2.0]: https://github.com/nasa/harmony-browse-image-generator/compare/1.1.0..1.2.0 [v1.1.0]: https://github.com/nasa/harmony-browse-image-generator/compare/1.0.2..1.1.0 diff --git a/README.md b/README.md index 1255e5d..962faa8 100644 --- a/README.md +++ b/README.md @@ -138,10 +138,10 @@ also with units of degrees. |- CONTRIBUTING.md |- LICENSE |- README.md -|- conda_requirements.txt |- dev-requirements.txt |- legacy-CHANGELOG.md |- pip_requirements.txt +|- pip_requirements_skip_snyk.txt ``` * `bin` - A directory containing utility scripts to build the service and test @@ -173,9 +173,6 @@ also with units of degrees. * `README.md` - This file, containing guidance on developing the service. -* `conda_requirements.txt` - A list of service dependencies, such as GDAL, that - cannot be installed via Pip. - * `dev-requirements.txt` - list of packages required for service development. * `legacy-CHANGELOG.md` - Notes for each version that was previously released @@ -184,6 +181,12 @@ also with units of degrees. * `pip_requirements.txt` - A list of service Python package dependencies. +* `pip_requirements_skip_snyk.txt` - A list of service Python package + dependencies that are not scanned by snyk for vulnerabilities. This file + contains only the `GDAL` package. It is separated because snyk's scanning is + naive and cannot pre-install required libraries so that `pip install GDAL` + fails and we have no work around. + ## Local development: @@ -193,13 +196,12 @@ regarding creation of a local Harmony instance. If testing small functions locally that do not require inputs from the main Harmony application, it is recommended that you create a Python virtual -environment via conda, and then install the necessary dependencies for the +environment, and then install the necessary dependencies for the service within that environment via conda and pip then install the pre-commit hooks. ``` > conda create --name hybig-env python==3.11 -> conda install --file conda_requirements.txt -> pip install -r pip_requirements.txt +> pip install -r pip_requirements.txt -r pip_requirements_skip_snyk.txt > pip install -r dev-requirements.txt > pre-commit install diff --git a/conda_requirements.txt b/conda_requirements.txt deleted file mode 100644 index fc2f8ef..0000000 --- a/conda_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -gdal==3.9 diff --git a/docker/service.Dockerfile b/docker/service.Dockerfile index 19e93ea..cad007d 100644 --- a/docker/service.Dockerfile +++ b/docker/service.Dockerfile @@ -10,47 +10,28 @@ # 2023-01-26: Updated for the Harmony Regridding Service. # 2023-04-04: Updated for HyBIG. # 2023-04-23: Updated conda clean and pip install to keep Docker image slim. +# 2024-06-18: Updates to remove conda dependency. # ############################################################################### -FROM continuumio/miniconda3 +FROM python:3.11 WORKDIR "/home" -# Create Conda environment: -COPY conda_requirements.txt conda_requirements.txt -RUN conda create --yes --name hybig --file conda_requirements.txt \ - python=3.11 --channel conda-forge --channel defaults -q && \ - conda clean --all --force-pkgs-dirs --yes +RUN apt-get update +RUN apt-get install -y libgdal-dev -# Install additional Pip dependencies -COPY pip_requirements.txt pip_requirements.txt -RUN conda run --name hybig pip install --no-input --no-cache-dir \ - -r pip_requirements.txt +# Install Pip dependencies +COPY pip_requirements*.txt /home/ + +RUN pip install --no-input --no-cache-dir \ + -r pip_requirements.txt \ + -r pip_requirements_skip_snyk.txt # Copy service code. COPY ./harmony_browse_image_generator harmony_browse_image_generator -# Set conda environment for HyBIG, as `conda run` will not stream logging. -# Setting these environment variables is the equivalent of `conda activate`. -ENV _CE_CONDA='' \ - _CE_M='' \ - CONDA_DEFAULT_ENV=hybig \ - CONDA_EXE=/opt/conda/bin/conda \ - CONDA_PREFIX=/opt/conda/envs/hybig \ - CONDA_PREFIX_1=/opt/conda \ - CONDA_PROMPT_MODIFIER=(hybig) \ - CONDA_PYTHON_EXE=/opt/conda/hybig/python \ - CONDA_ROOT=/opt/conda \ - CONDA_SHLVL=2 \ - PATH="/opt/conda/envs/hybig/bin:${PATH}" \ - SHLVL=1 - # Set GDAL related environment variables. -ENV CPL_ZIP_ENCODING=UTF-8 \ - GDAL_DATA=/opt/conda/envs/hybig/share/gdal \ - GSETTINGS_SCHEMA_DIR=/opt/conda/envs/hybig/share/glib-2.0/schemas \ - GSETTINGS_SCHEMA_DIR_CONDA_BACKUP='' \ - PROJ_LIB=/opt/conda/envs/hybig/share/proj +ENV CPL_ZIP_ENCODING=UTF-8 # Configure a container to be executable via the `docker run` command. ENTRYPOINT ["python", "-m", "harmony_browse_image_generator"] diff --git a/docker/service_version.txt b/docker/service_version.txt index 6085e94..23aa839 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.2.1 +1.2.2 diff --git a/docker/tests.Dockerfile b/docker/tests.Dockerfile index 915a479..22857b0 100644 --- a/docker/tests.Dockerfile +++ b/docker/tests.Dockerfile @@ -9,13 +9,15 @@ # 2023-04-04: Updated for HyBIG. # 2023-04-17: Added --no-cache-dir to keep Docker images slim. # 2024-01-22: Updated to use new open-source service image name. +# 2024-06-18: Removes Conda. # ############################################################################### FROM ghcr.io/nasa/harmony-browse-image-generator # Install additional Pip requirements (for testing) COPY tests/pip_test_requirements.txt . -RUN conda run --name hybig pip install --no-input --no-cache-dir \ + +RUN pip install --no-input --no-cache-dir \ -r pip_test_requirements.txt # Copy test directory containing Python unittest suite, test data and utilities diff --git a/pip_requirements_skip_snyk.txt b/pip_requirements_skip_snyk.txt new file mode 100644 index 0000000..b4a7e1d --- /dev/null +++ b/pip_requirements_skip_snyk.txt @@ -0,0 +1,3 @@ +# Because snyk can't install gdal properly during automated scans, we have this +# file that includes the GDAL package alone which is not scanned by snyk +GDAL==3.6.2