chore(deps): Bump nixpkgs

- Reintroduce now-working GitHub Workflows validation - Use Python 3.12 - Bump all Python packages - Work around argon2-cffi-bindings being incompatible with Python 3.12 <nix-community/poetry2nix#1795> - Patch sat-search and sat-stac for Python 3.12 compatibility - Add Poetry packages to `python` with multiple `withPackages` calls to avoid collisions - Avoid using very out of date sat-stac package
linz · Sep 6, 2024 · 335deb1 · 335deb1
1 parent 5e652dd
commit 335deb1
Show file tree

Hide file tree

Showing 11 changed files with 2,754 additions and 2,532 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -105,7 +105,7 @@ jobs:
           - windows-2022
         python:
           - "3.9"
-          - "3.11"
+          - "3.12"
         sentinel:
           - "1"
           - "2"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,16 +11,15 @@ repos:
         language: system
         stages: [commit]
 
-      # TODO: Reintroduce once https://github.com/NixOS/nixpkgs/pull/273831 is in use
-      #      - id: check-github-workflows
-      #        name: check-github-workflows
-      #        description: Validate GitHub Workflows
-      #        entry: check-jsonschema
-      #        args: [--builtin-schema=vendor.github-workflows]
-      #        files: ^\.github/workflows/[^/]+$
-      #        types: [yaml]
-      #        language: system
-      #        stages: [commit]
+      - id: check-github-workflows
+        name: check-github-workflows
+        description: Validate GitHub Workflows
+        entry: check-jsonschema
+        args: [--builtin-schema=vendor.github-workflows]
+        files: ^\.github/workflows/[^/]+$
+        types: [yaml]
+        language: system
+        stages: [commit]
 
       - id: deadnix
         name: deadnix

diff --git a/flooding/sentinel1_water_extraction/poetry.lock b/flooding/sentinel1_water_extraction/poetry.lock
diff --git a/flooding/sentinel2_water_extraction/README.md b/flooding/sentinel2_water_extraction/README.md
@@ -40,9 +40,9 @@ If you can't use Nix, you will need the following. For each prerequisite, the
    -  Linux (tested on Ubuntu 22.04, Jammy Jellyfish and 24.04, Noble Numbat)
    -  macOS (tested on version 12, Monterey)
    -  Windows (tested on Windows Server 2022)
--  [Python](https://www.python.org/downloads/) (tested with 3.9 and 3.11)
+-  [Python](https://www.python.org/downloads/) (tested with 3.9 and 3.12)
 -  [Poetry](https://python-poetry.org/docs/#installation) (tested with 1.8.3)
--  [GDAL](https://gdal.org/download.html) (tested with 3.6.4)
+-  [GDAL](https://gdal.org/download.html) (tested with 3.9.2)
 
 Once you have all of the above,
 `cd [emergency-management-tools path]/flooding/sentinel2_water_extraction` and

diff --git a/flooding/sentinel2_water_extraction/poetry.lock b/flooding/sentinel2_water_extraction/poetry.lock
diff --git a/flooding/sentinel2_water_extraction/poetryPackages.nix b/flooding/sentinel2_water_extraction/poetryPackages.nix
@@ -8,4 +8,9 @@ poetry2nix.mkPoetryPackages {
     path = ./.;
     name = "sentinel2_water_extraction_packages";
   };
+  overrides = poetry2nix.overrides.withDefaults (_final: prev: {
+    argon2-cffi-bindings = prev.argon2-cffi-bindings.override {
+      preferWheel = true;
+    };
+  });
 }
diff --git a/flooding/sentinel2_water_extraction/sentinel2_water_extraction.ipynb b/flooding/sentinel2_water_extraction/sentinel2_water_extraction.ipynb
@@ -96,7 +96,7 @@
     "# suppress warnings - raised by external rasterio library\n",
     "import warnings\n",
     "from datetime import date\n",
-    "from typing import Annotated, List, Optional, Tuple, TypedDict\n",
+    "from typing import Annotated, Any, List, Optional, Tuple, TypedDict\n",
     "\n",
     "import geopandas as gpd\n",
     "import matplotlib.pyplot as plt\n",
@@ -108,7 +108,6 @@
     "from rasterio.plot import show\n",
     "from rio_tiler.io import COGReader\n",
     "from satsearch import Search\n",
-    "from satstac import ItemCollection\n",
     "from shapely.geometry import shape\n",
     "from typing_extensions import NotRequired\n",
     "\n",
@@ -268,7 +267,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def image_search(bbox: Annotated[List[float], 4], date_range: str) -> ItemCollection:\n",
+    "def image_search(bbox: Annotated[List[float], 4], date_range: str) -> Any:\n",
     "    \"\"\"Use SatSearch to find all Sentinel-2 images that meet our criteria.\"\"\"\n",
     "    # Note, we are not querying cloud cover and accepting\n",
     "    # all images irrelevant of cloud metadata.\n",
@@ -286,7 +285,7 @@
     "        print(\"NO IMAGE FOUND\")\n",
     "        sys.exit()\n",
     "\n",
-    "    return search.items()"
+    "    return search.collections()"
    ]
   },
   {

diff --git a/index.markdown b/index.markdown
@@ -4,5 +4,4 @@ layout: default
 
 -   [Radar Flood Water Extraction](https://github.com/linz/emergency-management-tools/tree/master/flooding/sentinel1_water_extraction)
 -   [Optical Flood Water Extraction](https://github.com/linz/emergency-management-tools/tree/master/flooding/sentinel2_water_extraction)
--   [Convert date/time from machine-readable to
-    human-readable]({{ 'machine-readable-to-human-readable-date-time/' | relative_url }})
+-   [Convert date/time from machine-readable to human-readable]({{ 'machine-readable-to-human-readable-date-time/' | relative_url }})
diff --git a/nixpkgs.nix b/nixpkgs.nix
@@ -1,6 +1,7 @@
 import (
   builtins.fetchTarball {
-    url = "https://github.com/nixos/nixpkgs/archive/2057814051972fa1453ddfb0d98badbea9b83c06.tar.gz";
-    sha256 = "1a8jafyawg8mysv787q0cwghkza9kahbbniism7v2rcxas89b575";
+    name = "nixos-unstable-2024-09-06";
+    url = "https://github.com/nixos/nixpkgs/archive/ad416d066ca1222956472ab7d0555a6946746a80.tar.gz";
+    sha256 = "0b9x5sghija7gy4lzf6l9s2kchdwj0zxs37ndlms2r8z1g3gwa7v";
   }
 ) {}
diff --git a/poetry2nix.nix b/poetry2nix.nix
@@ -1,10 +1,85 @@
-{pkgs}:
-import (
-  builtins.fetchTarball {
-    name = "poetry2nix-2024.8.2346599";
-    url = "https://github.com/nix-community/poetry2nix/archive/7619e43c2b48c29e24b88a415256f09df96ec276.tar.gz";
-    sha256 = "10nf9213h7vkzrqhrgk88r5sd2rq19cvdpnn59f65ifak790sgc0";
-  }
-) {
-  inherit pkgs;
-}
+{pkgs}: let
+  poetry2nix =
+    import (
+      builtins.fetchTarball {
+        url = "https://github.com/nix-community/poetry2nix/archive/2024.9.285936.tar.gz";
+        sha256 = "11a3wd1lgcq4qknh3almy9pcyqgm9f1q9iihdvy08g2hvmj2zris";
+      }
+    ) {
+      inherit pkgs;
+    };
+in
+  poetry2nix.overrideScope (_finalPoetry2nix: prevPoetry2nix: {
+    defaultPoetryOverrides = prevPoetry2nix.defaultPoetryOverrides.extend (final: prev: {
+      jupyter-server = prev.jupyter-server.overridePythonAttrs (old: {
+        # Already in poetry2nix but needed here because we're mixing old and new versions of jupyter-server
+        nativeBuildInputs = (old.nativeBuildInputs or []) ++ [final.jupyter-packaging];
+      });
+      sat-search = prev.sat-search.overridePythonAttrs (old: {
+        # https://github.com/sat-utils/sat-search/pull/136
+        patches =
+          (old.patches or [])
+          ++ pkgs.lib.optionals (!(old.src.isWheel or false)) [
+            (pkgs.writeText "sat-search-importlib.patch" ''
+              --- a/setup.py
+              +++ b/setup.py
+              @@ -1,9 +1,20 @@
+               #!/usr/bin/env python
+              +import importlib.util
+              +import importlib.machinery
+               from setuptools import setup, find_packages
+              -from imp import load_source
+               from os import path
+               import io
+
+              +def load_source(modname, filename):
+              +    loader = importlib.machinery.SourceFileLoader(modname, filename)
+              +    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+              +    module = importlib.util.module_from_spec(spec)
+              +    # The module is always executed and not cached in sys.modules.
+              +    # Uncomment the following line to cache the module.
+              +    # sys.modules[module.__name__] = module
+              +    loader.exec_module(module)
+              +    return module
+              +
+               __version__ = load_source('satsearch.version', 'satsearch/version.py').__version__
+
+               here = path.abspath(path.dirname(__file__))
+            '')
+          ];
+      });
+      sat-stac = prev.sat-stac.overridePythonAttrs (old: {
+        # https://github.com/sat-utils/sat-stac/pull/75
+        patches =
+          (old.patches or [])
+          ++ pkgs.lib.optionals (!(old.src.isWheel or false)) [
+            (pkgs.writeText "sat-stac-importlib.patch" ''
+              --- a/setup.py
+              +++ b/setup.py
+              @@ -1,9 +1,20 @@
+               #!/usr/bin/env python
+              +import importlib.util
+              +import importlib.machinery
+               from setuptools import setup, find_packages
+              -from imp import load_source
+               from os import path
+               import io
+
+              +def load_source(modname, filename):
+              +    loader = importlib.machinery.SourceFileLoader(modname, filename)
+              +    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+              +    module = importlib.util.module_from_spec(spec)
+              +    # The module is always executed and not cached in sys.modules.
+              +    # Uncomment the following line to cache the module.
+              +    # sys.modules[module.__name__] = module
+              +    loader.exec_module(module)
+              +    return module
+              +
+               __version__ = load_source('satstac.version', 'satstac/version.py').__version__
+
+               here = path.abspath(path.dirname(__file__))
+            '')
+          ];
+      });
+    });
+  })
diff --git a/shell.nix b/shell.nix
@@ -14,23 +14,19 @@ let
   };
   pythonWithAllPackages =
     (
-      python.withPackages (
-        ps:
-          [
+      (
+        python.withPackages (
+          ps: [
             ps.mypy
             ps.pip
             ps.pylint
             ps.types-python-dateutil
           ]
-          ++ sentinel1WaterExtractionPoetryPackages.poetryPackages
-          ++ sentinel2WaterExtractionPoetryPackages.poetryPackages
+        )
       )
+      .withPackages (_ps: sentinel1WaterExtractionPoetryPackages.poetryPackages)
     )
-    .override (
-      _args: {
-        ignoreCollisions = true;
-      }
-    );
+    .withPackages (_ps: sentinel2WaterExtractionPoetryPackages.poetryPackages);
 in
   pkgs.mkShell {
     packages = [