Skip to content

Commit

Permalink
chore(deps): Bump nixpkgs
Browse files Browse the repository at this point in the history
- Reintroduce now-working GitHub Workflows validation
- Use Python 3.12
- Bump all Python packages
- Work around argon2-cffi-bindings being incompatible with Python 3.12
  <nix-community/poetry2nix#1795>
- Patch sat-search and sat-stac for Python 3.12 compatibility
- Add Poetry packages to `python` with multiple `withPackages` calls to avoid
  collisions
- Avoid using very out of date sat-stac package
  • Loading branch information
l0b0 committed Sep 6, 2024
1 parent 5e652dd commit 0ea19e5
Show file tree
Hide file tree
Showing 11 changed files with 2,755 additions and 2,533 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ jobs:
- windows-2022
python:
- "3.9"
- "3.11"
- "3.12"
sentinel:
- "1"
- "2"
Expand Down Expand Up @@ -143,7 +143,7 @@ jobs:

- name: Get GDAL Python package version
run: |
GDAL_VERSION="$(grep --max-count=1 --only-matching 'file = "GDAL-.*\.tar\.gz"' poetry.lock)"
GDAL_VERSION="$(grep --max-count=1 --only-matching 'file = "gdal-.*\.tar\.gz"' poetry.lock)"
# Use character removal compatible with macOS 12
GDAL_VERSION="${GDAL_VERSION#?????????????}"
GDAL_VERSION="${GDAL_VERSION%????????}"
Expand Down
19 changes: 9 additions & 10 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,15 @@ repos:
language: system
stages: [commit]

# TODO: Reintroduce once https://github.com/NixOS/nixpkgs/pull/273831 is in use
# - id: check-github-workflows
# name: check-github-workflows
# description: Validate GitHub Workflows
# entry: check-jsonschema
# args: [--builtin-schema=vendor.github-workflows]
# files: ^\.github/workflows/[^/]+$
# types: [yaml]
# language: system
# stages: [commit]
- id: check-github-workflows
name: check-github-workflows
description: Validate GitHub Workflows
entry: check-jsonschema
args: [--builtin-schema=vendor.github-workflows]
files: ^\.github/workflows/[^/]+$
types: [yaml]
language: system
stages: [commit]

- id: deadnix
name: deadnix
Expand Down
1,726 changes: 897 additions & 829 deletions flooding/sentinel1_water_extraction/poetry.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions flooding/sentinel2_water_extraction/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ If you can't use Nix, you will need the following. For each prerequisite, the
- Linux (tested on Ubuntu 22.04, Jammy Jellyfish and 24.04, Noble Numbat)
- macOS (tested on version 12, Monterey)
- Windows (tested on Windows Server 2022)
- [Python](https://www.python.org/downloads/) (tested with 3.9 and 3.11)
- [Python](https://www.python.org/downloads/) (tested with 3.9 and 3.12)
- [Poetry](https://python-poetry.org/docs/#installation) (tested with 1.8.3)
- [GDAL](https://gdal.org/download.html) (tested with 3.6.4)
- [GDAL](https://gdal.org/download.html) (tested with 3.9.2)

Once you have all of the above,
`cd [emergency-management-tools path]/flooding/sentinel2_water_extraction` and
Expand Down
3,404 changes: 1,742 additions & 1,662 deletions flooding/sentinel2_water_extraction/poetry.lock

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions flooding/sentinel2_water_extraction/poetryPackages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,9 @@ poetry2nix.mkPoetryPackages {
path = ./.;
name = "sentinel2_water_extraction_packages";
};
overrides = poetry2nix.overrides.withDefaults (_final: prev: {
argon2-cffi-bindings = prev.argon2-cffi-bindings.override {
preferWheel = true;
};
});
}
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@
"# suppress warnings - raised by external rasterio library\n",
"import warnings\n",
"from datetime import date\n",
"from typing import Annotated, List, Optional, Tuple, TypedDict\n",
"from typing import Annotated, Any, List, Optional, Tuple, TypedDict\n",
"\n",
"import geopandas as gpd\n",
"import matplotlib.pyplot as plt\n",
Expand All @@ -108,7 +108,6 @@
"from rasterio.plot import show\n",
"from rio_tiler.io import COGReader\n",
"from satsearch import Search\n",
"from satstac import ItemCollection\n",
"from shapely.geometry import shape\n",
"from typing_extensions import NotRequired\n",
"\n",
Expand Down Expand Up @@ -268,7 +267,7 @@
"metadata": {},
"outputs": [],
"source": [
"def image_search(bbox: Annotated[List[float], 4], date_range: str) -> ItemCollection:\n",
"def image_search(bbox: Annotated[List[float], 4], date_range: str) -> Any:\n",
" \"\"\"Use SatSearch to find all Sentinel-2 images that meet our criteria.\"\"\"\n",
" # Note, we are not querying cloud cover and accepting\n",
" # all images irrelevant of cloud metadata.\n",
Expand All @@ -286,7 +285,7 @@
" print(\"NO IMAGE FOUND\")\n",
" sys.exit()\n",
"\n",
" return search.items()"
" return search.collections()"
]
},
{
Expand Down
3 changes: 1 addition & 2 deletions index.markdown
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,4 @@ layout: default

- [Radar Flood Water Extraction](https://github.com/linz/emergency-management-tools/tree/master/flooding/sentinel1_water_extraction)
- [Optical Flood Water Extraction](https://github.com/linz/emergency-management-tools/tree/master/flooding/sentinel2_water_extraction)
- [Convert date/time from machine-readable to
human-readable]({{ 'machine-readable-to-human-readable-date-time/' | relative_url }})
- [Convert date/time from machine-readable to human-readable]({{ 'machine-readable-to-human-readable-date-time/' | relative_url }})
5 changes: 3 additions & 2 deletions nixpkgs.nix
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import (
builtins.fetchTarball {
url = "https://github.com/nixos/nixpkgs/archive/2057814051972fa1453ddfb0d98badbea9b83c06.tar.gz";
sha256 = "1a8jafyawg8mysv787q0cwghkza9kahbbniism7v2rcxas89b575";
name = "nixos-unstable-2024-09-06";
url = "https://github.com/nixos/nixpkgs/archive/ad416d066ca1222956472ab7d0555a6946746a80.tar.gz";
sha256 = "0b9x5sghija7gy4lzf6l9s2kchdwj0zxs37ndlms2r8z1g3gwa7v";
}
) {}
95 changes: 85 additions & 10 deletions poetry2nix.nix
Original file line number Diff line number Diff line change
@@ -1,10 +1,85 @@
{pkgs}:
import (
builtins.fetchTarball {
name = "poetry2nix-2024.8.2346599";
url = "https://github.com/nix-community/poetry2nix/archive/7619e43c2b48c29e24b88a415256f09df96ec276.tar.gz";
sha256 = "10nf9213h7vkzrqhrgk88r5sd2rq19cvdpnn59f65ifak790sgc0";
}
) {
inherit pkgs;
}
{pkgs}: let
poetry2nix =
import (
builtins.fetchTarball {
url = "https://github.com/nix-community/poetry2nix/archive/2024.9.285936.tar.gz";
sha256 = "11a3wd1lgcq4qknh3almy9pcyqgm9f1q9iihdvy08g2hvmj2zris";
}
) {
inherit pkgs;
};
in
poetry2nix.overrideScope (_finalPoetry2nix: prevPoetry2nix: {
defaultPoetryOverrides = prevPoetry2nix.defaultPoetryOverrides.extend (final: prev: {
jupyter-server = prev.jupyter-server.overridePythonAttrs (old: {
# Already in poetry2nix but needed here because we're mixing old and new versions of jupyter-server
nativeBuildInputs = (old.nativeBuildInputs or []) ++ [final.jupyter-packaging];
});
sat-search = prev.sat-search.overridePythonAttrs (old: {
# https://github.com/sat-utils/sat-search/pull/136
patches =
(old.patches or [])
++ pkgs.lib.optionals (!(old.src.isWheel or false)) [
(pkgs.writeText "sat-search-importlib.patch" ''
--- a/setup.py
+++ b/setup.py
@@ -1,9 +1,20 @@
#!/usr/bin/env python
+import importlib.util
+import importlib.machinery
from setuptools import setup, find_packages
-from imp import load_source
from os import path
import io
+def load_source(modname, filename):
+ loader = importlib.machinery.SourceFileLoader(modname, filename)
+ spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+ module = importlib.util.module_from_spec(spec)
+ # The module is always executed and not cached in sys.modules.
+ # Uncomment the following line to cache the module.
+ # sys.modules[module.__name__] = module
+ loader.exec_module(module)
+ return module
+
__version__ = load_source('satsearch.version', 'satsearch/version.py').__version__
here = path.abspath(path.dirname(__file__))
'')
];
});
sat-stac = prev.sat-stac.overridePythonAttrs (old: {
# https://github.com/sat-utils/sat-stac/pull/75
patches =
(old.patches or [])
++ pkgs.lib.optionals (!(old.src.isWheel or false)) [
(pkgs.writeText "sat-stac-importlib.patch" ''
--- a/setup.py
+++ b/setup.py
@@ -1,9 +1,20 @@
#!/usr/bin/env python
+import importlib.util
+import importlib.machinery
from setuptools import setup, find_packages
-from imp import load_source
from os import path
import io
+def load_source(modname, filename):
+ loader = importlib.machinery.SourceFileLoader(modname, filename)
+ spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+ module = importlib.util.module_from_spec(spec)
+ # The module is always executed and not cached in sys.modules.
+ # Uncomment the following line to cache the module.
+ # sys.modules[module.__name__] = module
+ loader.exec_module(module)
+ return module
+
__version__ = load_source('satstac.version', 'satstac/version.py').__version__
here = path.abspath(path.dirname(__file__))
'')
];
});
});
})
16 changes: 6 additions & 10 deletions shell.nix
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,19 @@ let
};
pythonWithAllPackages =
(
python.withPackages (
ps:
[
(
python.withPackages (
ps: [
ps.mypy
ps.pip
ps.pylint
ps.types-python-dateutil
]
++ sentinel1WaterExtractionPoetryPackages.poetryPackages
++ sentinel2WaterExtractionPoetryPackages.poetryPackages
)
)
.withPackages (_ps: sentinel1WaterExtractionPoetryPackages.poetryPackages)
)
.override (
_args: {
ignoreCollisions = true;
}
);
.withPackages (_ps: sentinel2WaterExtractionPoetryPackages.poetryPackages);
in
pkgs.mkShell {
packages = [
Expand Down

0 comments on commit 0ea19e5

Please sign in to comment.