Updates after local testing

NKI-AI · Aug 7, 2024 · 4e17a83 · 4e17a83
1 parent 09eba57
commit 4e17a83
Show file tree

Hide file tree

Showing 10 changed files with 123 additions and 79 deletions.
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
@@ -1,4 +1,3 @@
-
 name: Black
 on:
   push:
@@ -13,3 +12,4 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
       - uses: psf/black@stable
+
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -10,20 +10,33 @@ jobs:
       matrix:
         python-version: ["3.10"]
     steps:
-    - name: Install minimal dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y libopenslide0 libgeos-dev libvips42 libtiff5-dev
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install build dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+        sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev
+    - name: Build and install OpenSlide
+      run: |
+        git clone https://github.com/openslide/openslide.git
+        cd openslide
+        meson setup builddir
+        meson compile -C builddir
+        sudo meson install -C builddir
+        cd ..
+    - name: Build and install libvips
       run: |
-        python -m pip install --upgrade pip setuptools wheel coverage scikit-build Cython pybind11 numpy
-        python setup.py build_ext --inplace
-        python -m pip install -e ".[dev]"
+        git clone https://github.com/libvips/libvips.git
+        cd libvips
+        meson setup builddir --prefix=/usr/local
+        meson compile -C builddir
+        sudo meson install -C builddir
+        sudo ldconfig
+        cd ..
     - name: Run Coverage
       run: |
         coverage run -m pytest

diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -10,21 +10,33 @@ jobs:
     runs-on: ubuntu-latest
     name: mypy
     steps:
-    -   name: Install minimal dependencies
-        run: |
-            sudo apt install -y libgeos-dev libtiff5-dev
     - uses: actions/checkout@v4
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
         python-version: "3.10"
+    - name: Install Rust for pyhaloxml minimally
+      run: |
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+        export PATH="$HOME/.cargo/bin:$PATH"
+        rustup toolchain install stable
+    - name: Get pybind11 CMake path
+      id: get-pybind11-path
+      run: |
+        python -m pip install pybind11
+        echo "PYBIND11_CMAKE_PATH=$(python -c 'import pybind11; import os; print(os.path.join(pybind11.get_cmake_dir(), "pybind11Config.cmake"))')" >> $GITHUB_ENV
+    - name: Set CMAKE_PREFIX_PATH for pybind11
+      run: |
+        echo "CMAKE_PREFIX_PATH=$(dirname ${PYBIND11_CMAKE_PATH})" >> $GITHUB_ENV
     - name: Install Dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install mypy scikit-build
+        python -m pip install mypy numpy==1.26.4 Cython pybind11
+        python setup.py build_ext --inplace
         python -m pip install -e ".[dev]"
         python -m pip install pyhaloxml
         python -m pip install darwin-py
     - name: mypy
       run: |
-        mypy .
+        mypy .
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -10,9 +10,29 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - name: Check out repository
-      uses: actions/checkout@v4
-
+    - name: Install build dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+        sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev
+    - name: Build and install OpenSlide
+      run: |
+        git clone https://github.com/openslide/openslide.git
+        cd openslide
+        meson setup builddir
+        meson compile -C builddir
+        sudo meson install -C builddir
+        cd ..
+    - name: Build and install libvips
+      run: |
+        git clone https://github.com/libvips/libvips.git
+        cd libvips
+        meson setup builddir --prefix=/usr/local
+        meson compile -C builddir
+        sudo meson install -C builddir
+        sudo ldconfig
+        cd ..
+    - uses: actions/checkout@v4
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:

diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml
@@ -5,61 +5,53 @@ on:
     branches:
       - main
   pull_request:
-
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.10", "3.11"]
-
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install system dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libopenslide0 libgeos-dev libvips42 libtiff5-dev pkg-config cmake ninja-build
-
-    - name: Cache pip
-      uses: actions/cache@v3
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py', '**/pyproject.toml') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-
-    - name: Install Python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install tox tox-gh-actions
-
-    - name: Install build dependencies
-      run: |
-        pip install setuptools wheel scikit-build cmake ninja Cython numpy==1.26.4 pybind11
-
-    - name: Install project dependencies
-      run: |
-        pip install -e .[dev]
-
-    - name: Build package
-      run: |
-        python setup.py build_ext --inplace -- -DCMAKE_VERBOSE_MAKEFILE=ON
-        pip install -e .
-
-    - name: Run tests with tox
-      run: tox -vv
-
-    - name: Upload test results
-      uses: actions/upload-artifact@v3
-      if: failure()
-      with:
-        name: test-results
-        path: |
-          .tox/**/*.log
-          _skbuild/**/*.log
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install build dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+          sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev
+          python -m pip install scikit-build cmake ninja Cython pybind11 numpy==1.26.4
+      - name: Build and install OpenSlide
+        run: |
+          cd /tmp
+          git clone https://github.com/openslide/openslide.git
+          cd openslide
+          meson setup builddir
+          meson compile -C builddir
+          sudo meson install -C builddir
+      - name: Build and install libvips
+        run: |
+          cd /tmp
+          git clone https://github.com/libvips/libvips.git
+          cd libvips
+          meson setup builddir --prefix=/usr/local
+          meson compile -C builddir
+          sudo meson install -C builddir
+          sudo ldconfig
+      - name: Get pybind11 CMake path
+        id: get-pybind11-path
+        run: |
+          python -m pip install pybind11
+          echo "PYBIND11_CMAKE_PATH=$(python -c 'import pybind11; import os; print(os.path.join(pybind11.get_cmake_dir(), "pybind11Config.cmake"))')" >> $GITHUB_ENV
+      - name: Set CMAKE_PREFIX_PATH for pybind11
+        run: |
+          echo "CMAKE_PREFIX_PATH=$(dirname ${PYBIND11_CMAKE_PATH})" >> $GITHUB_ENV
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox tox-gh-actions
+          pip install -e .
+      - name: Test with tox
+        run: tox
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,12 +1,14 @@
 cmake_minimum_required(VERSION 3.12)
+cmake_policy(SET CMP0148 NEW)
 project(dlup)
 
 set(CMAKE_CXX_STANDARD 17)
 
-find_package(PythonExtensions REQUIRED)
+find_package(Python COMPONENTS Interpreter Development REQUIRED)
 find_package(Cython REQUIRED)
-find_package(pybind11 CONFIG REQUIRED)
 
+set(PYBIND11_FINDPYTHON ON)
+find_package(pybind11 CONFIG REQUIRED)
 
 pybind11_add_module(_libtiff_tiff_writer src/libtiff_tiff_writer.cpp)
 install(TARGETS _libtiff_tiff_writer DESTINATION .)
diff --git a/setup.py b/setup.py
@@ -26,7 +26,7 @@
     "tqdm>=2.66.4",
     "pillow>=10.3.0",
     "openslide-python>=1.3.1",
-    "opencv-python>=4.9.0.80",
+    "opencv-python-headless>=4.9.0.80",
     "shapely>=2.0.4",
     "packaging>=24.0",
     "pybind11>=2.8.0",
@@ -59,6 +59,12 @@ def get_ext_modules() -> Extension:
     cmake_install_dir="dlup",
     cmake_with_sdist=True,
     cmake_languages=("C", "CXX"),
+    cmake_args=[
+        "-DCMAKE_BUILD_TYPE=Release",
+        "-DCMAKE_C_FLAGS=-O3 -march=native -mtune=native",
+        "-DCMAKE_CXX_FLAGS=-O3 -march=native -mtune=native",
+        "-DPYBIND11_FINDPYTHON=ON",
+    ],
     python_requires=">=3.10",
     install_requires=install_requires,
     extras_require={

diff --git a/tests/common.py b/tests/common.py
@@ -65,7 +65,7 @@ def get_sample_nonuniform_image(size: tuple[int, int] = (256, 256), divisions: i
     cell_height = height // y_divisions
 
     # Create an array to store the image
-    image_array = np.zeros((height, width, 4), dtype=np.uint8)
+    image_array = np.zeros((height, width, 4), dtype=float)
 
     # Define a set of distinct colors
     color_palette = [
@@ -105,4 +105,4 @@ def get_sample_nonuniform_image(size: tuple[int, int] = (256, 256), divisions: i
     for k in range(3):  # Apply only to RGB channels, not alpha
         image_array[:, :, k] = image_array[:, :, k] * sine_wave
 
-    return pyvips.Image.new_from_array(image_array)
+    return pyvips.Image.new_from_array(image_array.astype(np.uint8))
diff --git a/tests/test_writers.py b/tests/test_writers.py
@@ -6,7 +6,6 @@
 import openslide
 import pytest
 import pyvips
-from packaging.version import Version
 from PIL import Image, ImageColor
 
 from dlup import SlideImage
@@ -72,10 +71,10 @@ def test_tiff_writer(self, shape, target_mpp):
                 assert slide0._loader == "tiffload"
                 assert slide1._loader == "openslideload"
 
-                if Version(openslide.__library_version__) < Version("4.0.0"):
-                    warnings.warn("Openslide version is too old, skipping some tests.")
-                else:
-                    assert np.allclose(slide0.spacing, slide1.spacing)
+                if not slide1.spacing:
+                    slide1.spacing = slide0.spacing
+
+                assert np.allclose(slide0.spacing, slide1.spacing)
                 assert slide0.level_count == slide1.level_count
                 assert slide0.dimensions == slide1.dimensions
                 assert np.allclose(slide0.level_downsamples, slide1.level_downsamples)

diff --git a/tox.ini b/tox.ini
@@ -4,7 +4,7 @@ isolated_build = True
 
 [testenv]
 deps =
-    numpy
+    numpy==1.26.4
     Cython>=0.29
     scikit-build
 extras = dev,darwin