Feature/c tiffwriter (#249)

* Added C++ TIFF Writer * Bump version to 0.7 * Rebuild setup toolchain to pyproject.toml * Remove CLI utilities and tests * Rename dlup.types to dlup._types * Reduce cyclic imports * Remove color transform from internal_handler='pil' (working towards v1.0) * Bugfixes
NKI-AI · Aug 12, 2024 · 548fc87 · 548fc87
1 parent 3a6a10f
commit 548fc87
Show file tree

Hide file tree

Showing 51 changed files with 1,518 additions and 712 deletions.
diff --git a/.clang-format b/.clang-format
@@ -0,0 +1,114 @@
+---
+Language: Cpp
+ColumnLimit: 120
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands: true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass: false
+  AfterControlStatement: false
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterObjCDeclaration: false
+  AfterStruct: false
+  AfterUnion: false
+  AfterExternBlock: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat: false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks: Preserve
+IncludeCategories:
+  - Regex: '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority: 2
+  - Regex: '^(<|"(gtest|gmock|isl|json)/)'
+    Priority: 3
+  - Regex: '.*'
+    Priority: 1
+IncludeIsMainRegex: '(Test)?$'
+IndentCaseLabels: false
+IndentPPDirectives: None
+IndentWidth: 4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd: ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments: true
+SortIncludes: true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard: Cpp11
+TabWidth: 4
+UseTab: Never
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
@@ -1,4 +1,3 @@
-
 name: Black
 on:
   push:

diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -1,30 +1,67 @@
 name: CodeCov
 on:
-  - push
-  - pull_request
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10"]
+    env:
+      CODECOV_CI: true
     steps:
-    - name: Install minimal dependencies
+    - name: Install build dependencies
       run: |
         sudo apt update
-        sudo apt install -y libopenslide0 libgeos-dev libvips42
+        sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+        sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev ninja-build
+    - name: Build and install OpenSlide
+      run: |
+        git clone https://github.com/openslide/openslide.git
+        cd openslide
+        meson setup builddir
+        meson compile -C builddir
+        sudo meson install -C builddir
+        cd ..
+    - name: Build and install libvips
+      run: |
+        git clone https://github.com/libvips/libvips.git
+        cd libvips
+        meson setup builddir --prefix=/usr/local
+        meson compile -C builddir
+        sudo meson install -C builddir
+        sudo ldconfig
+        cd ..
     - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+        python-version: "3.10"
+    - name: Clean up any existing installations
+      run: |
+        sudo rm -rf /usr/local/lib/python3.10/site-packages/dlup*
+        sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/dlup*
+        sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/_dlup_editable_loader.py
+        sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/easy-install.pth
+        sudo rm -rf dlup/build
+        sudo rm -rf /tmp/*
+    - name: Install environment
       run: |
-        python -m pip install --upgrade pip setuptools wheel coverage
-        python -m pip install -e ".[dev]"
-    - name: Run Coverage
+        python -m pip install --upgrade pip
+        python -m pip install ninja meson meson-python>=0.15.0 numpy==1.26.4 Cython>=0.29 spin pybind11
+        python -m pip install tifffile>=2024.7.2 pyvips>=2.2.3 tqdm>=2.66.4 pillow>=10.3.0 openslide-python>=1.3.1
+        python -m pip install opencv-python-headless>=4.9.0.80 shapely>=2.0.4 pybind11>=2.8.0 pydantic coverage pytest psutil darwin-py pytest-mock
+        echo "Python executable: $(which python)"
+        echo "Python version: $(python --version)"
+        echo "Current directory: $PWD"
+        meson setup builddir
+        meson compile -C builddir
+        meson install -C builddir
+    - name: Run coverage
       run: |
+        mv dlup _dlup # This is needed because otherwise it won't find the compiled libraries
+        export PYTHONPATH=$(python -c "import site; print(site.getsitepackages()[0])")
         coverage run -m pytest
     - name: Upload Coverage to Codecov
       uses: codecov/codecov-action@v4

diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -8,23 +8,47 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: mypy
     steps:
-    -   name: Install minimal dependencies
-        run: |
-            sudo apt install -y libgeos-dev
+    - name: Install build dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+        sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev
+    - name: Build and install OpenSlide
+      run: |
+        git clone https://github.com/openslide/openslide.git
+        cd openslide
+        meson setup builddir
+        meson compile -C builddir
+        sudo meson install -C builddir
+        cd ..
+    - name: Build and install libvips
+      run: |
+        git clone https://github.com/libvips/libvips.git
+        cd libvips
+        meson setup builddir --prefix=/usr/local
+        meson compile -C builddir
+        sudo meson install -C builddir
+        sudo ldconfig
+        cd ..
     - uses: actions/checkout@v4
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - name: Install Dependencies
+    - name: Install build dependencies
       run: |
         python -m pip install --upgrade pip
+        python -m pip install ninja Cython pybind11 numpy meson
+    - name: Install additional dependencies
+      run: |
+        python -m pip install pylint pyhaloxml darwin-py ninja
+    - name: Install package
+      run: |
+        meson setup builddir
+        meson compile -C builddir
         python -m pip install mypy
-        python -m pip install -e ".[dev]"
-        python -m pip install pyhaloxml
-        python -m pip install darwin-py
-    - name: mypy
+        python -m pip install -e .
+    - name: Run mypy
       run: |
         mypy .
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -1,4 +1,4 @@
-name: Pylint
+name: pylint
 on:
   push:
     branches:
@@ -9,22 +9,44 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - name: Install minimal dependencies
+    - name: Install build dependencies
       run: |
-        sudo apt install -y libopenslide0 libgeos-dev
+        sudo apt update
+        sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+        sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev
+    - name: Build and install OpenSlide
+      run: |
+        git clone https://github.com/openslide/openslide.git
+        cd openslide
+        meson setup builddir
+        meson compile -C builddir
+        sudo meson install -C builddir
+        cd ..
+    - name: Build and install libvips
+      run: |
+        git clone https://github.com/libvips/libvips.git
+        cd libvips
+        meson setup builddir --prefix=/usr/local
+        meson compile -C builddir
+        sudo meson install -C builddir
+        sudo ldconfig
+        cd ..
     - uses: actions/checkout@v4
     - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
         python-version: "3.10"
-    - name: Install dependencies
+    - name: Install build dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install pylint cython
-        python -m pip install -e .
-        python -m pip install pyhaloxml
-        python -m pip install darwin-py
-        python setup.py build_ext --inplace
-    - name: Analysing the code with pylint
+        python -m pip install ninja Cython pybind11 numpy meson
+    - name: Install additional dependencies
+      run: |
+        python -m pip install pylint pyhaloxml darwin-py ninja
+    - name: Install package
+      run: |
+        python -m pip install pylint
+        python -m pip install .
+    - name: Run pylint
       run: |
         pylint dlup --errors-only
diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml
@@ -1,4 +1,4 @@
-name: Tox
+name: tox
 on:
   push:
     branches:
@@ -8,23 +8,54 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11"]
-
+    env:
+      CODECOV_CI: true
     steps:
-    - name: Install minimal dependencies
+    - name: Install build dependencies
       run: |
         sudo apt update
-        sudo apt install -y libopenslide0 libgeos-dev libvips42
+        sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev
+        sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev ninja-build
+    - name: Build and install OpenSlide
+      run: |
+        git clone https://github.com/openslide/openslide.git
+        cd openslide
+        meson setup builddir
+        meson compile -C builddir
+        sudo meson install -C builddir
+        cd ..
+    - name: Build and install libvips
+      run: |
+        git clone https://github.com/libvips/libvips.git
+        cd libvips
+        meson setup builddir --prefix=/usr/local
+        meson compile -C builddir
+        sudo meson install -C builddir
+        sudo ldconfig
+        cd ..
     - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+        python-version: "3.10"
+    - name: Clean up any existing installations
+      run: |
+        sudo rm -rf /usr/local/lib/python3.10/site-packages/dlup*
+        sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/dlup*
+        sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/_dlup_editable_loader.py
+        sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/easy-install.pth
+        sudo rm -rf dlup/build
+        sudo rm -rf /tmp/*
+    - name: Install environment
       run: |
         python -m pip install --upgrade pip
-        pip install tox tox-gh-actions
-    - name: Test with tox
-      run: tox
+        python -m pip install ninja meson meson-python>=0.15.0 numpy==1.26.4 Cython>=0.29 spin pybind11
+        python -m pip install tifffile>=2024.7.2 pyvips>=2.2.3 tqdm>=2.66.4 pillow>=10.3.0 openslide-python>=1.3.1
+        python -m pip install opencv-python-headless>=4.9.0.80 shapely>=2.0.4 pybind11>=2.8.0 pydantic coverage pytest psutil darwin-py pytest-mock
+        echo "Python executable: $(which python)"
+        echo "Python version: $(python --version)"
+        echo "Current directory: $PWD"
+    - name: Run tox
+      run: |
+        python -m pip install tox
+        tox