From 548fc87ae2311b7c1ef1913dae5e8b04ef8179f0 Mon Sep 17 00:00:00 2001 From: Jonas Teuwen <2347927+jonasteuwen@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:45:50 +0200 Subject: [PATCH] Feature/c tiffwriter (#249) * Added C++ TIFF Writer * Bump version to 0.7 * Rebuild setup toolchain to pyproject.toml * Remove CLI utilities and tests * Rename dlup.types to dlup._types * Reduce cyclic imports * Remove color transform from internal_handler='pil' (working towards v1.0) * Bugfixes --- .clang-format | 114 +++++ .github/workflows/black.yml | 1 - .github/workflows/codecov.yml | 63 ++- .github/workflows/mypy.yml | 42 +- .github/workflows/pylint.yml | 42 +- .github/workflows/tox.yml | 57 ++- .gitignore | 9 +- .pre-commit-config.yaml | 9 + .spin/cmds.py | 167 ++++++++ CITATION.cff | 4 +- MANIFEST.in | 4 + Makefile | 95 ----- README.md | 4 +- dlup/__init__.py | 7 +- dlup/_image.py | 25 +- dlup/_libtiff_tiff_writer.py | 20 + dlup/_libtiff_tiff_writer.pyi | 18 + dlup/_region.py | 2 +- dlup/{types.py => _types.py} | 0 dlup/annotations.py | 14 +- dlup/backends/__init__.py | 24 +- dlup/backends/common.py | 2 +- dlup/backends/openslide_backend.py | 2 +- dlup/backends/pyvips_backend.py | 16 +- dlup/backends/tifffile_backend.py | 2 +- dlup/cli/__init__.py | 65 --- dlup/cli/wsi.py | 42 -- dlup/data/dataset.py | 4 +- dlup/data/transforms.py | 3 +- dlup/logging.py | 2 +- dlup/utils/backends.py | 23 + dlup/utils/image.py | 2 +- dlup/writers.py | 213 +++++++--- docker/Dockerfile | 85 ---- docker/README.md | 12 - docker/jupyter_notebook_config.py | 21 - examples/resample_image_to_tiff.py | 15 +- meson.build | 101 +++++ pyproject.toml | 136 ++++-- setup.cfg | 14 +- setup.py | 101 ----- src/constants.h | 1 + src/image.h | 37 ++ src/libtiff_tiff_writer.cpp | 514 +++++++++++++++++++++++ tests/backends/test_openslide_backend.py | 4 +- tests/common.py | 4 +- tests/test_background.py | 4 +- tests/test_cli.py | 39 -- tests/test_image.py | 3 +- tests/test_writers.py | 30 +- tox.ini | 12 +- 51 files changed, 1518 insertions(+), 712 deletions(-) create mode 100644 .clang-format create mode 100644 .spin/cmds.py delete mode 100644 Makefile create mode 100644 dlup/_libtiff_tiff_writer.py create mode 100644 dlup/_libtiff_tiff_writer.pyi rename dlup/{types.py => _types.py} (100%) delete mode 100644 dlup/cli/__init__.py delete mode 100644 dlup/cli/wsi.py create mode 100644 dlup/utils/backends.py delete mode 100644 docker/Dockerfile delete mode 100644 docker/README.md delete mode 100644 docker/jupyter_notebook_config.py create mode 100644 meson.build delete mode 100644 setup.py create mode 100644 src/constants.h create mode 100644 src/image.h create mode 100644 src/libtiff_tiff_writer.cpp delete mode 100644 tests/test_cli.py diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..30b26b06 --- /dev/null +++ b/.clang-format @@ -0,0 +1,114 @@ +--- +Language: Cpp +ColumnLimit: 120 +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' +IndentCaseLabels: false +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: Never diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index ee607991..d4b82191 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,3 @@ - name: Black on: push: diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index aa05a4f4..01b80103 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -1,30 +1,67 @@ name: CodeCov on: - - push - - pull_request + push: + branches: + - main + pull_request: jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] + env: + CODECOV_CI: true steps: - - name: Install minimal dependencies + - name: Install build dependencies run: | sudo apt update - sudo apt install -y libopenslide0 libgeos-dev libvips42 + sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev + sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev ninja-build + - name: Build and install OpenSlide + run: | + git clone https://github.com/openslide/openslide.git + cd openslide + meson setup builddir + meson compile -C builddir + sudo meson install -C builddir + cd .. + - name: Build and install libvips + run: | + git clone https://github.com/libvips/libvips.git + cd libvips + meson setup builddir --prefix=/usr/local + meson compile -C builddir + sudo meson install -C builddir + sudo ldconfig + cd .. - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + python-version: "3.10" + - name: Clean up any existing installations + run: | + sudo rm -rf /usr/local/lib/python3.10/site-packages/dlup* + sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/dlup* + sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/_dlup_editable_loader.py + sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/easy-install.pth + sudo rm -rf dlup/build + sudo rm -rf /tmp/* + - name: Install environment run: | - python -m pip install --upgrade pip setuptools wheel coverage - python -m pip install -e ".[dev]" - - name: Run Coverage + python -m pip install --upgrade pip + python -m pip install ninja meson meson-python>=0.15.0 numpy==1.26.4 Cython>=0.29 spin pybind11 + python -m pip install tifffile>=2024.7.2 pyvips>=2.2.3 tqdm>=2.66.4 pillow>=10.3.0 openslide-python>=1.3.1 + python -m pip install opencv-python-headless>=4.9.0.80 shapely>=2.0.4 pybind11>=2.8.0 pydantic coverage pytest psutil darwin-py pytest-mock + echo "Python executable: $(which python)" + echo "Python version: $(python --version)" + echo "Current directory: $PWD" + meson setup builddir + meson compile -C builddir + meson install -C builddir + - name: Run coverage run: | + mv dlup _dlup # This is needed because otherwise it won't find the compiled libraries + export PYTHONPATH=$(python -c "import site; print(site.getsitepackages()[0])") coverage run -m pytest - name: Upload Coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 82ae3dd1..11d71550 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,23 +8,47 @@ on: jobs: build: runs-on: ubuntu-latest - name: mypy steps: - - name: Install minimal dependencies - run: | - sudo apt install -y libgeos-dev + - name: Install build dependencies + run: | + sudo apt update + sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev + sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev + - name: Build and install OpenSlide + run: | + git clone https://github.com/openslide/openslide.git + cd openslide + meson setup builddir + meson compile -C builddir + sudo meson install -C builddir + cd .. + - name: Build and install libvips + run: | + git clone https://github.com/libvips/libvips.git + cd libvips + meson setup builddir --prefix=/usr/local + meson compile -C builddir + sudo meson install -C builddir + sudo ldconfig + cd .. - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: "3.10" - - name: Install Dependencies + - name: Install build dependencies run: | python -m pip install --upgrade pip + python -m pip install ninja Cython pybind11 numpy meson + - name: Install additional dependencies + run: | + python -m pip install pylint pyhaloxml darwin-py ninja + - name: Install package + run: | + meson setup builddir + meson compile -C builddir python -m pip install mypy - python -m pip install -e ".[dev]" - python -m pip install pyhaloxml - python -m pip install darwin-py - - name: mypy + python -m pip install -e . + - name: Run mypy run: | mypy . diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 19f56c9a..edef6b09 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,4 +1,4 @@ -name: Pylint +name: pylint on: push: branches: @@ -9,22 +9,44 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Install minimal dependencies + - name: Install build dependencies run: | - sudo apt install -y libopenslide0 libgeos-dev + sudo apt update + sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev + sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev + - name: Build and install OpenSlide + run: | + git clone https://github.com/openslide/openslide.git + cd openslide + meson setup builddir + meson compile -C builddir + sudo meson install -C builddir + cd .. + - name: Build and install libvips + run: | + git clone https://github.com/libvips/libvips.git + cd libvips + meson setup builddir --prefix=/usr/local + meson compile -C builddir + sudo meson install -C builddir + sudo ldconfig + cd .. - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: "3.10" - - name: Install dependencies + - name: Install build dependencies run: | python -m pip install --upgrade pip - python -m pip install pylint cython - python -m pip install -e . - python -m pip install pyhaloxml - python -m pip install darwin-py - python setup.py build_ext --inplace - - name: Analysing the code with pylint + python -m pip install ninja Cython pybind11 numpy meson + - name: Install additional dependencies + run: | + python -m pip install pylint pyhaloxml darwin-py ninja + - name: Install package + run: | + python -m pip install pylint + python -m pip install . + - name: Run pylint run: | pylint dlup --errors-only diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index c0eeb6d2..5f5668fd 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -1,4 +1,4 @@ -name: Tox +name: tox on: push: branches: @@ -8,23 +8,54 @@ on: jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11"] - + env: + CODECOV_CI: true steps: - - name: Install minimal dependencies + - name: Install build dependencies run: | sudo apt update - sudo apt install -y libopenslide0 libgeos-dev libvips42 + sudo apt install -y meson libgl1-mesa-glx libcairo2-dev libgdk-pixbuf2.0-dev libglib2.0-dev libjpeg-dev libpng-dev libtiff5-dev libxml2-dev libopenjp2-7-dev libsqlite3-dev zlib1g-dev libzstd-dev + sudo apt install -y libfftw3-dev libexpat1-dev libgsf-1-dev liborc-0.4-dev libtiff5-dev ninja-build + - name: Build and install OpenSlide + run: | + git clone https://github.com/openslide/openslide.git + cd openslide + meson setup builddir + meson compile -C builddir + sudo meson install -C builddir + cd .. + - name: Build and install libvips + run: | + git clone https://github.com/libvips/libvips.git + cd libvips + meson setup builddir --prefix=/usr/local + meson compile -C builddir + sudo meson install -C builddir + sudo ldconfig + cd .. - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + python-version: "3.10" + - name: Clean up any existing installations + run: | + sudo rm -rf /usr/local/lib/python3.10/site-packages/dlup* + sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/dlup* + sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/_dlup_editable_loader.py + sudo rm -rf /opt/hostedtoolcache/Python/3.10.14/arm64/lib/python3.10/site-packages/easy-install.pth + sudo rm -rf dlup/build + sudo rm -rf /tmp/* + - name: Install environment run: | python -m pip install --upgrade pip - pip install tox tox-gh-actions - - name: Test with tox - run: tox + python -m pip install ninja meson meson-python>=0.15.0 numpy==1.26.4 Cython>=0.29 spin pybind11 + python -m pip install tifffile>=2024.7.2 pyvips>=2.2.3 tqdm>=2.66.4 pillow>=10.3.0 openslide-python>=1.3.1 + python -m pip install opencv-python-headless>=4.9.0.80 shapely>=2.0.4 pybind11>=2.8.0 pydantic coverage pytest psutil darwin-py pytest-mock + echo "Python executable: $(which python)" + echo "Python version: $(python --version)" + echo "Current directory: $PWD" + - name: Run tox + run: | + python -m pip install tox + tox diff --git a/.gitignore b/.gitignore index 9493f69c..b84500c1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,12 @@ _background.c _background.cpp _background.html +_background.cpython-*-darwin.so +_libtiff_tiff_writer.cpython-*.so + +# Output files +*.tif +*.tiff # Byte-compiled / optimized / DLL files __pycache__/ @@ -9,7 +15,7 @@ __pycache__/ *$py.class # C extensions -*.so +_skbuild # CMake CMakeCache.txt @@ -119,3 +125,4 @@ dlup/preprocessors/tests/data/test_output # OS files .DS_Store +_background.cpython-311-darwin.so diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6f92756d..42af2fcc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,6 +49,7 @@ repos: types: [python] args: [ + "dlup", "-rn", # Only display messages "-sn", # Don't display the score "--errors-only" # Only show the errors @@ -58,3 +59,11 @@ repos: hooks: - id: cython-lint - id: double-quote-cython-strings +- repo: local + hooks: + - id: clang-format + name: clang-format + entry: clang-format + language: system + files: \.(cpp|h)$ + args: ['-i'] diff --git a/.spin/cmds.py b/.spin/cmds.py new file mode 100644 index 00000000..809da674 --- /dev/null +++ b/.spin/cmds.py @@ -0,0 +1,167 @@ +import subprocess +import webbrowser +from pathlib import Path + +import click + + +@click.group() +def cli(): + """DLUP development commands""" + pass + + +@cli.command() +def build(): + """๐Ÿ”ง Build the project""" + subprocess.run(["meson", "setup", "builddir", "--prefix", str(Path.cwd())], check=True) + subprocess.run(["meson", "compile", "-C", "builddir"], check=True) + subprocess.run(["meson", "install", "-C", "builddir"], check=True) + + +@cli.command() +@click.option("-v", "--verbose", is_flag=True, help="Verbose output") +@click.argument("tests", nargs=-1) +def test(verbose, tests): + """๐Ÿ” Run tests""" + cmd = ["pytest"] + if verbose: + cmd.append("-v") + if tests: + cmd.extend(tests) + subprocess.run(cmd, check=True) + + +@cli.command() +def mypy(): + """๐Ÿฆ† Run mypy for type checking""" + subprocess.run(["mypy", "dlup"], check=True) + + +@cli.command() +def lint(): + """๐Ÿงน Run linting""" + subprocess.run(["flake8", "dlup", "tests"], check=True) + + +@cli.command() +def ipython(): + """๐Ÿ’ป Start IPython""" + subprocess.run(["ipython"], check=True) + + +@cli.command(context_settings=dict(ignore_unknown_options=True)) +@click.argument("args", nargs=-1, type=click.UNPROCESSED) +def python(args): + """๐Ÿ Start Python""" + subprocess.run(["python"] + list(args), check=True) + + +@cli.command() +def docs(): + """๐Ÿ“š Build documentation""" + docs_dir = Path("docs") + build_dir = docs_dir / "_build" + + # Remove old builds + if build_dir.exists(): + for item in build_dir.iterdir(): + if item.is_dir(): + for subitem in item.iterdir(): + if subitem.is_file(): + subitem.unlink() + item.rmdir() + else: + item.unlink() + + # Generate API docs + subprocess.run(["sphinx-apidoc", "-o", str(docs_dir), "dlup"], check=True) + + # Build HTML docs + subprocess.run(["sphinx-build", "-b", "html", str(docs_dir), str(build_dir / "html")], check=True) + + +@cli.command() +def viewdocs(): + """๐Ÿ“– View documentation in browser""" + doc_path = Path.cwd() / "docs" / "_build" / "html" / "index.html" + webbrowser.open(f"file://{doc_path.resolve()}") + + +@cli.command() +def uploaddocs(): + """๐Ÿ“ค Upload documentation""" + docs() + source = Path.cwd() / "docs" / "_build" / "html" + subprocess.run( + ["rsync", "-avh", f"{source}/", "docs@aiforoncology.nl:/var/www/html/docs/dlup", "--delete"], check=True + ) + + +@cli.command() +def servedocs(): + """๐Ÿ–ฅ๏ธ Serve documentation and watch for changes""" + subprocess.run(["sphinx-autobuild", "docs", "docs/_build/html"], check=True) + + +@cli.command() +def clean(): + """๐Ÿงน Clean all build, test, coverage, docs and Python artifacts""" + dirs_to_remove = ["build", "dist", "_skbuild", ".eggs", "htmlcov", ".tox", ".pytest_cache", "docs/_build"] + for dir in dirs_to_remove: + path = Path(dir) + if path.exists(): + for item in path.glob("**/*"): + if item.is_file(): + item.unlink() + elif item.is_dir(): + item.rmdir() + path.rmdir() + + patterns_to_remove = ["*.egg-info", "*.egg", "*.pyc", "*.pyo", "*~", "__pycache__", "*.o", "*.so"] + for pattern in patterns_to_remove: + for path in Path(".").rglob(pattern): + if path.is_file(): + path.unlink() + elif path.is_dir(): + path.rmdir() + + cython_compiled_files = ["dlup/_background.c"] + for file in cython_compiled_files: + path = Path(file) + if path.exists(): + path.unlink() + + +@cli.command() +def coverage(): + """๐Ÿงช Run tests and generate coverage report""" + subprocess.run(["coverage", "run", "--source", "dlup", "-m", "pytest"], check=True) + subprocess.run(["coverage", "report", "-m"], check=True) + subprocess.run(["coverage", "html"], check=True) + coverage_path = Path.cwd() / "htmlcov" / "index.html" + webbrowser.open(f"file://{coverage_path.resolve()}") + + +@cli.command() +def release(): + """๐Ÿ“ฆ Package and upload a release""" + dist() + subprocess.run(["twine", "upload", "dist/*"], check=True) + + +@cli.command() +def changelog(): + return + + +@cli.command() +def dist(): + """๐Ÿ“ฆ Build source and wheel package""" + clean() + subprocess.run(["python", "-m", "build"], check=True) + subprocess.run(["ls", "-l", "dist"], check=True) + + +if __name__ == "__main__": + cli() diff --git a/CITATION.cff b/CITATION.cff index b6244e68..61271bdc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -16,6 +16,6 @@ authors: given-names: "Eric" orchid: "https://orcid.org/0000-0002-3375-6248" title: "DLUP: Deep Learning Utilities for Pathology" -version: 0.6.1 -date-released: 2024-08-01 +version: 0.7.0 +date-released: 2024-08-09 url: "https://github.com/nki-ai/dlup" diff --git a/MANIFEST.in b/MANIFEST.in index c2e6744f..5e6cf239 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,14 @@ include CONTRIBUTING.rst include LICENSE include README.md +include meson.build +include pyproject.toml include dlup/py.typed include dlup/_background.pyx include dlup/_background.pyi +recursive-include dlup *.py *.pyi +recursive-include src *.cpp *.h *.hpp recursive-include tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/Makefile b/Makefile deleted file mode 100644 index 8eaaa318..00000000 --- a/Makefile +++ /dev/null @@ -1,95 +0,0 @@ -.PHONY: clean clean-test clean-pyc clean-build docs help -.DEFAULT_GOAL := help - -define BROWSER_PYSCRIPT -import os, webbrowser, sys - -from urllib.request import pathname2url - -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) -endef -export BROWSER_PYSCRIPT - -define PRINT_HELP_PYSCRIPT -import re, sys - -for line in sys.stdin: - match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) - if match: - target, help = match.groups() - print("%-20s %s" % (target, help)) -endef -export PRINT_HELP_PYSCRIPT - -BROWSER := python -c "$$BROWSER_PYSCRIPT" - -help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) - -clean: clean-build clean-pyc clean-test clean-docs ## remove all build, test, coverage, docs and Python artifacts - -clean-build: ## remove build artifacts - rm -fr build/ - rm -fr dist/ - rm -fr .eggs/ - rm -fr dlup/_background.{c,so,cpp,html} - find . -name '*.egg-info' -exec rm -fr {} + - find . -name '*.egg' -exec rm -f {} + - -clean-pyc: ## remove Python file artifacts - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -fr {} + - -clean-test: ## remove test and coverage artifacts - rm -fr .tox/ - rm -f .coverage - rm -fr htmlcov/ - rm -fr .pytest_cache - -clean-docs: ## clean sphinx docs - rm -f docs/dlup.rst - rm -f docs/modules.rst - rm -f docs/dlup.*.rst - rm -rf docs/_build - -lint: ## check style with flake8 - flake8 dlup tests - -test: ## run tests quickly with the default Python - pytest - -test-all: ## run tests on every Python version with tox - tox - -coverage: ## check code coverage quickly with the default Python - coverage run --source dlup -m pytest - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html - -docs: clean-docs ## generate Sphinx HTML documentation, including API docs - sphinx-apidoc -o docs/ dlup - $(MAKE) -C docs clean - $(MAKE) -C docs html - -viewdocs: - $(BROWSER) docs/_build/html/index.html - -uploaddocs: docs # Compile the docs - rsync -avh docs/_build/html/ docs@aiforoncology.nl:/var/www/html/docs/dlup --delete - -servedocs: docs ## compile the docs watching for changes - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . - -release: dist ## package and upload a release - twine upload dist/* - -dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel - ls -l dist - -install: clean ## install the package to the active Python's site-packages - python setup.py install diff --git a/README.md b/README.md index 7e4a8625..6558c2f6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you use DLUP in your research, please use the following BiBTeX entry: month = {8}, title = {{DLUP: Deep Learning Utilities for Pathology}}, url = {https://github.com/NKI-AI/dlup}, - version = {0.6.1}, + version = {0.7.0}, year = {2024} } ``` @@ -43,5 +43,5 @@ If you use DLUP in your research, please use the following BiBTeX entry: or the following plain bibliography: ``` -Teuwen, J., Romor, L., Pai, A., Schirris, Y., Marcus E. (2024). DLUP: Deep Learning Utilities for Pathology (Version 0.6.1) [Computer software]. https://github.com/NKI-AI/dlup +Teuwen, J., Romor, L., Pai, A., Schirris, Y., Marcus E. (2024). DLUP: Deep Learning Utilities for Pathology (Version 0.7.0) [Computer software]. https://github.com/NKI-AI/dlup ``` diff --git a/dlup/__init__.py b/dlup/__init__.py index a270833d..e0d3fb0a 100644 --- a/dlup/__init__.py +++ b/dlup/__init__.py @@ -2,16 +2,15 @@ import logging -from ._exceptions import UnsupportedSlideError from ._image import SlideImage from ._region import BoundaryMode, RegionView -from .annotations import AnnotationType, WsiAnnotations +from .annotations import AnnotationClass, AnnotationType, WsiAnnotations pyvips_logger = logging.getLogger("pyvips") pyvips_logger.setLevel(logging.CRITICAL) __author__ = """dlup contributors""" __email__ = "j.teuwen@nki.nl" -__version__ = "0.6.1" +__version__ = "0.7.0" -__all__ = ("SlideImage", "WsiAnnotations", "AnnotationType", "RegionView", "UnsupportedSlideError", "BoundaryMode") +__all__ = ("SlideImage", "WsiAnnotations", "AnnotationType", "AnnotationClass", "RegionView", "BoundaryMode") diff --git a/dlup/_image.py b/dlup/_image.py index 31bf4ee2..db510676 100644 --- a/dlup/_image.py +++ b/dlup/_image.py @@ -24,11 +24,11 @@ import pyvips from pyvips.enums import Kernel as VipsKernel -from dlup import UnsupportedSlideError +from dlup._exceptions import UnsupportedSlideError from dlup._region import BoundaryMode, RegionView -from dlup.backends import ImageBackend from dlup.backends.common import AbstractSlideBackend -from dlup.types import GenericFloatArray, GenericIntArray, GenericNumber, GenericNumberArray, PathLike +from dlup._types import GenericFloatArray, GenericIntArray, GenericNumber, GenericNumberArray, PathLike +from dlup.utils.backends import ImageBackend from dlup.utils.image import check_if_mpp_is_valid _Box = tuple[GenericNumber, GenericNumber, GenericNumber, GenericNumber] @@ -255,21 +255,6 @@ def color_profile(self) -> io.BytesIO | None: """ return getattr(self._wsi, "color_profile", None) - @property - def _pil_color_transform(self) -> PIL.ImageCms.ImageCmsTransform | None: - if self.color_profile is None: - return None - - color_profile = PIL.ImageCms.getOpenProfile(self.color_profile) # type: ignore - - if self.__color_transforms is None: - to_profile = PIL.ImageCms.createProfile("sRGB") - intent = PIL.ImageCms.getDefaultIntent(color_profile) # type: ignore - self.__color_transform = PIL.ImageCms.buildTransform( - self.color_profile, to_profile, self._wsi.mode, self._wsi.mode, intent, 0 - ) - return self.__color_transform - def __enter__(self) -> "SlideImage": return self @@ -435,8 +420,8 @@ def read_region( box=box, ) - if self._apply_color_profile and self._pil_color_transform is not None: - PIL.ImageCms.applyTransform(pil_region, self._pil_color_transform, inPlace=True) + if self._apply_color_profile: + warnings.warn("Applying color profile is not supported with PIL backend.", UserWarning) return pyvips.Image.new_from_array(np.asarray(pil_region), interpretation=vips_region.interpretation) diff --git a/dlup/_libtiff_tiff_writer.py b/dlup/_libtiff_tiff_writer.py new file mode 100644 index 00000000..1c79526c --- /dev/null +++ b/dlup/_libtiff_tiff_writer.py @@ -0,0 +1,20 @@ +# Copyright (c) dlup contributors +"""This module is only required for the linters""" +from typing import Any + +import numpy as np +from numpy.typing import NDArray + + +class LibtiffTiffWriter: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def write_tile(self, tile: NDArray[np.int_], row: int, col: int) -> None: + pass + + def write_pyramid(self) -> None: + pass + + def finalize(self) -> None: + pass diff --git a/dlup/_libtiff_tiff_writer.pyi b/dlup/_libtiff_tiff_writer.pyi new file mode 100644 index 00000000..3a8b9711 --- /dev/null +++ b/dlup/_libtiff_tiff_writer.pyi @@ -0,0 +1,18 @@ +from pathlib import Path + +import numpy as np +from numpy.typing import NDArray + +class LibtiffTiffWriter: + def __init__( + self, + file_path: str | Path, + size: tuple[int, int, int], + mpp: tuple[float, float], + tile_size: tuple[int, int], + compression: str, + quality: int, + ) -> None: ... + def write_tile(self, tile: NDArray[np.int_], row: int, col: int) -> None: ... + def write_pyramid(self) -> None: ... + def finalize(self) -> None: ... diff --git a/dlup/_region.py b/dlup/_region.py index 408fd0b1..c78128ef 100644 --- a/dlup/_region.py +++ b/dlup/_region.py @@ -8,7 +8,7 @@ import numpy as np import pyvips -from dlup.types import GenericFloatArray, GenericIntArray +from dlup._types import GenericFloatArray, GenericIntArray class BoundaryMode(str, Enum): diff --git a/dlup/types.py b/dlup/_types.py similarity index 100% rename from dlup/types.py rename to dlup/_types.py diff --git a/dlup/annotations.py b/dlup/annotations.py index 80351c19..d9322cbf 100644 --- a/dlup/annotations.py +++ b/dlup/annotations.py @@ -41,7 +41,7 @@ from shapely.validation import make_valid from dlup._exceptions import AnnotationError -from dlup.types import GenericNumber, PathLike +from dlup._types import GenericNumber, PathLike from dlup.utils.imports import DARWIN_SDK_AVAILABLE, PYHALOXML_AVAILABLE # TODO: @@ -483,9 +483,7 @@ def shape( raise AnnotationError("z_index is not supported for point annotations.") if geom_type == "point": - annotation_class = AnnotationClass( - label=label, annotation_type=AnnotationType.POINT, color=color, z_index=None - ) + annotation_class = AnnotationClass(label=label, annotation_type=AnnotationType.POINT, color=color, z_index=None) return [ Point( np.asarray(coordinates["coordinates"]), @@ -493,9 +491,7 @@ def shape( ) ] if geom_type == "multipoint": - annotation_class = AnnotationClass( - label=label, annotation_type=AnnotationType.POINT, color=color, z_index=None - ) + annotation_class = AnnotationClass(label=label, annotation_type=AnnotationType.POINT, color=color, z_index=None) return [Point(np.asarray(c), a_cls=annotation_class) for c in coordinates["coordinates"]] if geom_type == "polygon": @@ -956,9 +952,7 @@ def from_darwin_json( z_index = None if annotation_type == AnnotationType.POINT or z_indices is None else z_indices[name] curr_data = curr_annotation.data - _cls = AnnotationClass( - label=name, annotation_type=annotation_type, color=annotation_color, z_index=z_index - ) + _cls = AnnotationClass(label=name, annotation_type=annotation_type, color=annotation_color, z_index=z_index) if annotation_type == AnnotationType.POINT: curr_point = Point((curr_data["x"], curr_data["y"]), a_cls=_cls) layers.append(curr_point) diff --git a/dlup/backends/__init__.py b/dlup/backends/__init__.py index d17635d4..939f47fc 100644 --- a/dlup/backends/__init__.py +++ b/dlup/backends/__init__.py @@ -1,22 +1,4 @@ # Copyright (c) dlup contributors -from __future__ import annotations - -from enum import Enum -from typing import Any, Callable - -from dlup.backends.openslide_backend import OpenSlideSlide -from dlup.backends.tifffile_backend import TifffileSlide -from dlup.types import PathLike - -from .pyvips_backend import PyVipsSlide - - -class ImageBackend(Enum): - """Available image experimental_backends.""" - - OPENSLIDE: Callable[[PathLike], OpenSlideSlide] = OpenSlideSlide - PYVIPS: Callable[[PathLike], PyVipsSlide] = PyVipsSlide - TIFFFILE: Callable[[PathLike], TifffileSlide] = TifffileSlide - - def __call__(self, *args: "ImageBackend" | str) -> Any: - return self.value(*args) +from .openslide_backend import OpenSlideSlide as OpenSlideSlide # noqa: F401 +from .pyvips_backend import PyVipsSlide as PyVipsSlide # noqa: F401 +from .tifffile_backend import TifffileSlide as TifffileSlide # noqa: F401 diff --git a/dlup/backends/common.py b/dlup/backends/common.py index d3a3c5a6..2200bc87 100644 --- a/dlup/backends/common.py +++ b/dlup/backends/common.py @@ -9,7 +9,7 @@ import numpy as np import pyvips -from dlup.types import PathLike +from dlup._types import PathLike from dlup.utils.image import check_if_mpp_is_valid diff --git a/dlup/backends/openslide_backend.py b/dlup/backends/openslide_backend.py index a13930a9..23885c89 100644 --- a/dlup/backends/openslide_backend.py +++ b/dlup/backends/openslide_backend.py @@ -13,7 +13,7 @@ from packaging.version import Version from dlup.backends.common import AbstractSlideBackend -from dlup.types import PathLike +from dlup._types import PathLike from dlup.utils.image import check_if_mpp_is_valid TIFF_PROPERTY_NAME_RESOLUTION_UNIT = "tiff.ResolutionUnit" diff --git a/dlup/backends/pyvips_backend.py b/dlup/backends/pyvips_backend.py index 75e14518..82e9f2d6 100644 --- a/dlup/backends/pyvips_backend.py +++ b/dlup/backends/pyvips_backend.py @@ -10,9 +10,8 @@ import pyvips from packaging.version import Version -from dlup import UnsupportedSlideError from dlup.backends.common import AbstractSlideBackend -from dlup.types import PathLike +from dlup._types import PathLike from dlup.utils.image import check_if_mpp_is_valid PYVIPS_ASSOCIATED_IMAGES = "slide-associated-images" @@ -97,18 +96,7 @@ def _read_as_openslide(self, path: PathLike) -> None: for level in range(0, self._level_count): image = self._get_image(level) - openslide_shape = ( - int(image.get(f"openslide.level[{level}].width")), - int(image.get(f"openslide.level[{level}].height")), - ) - pyvips_shape = (image.width, image.height) - if not openslide_shape == pyvips_shape: - raise UnsupportedSlideError( - f"Reading {path} failed as openslide metadata reports different shapes than pyvips. " - f"Got {openslide_shape} and {pyvips_shape}." - ) - - self._shapes.append(pyvips_shape) + self._shapes.append((image.width, image.height)) self._downsamples.append(float(image.get(f"openslide.level[{level}].downsample"))) mpp_x, mpp_y = None, None diff --git a/dlup/backends/tifffile_backend.py b/dlup/backends/tifffile_backend.py index c4c12988..f165588e 100644 --- a/dlup/backends/tifffile_backend.py +++ b/dlup/backends/tifffile_backend.py @@ -6,7 +6,7 @@ import tifffile from dlup.backends.common import AbstractSlideBackend -from dlup.types import PathLike +from dlup._types import PathLike from dlup.utils.tifffile_utils import get_tile diff --git a/dlup/cli/__init__.py b/dlup/cli/__init__.py deleted file mode 100644 index 4aef6aaa..00000000 --- a/dlup/cli/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) dlup contributors -"""DLUP Command-line interface. This is the file which builds the main parser.""" -import argparse -import pathlib - - -def dir_path(path: str) -> pathlib.Path: - """Check if the path is a valid directory. - Parameters - ---------- - path : str - Returns - ------- - pathlib.Path - The path as a pathlib.Path object. - """ - _path = pathlib.Path(path) - if _path.is_dir(): - return _path - raise argparse.ArgumentTypeError(f"{path} is not a valid directory.") - - -def file_path(path: str, need_exists: bool = True) -> pathlib.Path: - """Check if the path is a valid file. - Parameters - ---------- - path : str - need_exists : bool - - Returns - ------- - pathlib.Path - The path as a pathlib.Path object. - """ - _path = pathlib.Path(path) - if need_exists: - if _path.is_file(): - return _path - raise argparse.ArgumentTypeError(f"{path} is not a valid file.") - return _path - - -def main() -> None: - """ - Console script for dlup. - """ - # From https://stackoverflow.com/questions/17073688/how-to-use-argparse-subparsers-correctly - root_parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - root_subparsers = root_parser.add_subparsers(help="Possible DLUP CLI utils to run.") - root_subparsers.required = True - root_subparsers.dest = "subcommand" - - # Prevent circular import - from dlup.cli.wsi import register_parser as register_wsi_subcommand - - # Whole slide images related commands. - register_wsi_subcommand(root_subparsers) - - args = root_parser.parse_args() - args.subcommand(args) - - -if __name__ == "__main__": - main() diff --git a/dlup/cli/wsi.py b/dlup/cli/wsi.py deleted file mode 100644 index c16f9690..00000000 --- a/dlup/cli/wsi.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) dlup contributors -import argparse -import json -import pathlib - -from dlup import SlideImage - - -def info(args: argparse.Namespace) -> None: - """Return available slide properties.""" - slide = SlideImage.from_file_path(args.slide_file_path) - props = slide.properties - if not props: - return print("No properties found.") - if args.json: - print(json.dumps(dict(props))) - return - - for k, v in props.items(): - print(f"{k}\t{v}") - - -def register_parser(parser: argparse._SubParsersAction) -> None: # type: ignore - """Register wsi commands to a root parser.""" - wsi_parser = parser.add_parser("wsi", help="WSI parser") - wsi_subparsers = wsi_parser.add_subparsers(help="WSI subparser") - wsi_subparsers.required = True - wsi_subparsers.dest = "subcommand" - - # Get generic slide infos. - info_parser = wsi_subparsers.add_parser("info", help="Return available slide properties.") - info_parser.add_argument( - "slide_file_path", - type=pathlib.Path, - help="Input slide image.", - ) - info_parser.add_argument( - "--json", - action="store_true", - help="Print available properties in json format.", - ) - info_parser.set_defaults(subcommand=info) diff --git a/dlup/data/dataset.py b/dlup/data/dataset.py index 69d55ace..8c9a28f7 100644 --- a/dlup/data/dataset.py +++ b/dlup/data/dataset.py @@ -33,12 +33,12 @@ from dlup import BoundaryMode, SlideImage from dlup.annotations import Point, Polygon, WsiAnnotations -from dlup.backends import ImageBackend from dlup.backends.common import AbstractSlideBackend from dlup.background import compute_masked_indices from dlup.tiling import Grid, GridOrder, TilingMode from dlup.tools import ConcatSequences, MapSequence -from dlup.types import PathLike, ROIType +from dlup._types import PathLike, ROIType +from dlup.utils.backends import ImageBackend MaskTypes = Union["SlideImage", npt.NDArray[np.int_], "WsiAnnotations"] diff --git a/dlup/data/transforms.py b/dlup/data/transforms.py index 71373cd6..bc2e8979 100644 --- a/dlup/data/transforms.py +++ b/dlup/data/transforms.py @@ -1,4 +1,5 @@ # Copyright (c) dlup contributors +"""This module contains the transforms which can be applied to the output of a Dataset class""" from __future__ import annotations from collections import defaultdict @@ -104,7 +105,7 @@ def convert_annotations( original_values = None interiors = [np.asarray(pi.coords).round().astype(np.int32) for pi in curr_annotation.interiors] - if interiors is not []: + if interiors != []: original_values = mask.copy() holes_mask = np.zeros(region_size, dtype=np.int32) # Get a mask where the holes are diff --git a/dlup/logging.py b/dlup/logging.py index 845ddf00..06b4ae90 100644 --- a/dlup/logging.py +++ b/dlup/logging.py @@ -6,7 +6,7 @@ import pathlib import sys -from dlup.types import PathLike +from dlup._types import PathLike def setup_logging( diff --git a/dlup/utils/backends.py b/dlup/utils/backends.py new file mode 100644 index 00000000..944adcfa --- /dev/null +++ b/dlup/utils/backends.py @@ -0,0 +1,23 @@ +# Copyright (c) dlup contributors +"""Utilities to handle backends.""" +from __future__ import annotations + +from enum import Enum +from typing import Any, Callable + +from dlup._types import PathLike + + +class ImageBackend(Enum): + """Available image experimental_backends.""" + + from dlup.backends.openslide_backend import OpenSlideSlide + from dlup.backends.pyvips_backend import PyVipsSlide + from dlup.backends.tifffile_backend import TifffileSlide + + OPENSLIDE: Callable[[PathLike], OpenSlideSlide] = OpenSlideSlide + PYVIPS: Callable[[PathLike], PyVipsSlide] = PyVipsSlide + TIFFFILE: Callable[[PathLike], TifffileSlide] = TifffileSlide + + def __call__(self, *args: "ImageBackend" | str) -> Any: + return self.value(*args) diff --git a/dlup/utils/image.py b/dlup/utils/image.py index d8f46bac..67defc10 100644 --- a/dlup/utils/image.py +++ b/dlup/utils/image.py @@ -2,7 +2,7 @@ """Utilities for handling WSIs.""" import math -from dlup import UnsupportedSlideError +from dlup._exceptions import UnsupportedSlideError def check_if_mpp_is_valid(mpp_x: float, mpp_y: float, *, rel_tol: float = 0.015) -> None: diff --git a/dlup/writers.py b/dlup/writers.py index 4ca19044..aeba5b17 100644 --- a/dlup/writers.py +++ b/dlup/writers.py @@ -4,6 +4,7 @@ """ from __future__ import annotations +import abc import pathlib import shutil import tempfile @@ -17,40 +18,41 @@ from tifffile import tifffile import dlup -from dlup.tiling import Grid, TilingMode -from dlup.types import PathLike +from dlup._libtiff_tiff_writer import LibtiffTiffWriter +from dlup.tiling import Grid, GridOrder, TilingMode +from dlup._types import PathLike from dlup.utils.tifffile_utils import get_tile class TiffCompression(str, Enum): """Compression types for tiff files.""" - NONE = "none" # No compression - CCITTFAX4 = "ccittfax4" # Fax4 compression - JPEG = "jpeg" # Jpeg compression - DEFLATE = "deflate" # zip compression - PACKBITS = "packbits" # packbits compression - LZW = "lzw" # LZW compression, not implemented in tifffile - WEBP = "webp" # WEBP compression - ZSTD = "zstd" # ZSTD compression - JP2K = "jp2k" # JP2K compression - JP2K_LOSSY = "jp2k_lossy" - PNG = "png" + NONE = "NONE" # No compression + CCITTFAX4 = "CCITTFAX4" # Fax4 compression + JPEG = "JPEG" # Jpeg compression + DEFLATE = "DEFLATE" # zip compression + PACKBITS = "PACKBITS" # packbits compression + LZW = "LZW" # LZW compression, not implemented in tifffile + WEBP = "WEBP" # WEBP compression + ZSTD = "ZSTD" # ZSTD compression + JP2K = "JP2K" # JP2K compression + JP2K_LOSSY = "JP2K_LOSSY" + PNG = "PNG" # Mapping to map TiffCompression to their respective values in tifffile. TIFFFILE_COMPRESSION = { - "none": None, - "ccittfax4": "CCITT_T4", - "jpeg": "jpeg", - "deflate": "deflate", - "packbits": "packbits", - "lzw": "lzw", - "webp": "webp", - "zstd": "zstd", - "jp2k": "jpeg2000", - "jp2k_lossy": "jpeg_2000_lossy", - "png": "png", + "NONE": None, + "CCITTFAX4": "CCITT_T4", + "JPEG": "jpeg", + "DEFLATE": "deflate", + "PACKBITS": "packbits", + "LZW": "lzw", + "WEBP": "webp", + "ZSTD": "zstd", + "JP2K": "jpeg2000", + "JP2K_LOSSY": "jpeg_2000_lossy", + "PNG": "png", } @@ -86,9 +88,114 @@ def _color_dict_to_color_lut(color_map: dict[int, str]) -> npt.NDArray[np.uint16 return color_lut -class ImageWriter: +class ImageWriter(abc.ABC): """Base writer class""" + def __init__( + self, + filename: PathLike, + size: tuple[int, int] | tuple[int, int, int], + mpp: float | tuple[float, float], + tile_size: tuple[int, int] = (512, 512), + pyramid: bool = False, + colormap: dict[int, str] | None = None, + compression: TiffCompression | None = TiffCompression.JPEG, + is_mask: bool = False, + quality: int | None = 100, + metadata: dict[str, str] | None = None, + ): + + if compression is None: + compression = TiffCompression.NONE + + self._filename = filename + self._tile_size = tile_size + self._size = (*size[::-1], 1) if len(size) == 2 else (size[1], size[0], size[2]) + self._mpp: tuple[float, float] = (mpp, mpp) if isinstance(mpp, (int, float)) else mpp + self._pyramid = pyramid + self._colormap = _color_dict_to_color_lut(colormap) if colormap is not None else None + self._compression = compression + self._is_mask = is_mask + self._quality = quality + self._metadata = metadata + + def from_pil(self, pil_image: PIL.Image.Image) -> None: + """ + Create tiff image from a PIL image + + Parameters + ---------- + pil_image : PIL.Image + """ + if not np.all(np.asarray(pil_image.size)[::-1] >= self._tile_size): + raise RuntimeError( + f"PIL Image must be larger than set tile size. Got {pil_image.size} and {self._tile_size}." + ) + iterator = _tiles_iterator_from_pil_image(pil_image, self._tile_size, order="F") + self.from_tiles_iterator(iterator) + + @abc.abstractmethod + def from_tiles_iterator(self, iterator: Iterator[npt.NDArray[np.int_]]) -> None: + """""" + + +class LibtiffImageWriter(ImageWriter): + """Image writer that writes tile-by-tile to tiff using LibtiffWriter.""" + + def __init__( + self, + filename: PathLike, + size: tuple[int, int] | tuple[int, int, int], + mpp: float | tuple[float, float], + tile_size: tuple[int, int] = (512, 512), + pyramid: bool = False, + colormap: dict[int, str] | None = None, + compression: TiffCompression | None = TiffCompression.JPEG, + is_mask: bool = False, + quality: int | None = 100, + metadata: dict[str, str] | None = None, + ): + super().__init__( + filename, + size, + mpp, + tile_size, + pyramid, + colormap, + compression, + is_mask, + quality, + metadata, + ) + + compression_value: str + if isinstance(self._compression, TiffCompression): + compression_value = self._compression.value + else: + compression_value = self._compression + + self._writer = LibtiffTiffWriter( + self._filename, + self._size, + self._mpp, + self._tile_size, + compression_value, + self._quality if self._quality is not None else 100, + ) + + def from_tiles_iterator(self, iterator: Iterator[npt.NDArray[np.int_]]) -> None: + tiles_per_row = (self._size[1] + self._tile_size[1] - 1) // self._tile_size[1] + + for idx, tile in enumerate(iterator): + row = (idx // tiles_per_row) * self._tile_size[0] + col = (idx % tiles_per_row) * self._tile_size[1] + self._writer.write_tile(tile, row, col) + + if self._pyramid: + self._writer.write_pyramid() + + self._writer.finalize() + class TifffileImageWriter(ImageWriter): """Image writer that writes tile-by-tile to tiff.""" @@ -103,7 +210,6 @@ def __init__( colormap: dict[int, str] | None = None, compression: TiffCompression | None = TiffCompression.JPEG, is_mask: bool = False, - anti_aliasing: bool = False, quality: int | None = 100, metadata: dict[str, str] | None = None, ): @@ -134,38 +240,18 @@ def __init__( metadata : dict[str, str] Metadata to write to the tiff file. """ - self._filename = pathlib.Path(filename) - self._tile_size = tile_size - - self._size = (*size[::-1], 1) if len(size) == 2 else (size[1], size[0], size[2]) - self._mpp: tuple[float, float] = (mpp, mpp) if isinstance(mpp, (int, float)) else mpp - - if compression is None: - compression = TiffCompression.NONE - - self._is_mask = is_mask - - self._anti_aliasing = anti_aliasing - self._compression = compression - self._pyramid = pyramid - self._quality = quality - self._metadata = metadata - self._colormap = _color_dict_to_color_lut(colormap) if colormap is not None else None - - def from_pil(self, pil_image: PIL.Image.Image) -> None: - """ - Create tiff image from a PIL image - - Parameters - ---------- - pil_image : PIL.Image - """ - if not np.all(np.asarray(pil_image.size)[::-1] >= self._tile_size): - raise RuntimeError( - f"PIL Image must be larger than set tile size. Got {pil_image.size} and {self._tile_size}." - ) - iterator = _tiles_iterator_from_pil_image(pil_image, self._tile_size) - self.from_tiles_iterator(iterator) + super().__init__( + filename, + size, + mpp, + tile_size, + pyramid, + colormap, + compression, + is_mask, + quality, + metadata, + ) def from_tiles_iterator(self, iterator: Iterator[npt.NDArray[np.int_]]) -> None: """ @@ -182,13 +268,11 @@ def from_tiles_iterator(self, iterator: Iterator[npt.NDArray[np.int_]]) -> None: filename = pathlib.Path(self._filename) native_size = self._size[:-1] - software = f"dlup {dlup.__version__} with tifffile.py backend" + software = f"dlup {dlup.__version__} (tifffile.py {tifffile.__version__})" n_subresolutions = 0 if self._pyramid: n_subresolutions = int(np.ceil(np.log2(np.asarray(native_size) / np.asarray(self._tile_size))).min()) - shapes = [ - np.floor(np.asarray(native_size) / 2**n).astype(int).tolist() for n in range(0, n_subresolutions + 1) - ] + shapes = [np.floor(np.asarray(native_size) / 2**n).astype(int).tolist() for n in range(0, n_subresolutions + 1)] # TODO: add to metadata "axes": "TCYXS", and "SignificantBits": 10, metadata = { @@ -276,7 +360,7 @@ def _write_page( def _tiles_iterator_from_pil_image( - pil_image: PIL.Image.Image, tile_size: tuple[int, int] + pil_image: PIL.Image.Image, tile_size: tuple[int, int], order: str | GridOrder = "F" ) -> Generator[npt.NDArray[np.int_], None, None]: """ Given a PIL image return a tile-iterator. @@ -285,6 +369,7 @@ def _tiles_iterator_from_pil_image( ---------- pil_image : PIL.Image tile_size : tuple + order : GridOrder or str Yields ------ @@ -297,7 +382,7 @@ def _tiles_iterator_from_pil_image( tile_size=tile_size, tile_overlap=(0, 0), mode=TilingMode.overflow, - order="F", + order=order, ) for tile_coordinates in grid: arr = np.asarray(pil_image) diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index dedba8f6..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,85 +0,0 @@ -FROM ubuntu:18.04 -ARG PYTHON="3.8" -ARG UNAME="dlup" -ARG BUILD_WORKERS="4" - -RUN apt-get -qq update -RUN apt-get update && apt-get install -y libxrender1 build-essential sudo \ - autoconf automake libtool pkg-config libtiff-dev libopenjp2-7-dev libglib2.0-dev \ - libxml++2.6-dev libsqlite3-dev libgdk-pixbuf2.0-dev libgl1-mesa-glx git wget rsync \ - fftw3-dev liblapacke-dev libpng-dev libopenblas-dev libxext-dev jq sudo \ - libfreetype6 libfreetype6-dev \ - # Purge pixman and cairo to be sure - && apt-get remove libpixman-1-dev libcairo2-dev \ - && apt-get purge libpixman-1-dev libcairo2-dev \ - && apt-get autoremove && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install pixman 0.40, as Ubuntu repository holds a version with a bug which can cause difficulties reading thumbnails -RUN cd /tmp \ - && wget https://www.cairographics.org/releases/pixman-0.40.0.tar.gz \ - && tar xvf pixman-0.40.0.tar.gz && rm pixman-0.40.0.tar.gz && cd pixman-0.40.0 \ - && ./configure && make -j$BUILD_WORKERS && make install \ - && cd /tmp && rm -rf pixman-0.40.0 - -# Install cairo 1.16 -RUN cd /tmp \ - && wget https://www.cairographics.org/releases/cairo-1.16.0.tar.xz \ - && tar xvf cairo-1.16.0.tar.xz && rm cairo-1.16.0.tar.xz && cd cairo-1.16.0 \ - && ./configure && make -j$BUILD_WORKERS && make install \ - && cd /tmp && rm -rf cairo-1.16.0 - -# Install OpenSlide for NKI-AI repository. -RUN git clone https://github.com/NKI-AI/openslide.git /tmp/openslide \ - && cd /tmp/openslide \ - && autoreconf -i \ - && ./configure && make -j$BUILD_WORKERS && make install && ldconfig \ - && cd /tmp && rm -rf openslide - -# Make a user -# Rename /home to /users to prevent issues with singularity -RUN mkdir /users && echo $UNAME \ - && adduser --disabled-password --gecos '' --home /users/$UNAME $UNAME \ - && adduser $UNAME sudo \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -RUN mkdir /$UNAME -USER $UNAME - -RUN cd /tmp && wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && bash Miniconda3-latest-Linux-x86_64.sh -b \ - && rm Miniconda3-latest-Linux-x86_64.sh -ENV PATH "/users/$UNAME/miniconda3/bin:$PATH:$CUDA_ROOT" - -# Setup python packages -RUN conda update -n base conda -yq \ - && conda install python=${PYTHON} \ - && conda install astunparse ninja setuptools cmake future requests dataclasses \ - && conda install pyyaml mkl mkl-include setuptools cmake cffi typing boost \ - && conda install tqdm jupyter matplotlib scikit-image pandas joblib -yq \ - && conda install typing_extensions \ - && conda clean -ya \ - && python -m pip install numpy==1.20 tifftools -q \ - # Install openslide-python from NKI-AI - && python -m pip install git+https://github.com/NKI-AI/openslide-python.git - -# Install jupyter config to be able to run in the docker environment -RUN jupyter notebook --generate-config -ENV CONFIG_PATH "/users/$UNAME/.jupyter/jupyter_notebook_config.py" -COPY "docker/jupyter_notebook_config.py" ${CONFIG_PATH} - -# Copy files into the docker -COPY [".", "/$UNAME"] -USER root -WORKDIR /$UNAME -RUN python setup.py install -RUN chown -R $UNAME:$UNAME /$UNAME - -USER $UNAME - -# Verify installation -RUN python -c 'import openslide' -RUN python -c 'import dlup' - -# Provide an open entrypoint for the docker -ENTRYPOINT $0 $@ diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index c8a3f550..00000000 --- a/docker/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Docker for DLUP -A Dockerfile is provided for DLUP which provides you with all the required dependencies. - -To build the container, run the following command from the root directory: -``` -docker build -t dlup:latest . -f docker/Dockerfile -``` - -Running the container can for instance be done with: -``` -docker run -it --ipc=host --rm -v /data:/data dlup:latest /bin/bash -``` diff --git a/docker/jupyter_notebook_config.py b/docker/jupyter_notebook_config.py deleted file mode 100644 index 6805578b..00000000 --- a/docker/jupyter_notebook_config.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -import os - -from IPython.lib import passwd # pylint: disable=no-name-in-module - -c = get_config() # type: ignore # pylint: disable=undefined-variable # noqa -c.NotebookApp.ip = "0.0.0.0" -c.NotebookApp.port = int(os.getenv("PORT", 8888)) -c.NotebookApp.open_browser = False - -password = os.environ.get("PASSWORD", False) -if password: - c.NotebookApp.password = passwd(password) -else: - c.NotebookApp.password = "" - c.NotebookApp.token = "" - -try: - del os.environ["PASSWORD"] -except KeyError: - pass diff --git a/examples/resample_image_to_tiff.py b/examples/resample_image_to_tiff.py index 96881e99..35ec2312 100644 --- a/examples/resample_image_to_tiff.py +++ b/examples/resample_image_to_tiff.py @@ -7,6 +7,7 @@ """ import argparse +import time from pathlib import Path from typing import Iterator @@ -15,7 +16,7 @@ from dlup import SlideImage from dlup.data.dataset import TiledWsiDataset -from dlup.writers import TiffCompression, TifffileImageWriter +from dlup.writers import LibtiffImageWriter, TiffCompression, TifffileImageWriter def resample(args: argparse.Namespace) -> None: @@ -35,7 +36,9 @@ def resample(args: argparse.Namespace) -> None: ) scaled_region_view = dataset.slide_image.get_scaled_view(dataset.slide_image.get_scaling(mpp)) - writer = TifffileImageWriter( + writer_class = LibtiffImageWriter if args.use_libtiff else TifffileImageWriter + + writer = writer_class( args.output, size=(*scaled_region_view.size, 3), mpp=(mpp, mpp), @@ -51,7 +54,10 @@ def tiles_iterator(dataset: TiledWsiDataset) -> Iterator[npt.NDArray[np.int_]]: arr = tile["image"].flatten(background=(255, 255, 255)).numpy() yield arr + start = time.time() writer.from_tiles_iterator(tiles_iterator(dataset)) + end = time.time() + print(f"Time to write the TIFF file: {end - start:.2f} seconds") def main() -> None: @@ -60,6 +66,11 @@ def main() -> None: parser.add_argument("output", type=Path, help="Path to the output TIFF file.") parser.add_argument("--tile-size", type=int, nargs=2, default=(512, 512), help="Size of the tiles in the TIFF.") parser.add_argument("--mpp", type=float, required=False, help="Microns per pixel of the output TIFF file.") + parser.add_argument( + "--use-libtiff", + action="store_true", + help="Use libtiff for writing the TIFF file, otherwise use a tifffile.py writer.", + ) args = parser.parse_args() with SlideImage.from_file_path(args.input, internal_handler="vips", backend="PYVIPS") as img: diff --git a/meson.build b/meson.build new file mode 100644 index 00000000..da3db1b8 --- /dev/null +++ b/meson.build @@ -0,0 +1,101 @@ +project('dlup', 'cpp', 'cython', + version : '0.7.0', + default_options : ['warning_level=3', 'cpp_std=c++17']) + + +py_mod = import('python') +py = py_mod.find_installation(pure: false) +py_dep = py.dependency() + +libtiff_dep = dependency('libtiff-4', required : false) +if not libtiff_dep.found() + libtiff_dep = dependency('tiff', required : false) +endif +if not libtiff_dep.found() + libtiff_dep = cc.find_library('tiff', required : false) +endif +if not libtiff_dep.found() + error('libtiff not found. Please install libtiff development files.') +endif + +# Check for ZSTD library +zstd_dep = dependency('libzstd', required : false) +have_zstd = zstd_dep.found() +if have_zstd + message('ZSTD support enabled') +else + message('ZSTD support disabled') +endif + +# Capture the output of the run_command and convert to relative paths +numpy_include = run_command(py, ['-c', ''' +import os +import numpy +print(os.path.relpath(numpy.get_include(), os.getcwd())) +'''], check: true).stdout().strip() + +pybind11_include = run_command(py, ['-c', ''' +import os +import pybind11 +print(os.path.relpath(pybind11.get_include(), os.getcwd())) +'''], check: true).stdout().strip() + +# Use the relative paths +incdir_numpy = include_directories(numpy_include) +incdir_pybind11 = include_directories(pybind11_include) + +# Check if the CODECOV_CI environment variable is set to 'true' +codecov_ci = run_command('sh', ['-c', ''' +if [ "$CODECOV_CI" = "true" ]; then + echo "true" +else + echo "false" +fi +'''], check: true).stdout().strip() + +# Conditionally set install_dir based on the CODECOV_CI variable +if codecov_ci == 'true' + install_dir = run_command('sh', ['-c', ''' + if [ -z "$PYTHONPATH" ]; then + python -c "import sysconfig; print(sysconfig.get_path('purelib'))" + else + echo $PYTHONPATH + fi + '''], check: true).stdout().strip() + + # Ensure the install_dir is not empty + if install_dir == '' + error('Could not determine install_dir from PYTHONPATH or sysconfig.get_path().') + endif +else + install_dir = py.get_install_dir(pure: false) +endif + +message('Installing to: ' + install_dir) +install_subdir('dlup', install_dir : install_dir) + +_background = py.extension_module('_background', + 'dlup/_background.pyx', + include_directories : [incdir_numpy], + install : true, + install_dir : install_dir / 'dlup', + cpp_args : ['-O3', '-march=native', '-ffast-math']) + +# Define the base dependencies and compiler arguments +base_deps = [libtiff_dep] +base_cpp_args = ['-std=c++17', '-O3', '-march=native', '-ffast-math'] + +# Add ZSTD support if available +if have_zstd + base_deps += [zstd_dep] + base_cpp_args += ['-DHAVE_ZSTD'] +endif + +# pybind11 extension +_libtiff_tiff_writer = py.extension_module('_libtiff_tiff_writer', + 'src/libtiff_tiff_writer.cpp', + include_directories : [incdir_pybind11], + install : true, + install_dir : install_dir / 'dlup', + cpp_args : base_cpp_args, + dependencies : base_deps) diff --git a/pyproject.toml b/pyproject.toml index 5e39a4e2..e48d8941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,51 +1,129 @@ -# Example configuration for Black, edited for DLUP. +[build-system] +build-backend = "mesonpy" +requires = [ + "meson-python>=0.15.0", + "Cython>=0.29", + "numpy==1.26.4", + "pybind11", + "ninja", +] + +[project] +name = "dlup" +dynamic = ["version"] +description = "A package for digital pathology image analysis" +authors = [{name = "Jonas Teuwen", email = "j.teuwen@nki.nl"}] +maintainers = [ + {name = "DLUP Developers", email="j.teuwen@nki.nl"}, +] +requires-python = ">=3.10" +readme = "README.md" +license = {file = "LICENSE"} +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: Image Processing", + "Operating System :: OS Independent", +] +dependencies = [ + "numpy==1.26.4", + "tifftools>=1.5.2", + "tifffile>=2024.7.2", + "pyvips>=2.2.3", + "tqdm>=2.66.4", + "pillow>=10.3.0", + "openslide-python>=1.3.1", + "opencv-python-headless>=4.9.0.80", + "shapely>=2.0.4", + "packaging>=24.0", + "pybind11>=2.8.0", +] + +[project.optional-dependencies] +dev = [ + "psutil", + "pytest>=8.2.1", + "mypy>=1.10.0", + "pytest-mock>=3.14.0", + "sphinx_copybutton>=0.5.2", + "numpydoc>=1.7.0", + "myst_parser>=3.0.1", + "sphinx-book-theme>=1.1.2", + "pylint>=3.2.2", + "pydantic>=2.7.2", + "types-Pillow>=10.2.0", + "darwin-py>=0.8.62", +] +darwin = ["darwin-py>=0.8.59"] + +[project.urls] +Homepage = "https://github.com/NKI-AI/dlup" +Documentation = "https://docs.aiforoncology.nl/dlup/" +Source = "https://github.com/NKI-AI/dlup" +"Bug Tracker" = "https://github.com/NKI-AI/dlup/issues" -# NOTE: you have to use single-quoted strings in TOML for regular expressions. -# It's the equivalent of r-strings in Python. Multiline strings are treated as -# verbose regular expressions by Black. Use [ ] to denote a significant space -# character. +[tool.spin] +package = 'dlup' + +[tool.spin.commands] +"Build" = [ + ".spin/cmds.py:build", + ".spin/cmds.py:test", + ".spin/cmds.py:mypy", + ".spin/cmds.py:lint", +] +"Environments" = [ + "spin.cmds.meson.run", + ".spin/cmds.py:ipython", + ".spin/cmds.py:python", +] +"Documentation" = [ + ".spin/cmds.py:docs", + ".spin/cmds.py:changelog", +] [tool.black] -line-length = 119 # PyCharm line length -target-version = ['py38', 'py39'] +line-length = 120 +target-version = ['py310', 'py311'] include = '\.pyi?$' exclude = ''' /( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | \.idea - | _build - | buck-out - | build - | dist + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | \.idea + | _build + | buck-out + | build + | dist )/ ''' [tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -line_length = 119 +profile = "black" +line_length = 120 [tool.pylint.format] max-line-length = "120" [tool.pylint.'TYPECHECK'] -generated-members=['numpy.*', 'torch.*', 'np.*', 'cv2.*', 'openslide.*'] +generated-members = ['numpy.*', 'torch.*', 'np.*', 'cv2.*', 'openslide.*'] [tool.pylint.master] extension-pkg-whitelist = ["dlup._background"] ignore-patterns = '.*\.pyi' -[build-system] -requires = ["setuptools>=45", "wheel", "Cython>=0.29", "numpy"] -build-backend = "setuptools.build_meta" - [tool.cython-lint] max-line-length = 120 + +[tool.pytest.ini_options] +addopts = "--ignore=libvips" diff --git a/setup.cfg b/setup.cfg index aeab5661..2ba24f8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,16 @@ [bumpversion] -current_version = 0.6.1 +current_version = 0.7.0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? -serialize = +serialize = {major}.{minor}.{patch}-{release}{build} {major}.{minor}.{patch} [bumpversion:part:release] optional_value = prod first_value = dev -values = +values = dev prod @@ -22,10 +22,18 @@ replace = {new_version} search = {current_version} replace = {new_version} +[bumpversion:file:meson.build] +search = {current_version} +replace = {new_version} + [bumpversion:file:CITATION.cff] search = {current_version} replace = {new_version} +[bumpversion:file:src/constants.h] +search = {current_version} +replace = {new_version} + [aliases] test = pytest diff --git a/setup.py b/setup.py deleted file mode 100644 index 109e75ed..00000000 --- a/setup.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -"""The setup script.""" - -import ast -from typing import Any - -from Cython.Build import cythonize # type: ignore -from setuptools import Extension, find_packages, setup # type: ignore - -with open("dlup/__init__.py") as f: - for line in f: - if line.startswith("__version__"): - version = ast.parse(line).body[0].value.s # type: ignore - break - - -# Get the long description from the README file -with open("README.md") as f: - LONG_DESCRIPTION = f.read() - -install_requires = [ - "numpy==1.26.4", - "tifftools>=1.5.2", - "tifffile>=2024.7.2", - "pyvips>=2.2.3", - "tqdm>=2.66.4", - "pillow>=10.3.0", - "openslide-python>=1.3.1", - "opencv-python>=4.9.0.80", - "shapely>=2.0.4", - "packaging>=24.0", -] - - -class NumpyImportDefer: - def __getattr__(self, attr: Any) -> Any: - import numpy - - return getattr(numpy, attr) - - -numpy = NumpyImportDefer() - -extension = Extension( - name="dlup._background", - sources=["dlup/_background.pyx"], - include_dirs=[numpy.get_include()], - extra_compile_args=["-O3", "-march=native", "-ffast-math"], - extra_link_args=["-O3"], -) - -setup( - author="Jonas Teuwen", - author_email="j.teuwen@nki.nl", - long_description=LONG_DESCRIPTION, - long_description_content_type="text/markdown", - python_requires=">=3.10", - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - entry_points={ - "console_scripts": [ - "dlup=dlup.cli:main", - ], - }, - setup_requires=["Cython>=0.29"], - install_requires=install_requires, - extras_require={ - "dev": [ - "psutil", - "pytest>=8.2.1", - "mypy>=1.10.0", - "pytest-mock>=3.14.0", - "sphinx_copybutton>=0.5.2", - "numpydoc>=1.7.0", - "myst_parser>=3.0.1", - "sphinx-book-theme>=1.1.2", - "pylint>=3.2.2", - "pydantic>=2.7.2", - "types-Pillow>=10.2.0", - "darwin-py>=0.8.62", - ], - "darwin": ["darwin-py>=0.8.59"], - }, - license="Apache Software License 2.0", - include_package_data=True, - keywords="dlup", - name="dlup", - packages=find_packages(include=["dlup", "dlup.*"]), - url="https://github.com/NKI-AI/dlup", - version=version, - ext_modules=cythonize([extension], compiler_directives={"language_level": "3"}), - include_dirs=[numpy.get_include()], - zip_safe=False, -) diff --git a/src/constants.h b/src/constants.h new file mode 100644 index 00000000..c863ece5 --- /dev/null +++ b/src/constants.h @@ -0,0 +1 @@ +#define DLUP_VERSION "0.7.0" diff --git a/src/image.h b/src/image.h new file mode 100644 index 00000000..01541a16 --- /dev/null +++ b/src/image.h @@ -0,0 +1,37 @@ +// image.h + +#ifndef IMAGE_H +#define IMAGE_H + +#include +#include +#include + +namespace image_utils { + +void downsample2x2(const std::vector &input, uint32_t inputWidth, uint32_t inputHeight, + std::vector &output, uint32_t outputWidth, uint32_t outputHeight, int channels) { + for (uint32_t y = 0; y < outputHeight; ++y) { + for (uint32_t x = 0; x < outputWidth; ++x) { + for (int c = 0; c < channels; ++c) { + uint32_t sum = 0; + uint32_t count = 0; + for (uint32_t dy = 0; dy < 2; ++dy) { + for (uint32_t dx = 0; dx < 2; ++dx) { + uint32_t sx = 2 * x + dx; + uint32_t sy = 2 * y + dy; + if (sx < inputWidth && sy < inputHeight) { + sum += std::to_integer(input[(sy * inputWidth + sx) * channels + c]); + ++count; + } + } + } + output[(y * outputWidth + x) * channels + c] = static_cast(sum / count); + } + } + } +} + +} // namespace image_utils + +#endif // IMAGE_H diff --git a/src/libtiff_tiff_writer.cpp b/src/libtiff_tiff_writer.cpp new file mode 100644 index 00000000..7ec3bf2b --- /dev/null +++ b/src/libtiff_tiff_writer.cpp @@ -0,0 +1,514 @@ +#include "constants.h" +#include "image.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ZSTD +#include +#endif + +namespace fs = std::filesystem; +namespace py = pybind11; + +class TiffException : public std::runtime_error { +public: + explicit TiffException(const std::string &message) : std::runtime_error(message) {} +}; + +class TiffCompressionNotSupportedError : public TiffException { +public: + explicit TiffCompressionNotSupportedError(const std::string &message) + : TiffException("Compression not supported: " + message) {} +}; + +class TiffOpenException : public TiffException { +public: + explicit TiffOpenException(const std::string &message) : TiffException("Failed to open TIFF file: " + message) {} +}; + +class TiffWriteException : public TiffException { +public: + explicit TiffWriteException(const std::string &message) : TiffException("Failed to write TIFF data: " + message) {} +}; + +class TiffSetupException : public TiffException { +public: + explicit TiffSetupException(const std::string &message) : TiffException("Failed to setup TIFF: " + message) {} +}; + +class TiffReadException : public TiffException { +public: + explicit TiffReadException(const std::string &message) : TiffException("Failed to read TIFF data: " + message) {} +}; + +enum class CompressionType { NONE, JPEG, LZW, DEFLATE, ZSTD }; + +CompressionType string_to_compression_type(const std::string &compression) { + if (compression == "NONE") + return CompressionType::NONE; + if (compression == "JPEG") + return CompressionType::JPEG; + if (compression == "LZW") + return CompressionType::LZW; + if (compression == "DEFLATE") + return CompressionType::DEFLATE; + if (compression == "ZSTD") + return CompressionType::ZSTD; + throw std::invalid_argument("Invalid compression type: " + compression); +} + +struct TIFFDeleter { + void operator()(TIFF *tif) const noexcept { + if (tif) { + // Disable error reporting temporarily + TIFFErrorHandler oldHandler = TIFFSetErrorHandler(nullptr); + + // Attempt to flush any pending writes + if (TIFFFlush(tif) == 0) { + TIFFError("TIFFDeleter", "Failed to flush TIFF data"); + } + + TIFFClose(tif); + TIFFSetErrorHandler(oldHandler); + } + } +}; + +using TIFFPtr = std::unique_ptr; + +class LibtiffTiffWriter { +public: + LibtiffTiffWriter(fs::path filename, std::array imageSize, std::array mpp, + std::array tileSize, CompressionType compression = CompressionType::JPEG, + int quality = 100) + : filename(std::move(filename)), imageSize(imageSize), mpp(mpp), tileSize(tileSize), compression(compression), + quality(quality), tif(nullptr) { + + validateInputs(); + + TIFF *tiff_ptr = TIFFOpen(this->filename.c_str(), "w"); + if (!tiff_ptr) { + throw TiffOpenException("Unable to create TIFF file"); + } + tif.reset(tiff_ptr); + + setupTIFFDirectory(0); + } + + ~LibtiffTiffWriter(); + void writeTile(py::array_t tile, int row, int col); + void flush(); + void finalize(); + void writePyramid(); + +private: + std::string filename; + std::array imageSize; + std::array mpp; + std::array tileSize; + CompressionType compression; + int quality; + uint32_t tileCounter; + int numLevels = calculateLevels(); + TIFFPtr tif; + + void validateInputs() const; + int calculateLevels(); + std::pair calculateTiles(int level); + uint32_t calculateNumTiles(int level); + void setupTIFFDirectory(int level); + void writeTIFFDirectory(); + void writeDownsampledResolutionPage(int level); + + std::pair getLevelDimensions(int level); + std::vector read2x2TileGroup(TIFF *readTif, uint32_t row, uint32_t col, uint32_t prevWidth, + uint32_t prevHeight); + void setupReadTIFF(TIFF *readTif); +}; + +LibtiffTiffWriter::~LibtiffTiffWriter() { finalize(); } + +void LibtiffTiffWriter::writeTile(py::array_t tile, int row, + int col) { + auto numTiles = calculateNumTiles(0); + if (tileCounter >= numTiles) { + throw TiffWriteException("all tiles have already been written"); + } + auto buf = tile.request(); + if (buf.ndim < 2 || buf.ndim > 3) { + throw TiffWriteException("invalid number of dimensions in tile data. Expected 2 or 3, got " + + std::to_string(buf.ndim)); + } + auto [height, width, channels] = std::tuple{buf.shape[0], buf.shape[1], buf.ndim > 2 ? buf.shape[2] : 1}; + + // Verify dimensions and buffer size + size_t expected_size = static_cast(width) * height * channels; + if (static_cast(buf.size) != expected_size) { + throw TiffWriteException("buffer size does not match expected size. Expected " + std::to_string(expected_size) + + ", got " + std::to_string(buf.size)); + } + + // Check if tile coordinates are within bounds + if (row < 0 || row >= imageSize[0] || col < 0 || col >= imageSize[1]) { + auto [imageWidth, imageHeight] = getLevelDimensions(0); + throw TiffWriteException("tile coordinates out of bounds for row " + std::to_string(row) + ", col " + + std::to_string(col) + ". Image size is " + std::to_string(imageWidth) + "x" + + std::to_string(imageHeight)); + } + + // Write the tile + if (TIFFWriteTile(tif.get(), buf.ptr, col, row, 0, 0) < 0) { + throw TiffWriteException("TIFFWriteTile failed for row " + std::to_string(row) + ", col " + + std::to_string(col)); + } + tileCounter++; + if (tileCounter == numTiles) { + flush(); + } +} + +void LibtiffTiffWriter::validateInputs() const { + // check positivity of image size + if (imageSize[0] <= 0 || imageSize[1] <= 0 || imageSize[2] <= 0) { + throw std::invalid_argument("Invalid size parameters"); + } + + // check positivity of mpp + if (mpp[0] <= 0 || mpp[1] <= 0) { + throw std::invalid_argument("Invalid mpp value"); + } + + // check positivity of tile size + if (tileSize[0] <= 0 || tileSize[1] <= 0) { + throw std::invalid_argument("Invalid tile size"); + } + + // check quality parameter + if (quality < 0 || quality > 100) { + throw std::invalid_argument("Invalid quality value"); + } + + // check if tile size is power of two + if ((tileSize[0] & (tileSize[0] - 1)) != 0 || (tileSize[1] & (tileSize[1] - 1)) != 0) { + throw std::invalid_argument("Tile size must be a power of two"); + } +} + +int LibtiffTiffWriter::calculateLevels() { + int maxDim = std::max(imageSize[0], imageSize[1]); + int minTileDim = std::min(tileSize[0], tileSize[1]); + int numLevels = 1; + while (maxDim > minTileDim * 2) { + maxDim /= 2; + numLevels++; + } + return numLevels; +} + +std::pair LibtiffTiffWriter::calculateTiles(int level) { + auto [currentWidth, currentHeight] = getLevelDimensions(level); + auto [tileWidth, tileHeight] = tileSize; + + uint32_t numTilesX = (currentWidth + tileWidth - 1) / tileWidth; + uint32_t numTilesY = (currentHeight + tileHeight - 1) / tileHeight; + return {numTilesX, numTilesY}; +} + +uint32_t LibtiffTiffWriter::calculateNumTiles(int level) { + auto [numTilesX, numTilesY] = calculateTiles(level); + return numTilesX * numTilesY; +} + +std::pair LibtiffTiffWriter::getLevelDimensions(int level) { + uint32_t levelWidth = std::max(1, imageSize[1] >> level); + uint32_t levelHeight = std::max(1, imageSize[0] >> level); + return {levelWidth, levelHeight}; +} + +void LibtiffTiffWriter::flush() { + if (tif) { + if (TIFFFlush(tif.get()) != 1) { + throw TiffWriteException("failed to flush TIFF file"); + } + } +} + +void LibtiffTiffWriter::finalize() { + if (tif) { + // Only write directory if we haven't written all directories yet + if (TIFFCurrentDirectory(tif.get()) < TIFFNumberOfDirectories(tif.get()) - 1) { + TIFFWriteDirectory(tif.get()); + } + TIFFClose(tif.get()); + tif.release(); + } +} + +void LibtiffTiffWriter::setupReadTIFF(TIFF *readTif) { + auto set_field = [readTif](uint32_t tag, auto... value) { + if (TIFFSetField(readTif, tag, value...) != 1) { + throw TiffSetupException("failed to set TIFF field for reading: " + std::to_string(tag)); + } + }; + + uint16_t compression; + if (TIFFGetField(readTif, TIFFTAG_COMPRESSION, &compression) == 1) { + if (compression == COMPRESSION_JPEG) { + set_field(TIFFTAG_JPEGCOLORMODE, JPEGCOLORMODE_RGB); + } + } +} + +void LibtiffTiffWriter::setupTIFFDirectory(int level) { + auto set_field = [this](uint32_t tag, auto... value) { + if (TIFFSetField(tif.get(), tag, value...) != 1) { + throw TiffSetupException("failed to set TIFF field: " + std::to_string(tag)); + } + }; + + auto [width, height] = getLevelDimensions(level); + int channels = imageSize[2]; + + set_field(TIFFTAG_IMAGEWIDTH, width); + set_field(TIFFTAG_IMAGELENGTH, height); + set_field(TIFFTAG_SAMPLESPERPIXEL, channels); + set_field(TIFFTAG_BITSPERSAMPLE, 8); + set_field(TIFFTAG_ORIENTATION, ORIENTATION_TOPLEFT); + set_field(TIFFTAG_PLANARCONFIG, PLANARCONFIG_CONTIG); + set_field(TIFFTAG_TILEWIDTH, tileSize[1]); + set_field(TIFFTAG_TILELENGTH, tileSize[0]); + + if (channels == 3 || channels == 4) { + if (compression != CompressionType::JPEG) { + set_field(TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB); + } + } else { + set_field(TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_MINISBLACK); + } + + if (channels == 4) { + uint16_t extra_samples = EXTRASAMPLE_ASSOCALPHA; + set_field(TIFFTAG_EXTRASAMPLES, 1, &extra_samples); + } else if (channels > 4) { + std::vector extra_samples(channels - 3, EXTRASAMPLE_UNSPECIFIED); + set_field(TIFFTAG_EXTRASAMPLES, channels - 3, extra_samples.data()); + } + + switch (compression) { + case CompressionType::NONE: + set_field(TIFFTAG_COMPRESSION, COMPRESSION_NONE); + break; + case CompressionType::JPEG: + set_field(TIFFTAG_COMPRESSION, COMPRESSION_JPEG); + set_field(TIFFTAG_JPEGQUALITY, quality); + set_field(TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_YCBCR); + set_field(TIFFTAG_YCBCRSUBSAMPLING, 2, 2); + set_field(TIFFTAG_JPEGCOLORMODE, JPEGCOLORMODE_RGB); + break; + case CompressionType::LZW: + set_field(TIFFTAG_COMPRESSION, COMPRESSION_LZW); + break; + case CompressionType::DEFLATE: + set_field(TIFFTAG_COMPRESSION, COMPRESSION_ADOBE_DEFLATE); + break; + case CompressionType::ZSTD: +#ifdef HAVE_ZSTD + set_field(TIFFTAG_COMPRESSION, COMPRESSION_ZSTD); + set_field(TIFFTAG_ZSTD_LEVEL, 3); // 3 is the default + break; +#else + throw TiffCompressionNotSupportedError("ZSTD"); +#endif + default: + throw TiffSetupException("Unknown compression type"); + } + + // Convert mpp (micrometers per pixel) to pixels per centimeter + double pixels_per_cm_x = 10000.0 / mpp[0]; + double pixels_per_cm_y = 10000.0 / mpp[1]; + + set_field(TIFFTAG_RESOLUTIONUNIT, RESUNIT_CENTIMETER); + set_field(TIFFTAG_XRESOLUTION, pixels_per_cm_x); + set_field(TIFFTAG_YRESOLUTION, pixels_per_cm_y); + + // Set the image description + // TODO: This needs to be configurable + std::string description = "TODO"; + // set_field(TIFFTAG_IMAGEDESCRIPTION, description.c_str()); + + // Set the software tag with version from dlup + std::string software_tag = + "dlup " + std::string(DLUP_VERSION) + " (libtiff " + std::to_string(TIFFLIB_VERSION) + ")"; + set_field(TIFFTAG_SOFTWARE, software_tag.c_str()); + + // Set SubFileType for pyramid levels + if (level == 0) { + set_field(TIFFTAG_SUBFILETYPE, 0); + } else { + set_field(TIFFTAG_SUBFILETYPE, FILETYPE_REDUCEDIMAGE); + } +} + +std::vector LibtiffTiffWriter::read2x2TileGroup(TIFF *readTif, uint32_t row, uint32_t col, + uint32_t prevWidth, uint32_t prevHeight) { + auto [tileWidth, tileHeight] = tileSize; + int channels = imageSize[2]; + uint32_t fullGroupWidth = 2 * tileWidth; + uint32_t fullGroupHeight = 2 * tileHeight; + + // Initialize a zero buffer for the 2x2 group + std::vector groupBuffer(fullGroupWidth * fullGroupHeight * channels, std::byte(0)); + + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 2; ++j) { + uint32_t tileRow = row + i * tileHeight; + uint32_t tileCol = col + j * tileWidth; + + // Skip if this tile is out of bounds, this can happen when the image dimensions are smaller than 2x2 in + // tileSize + if (tileRow >= prevHeight || tileCol >= prevWidth) { + continue; + } + + std::vector tileBuf(TIFFTileSize(readTif)); + + if (TIFFReadTile(readTif, tileBuf.data(), tileCol, tileRow, 0, 0) < 0) { + throw TiffReadException("failed to read tile at row " + std::to_string(tileRow) + ", col " + + std::to_string(tileCol)); + } + + // Copy tile data to groupBuffer + uint32_t copyWidth = std::min(tileWidth, prevWidth - tileCol); + uint32_t copyHeight = std::min(tileHeight, prevHeight - tileRow); + for (uint32_t y = 0; y < copyHeight; ++y) { + for (uint32_t x = 0; x < copyWidth; ++x) { + for (int c = 0; c < channels; ++c) { + size_t groupIndex = + ((i * tileHeight + y) * fullGroupWidth + (j * tileWidth + x)) * channels + c; + size_t tileIndex = (y * tileWidth + x) * channels + c; + groupBuffer[groupIndex] = static_cast(tileBuf[tileIndex]); + } + } + } + } + } + + return groupBuffer; +} +void LibtiffTiffWriter::writeDownsampledResolutionPage(int level) { + if (level <= 0 || level >= numLevels) { + throw std::invalid_argument("Invalid level for downsampled resolution page"); + } + + auto [prevWidth, prevHeight] = getLevelDimensions(level - 1); + int channels = imageSize[2]; + auto [tileWidth, tileHeight] = tileSize; + + TIFFPtr readTif(TIFFOpen(filename.c_str(), "r")); + if (!readTif) { + throw TiffOpenException("failed to open TIFF file for reading"); + } + + if (!TIFFSetDirectory(readTif.get(), level - 1)) { + throw TiffReadException("failed to set directory to level " + std::to_string(level - 1)); + } + setupReadTIFF(readTif.get()); + + if (!TIFFSetDirectory(tif.get(), level - 1)) { + throw TiffReadException("failed to set directory to level " + std::to_string(level - 1)); + } + + if (!TIFFWriteDirectory(tif.get())) { + throw TiffWriteException("failed to create new directory for downsampled image"); + } + + setupTIFFDirectory(level); + + auto [numTilesX, numTilesY] = calculateTiles(level); + + for (uint32_t tileY = 0; tileY < numTilesY; ++tileY) { + for (uint32_t tileX = 0; tileX < numTilesX; ++tileX) { + uint32_t row = tileY * tileHeight * 2; + uint32_t col = tileX * tileWidth * 2; + + std::vector groupBuffer = read2x2TileGroup(readTif.get(), row, col, prevWidth, prevHeight); + std::vector downsampledBuffer(tileHeight * tileWidth * channels); + + image_utils::downsample2x2(groupBuffer, 2 * tileWidth, 2 * tileHeight, downsampledBuffer, tileWidth, + tileHeight, channels); + + if (TIFFWriteTile(tif.get(), reinterpret_cast(downsampledBuffer.data()), tileX * tileWidth, + tileY * tileHeight, 0, 0) < 0) { + throw TiffWriteException("failed to write downsampled tile at level " + std::to_string(level) + + ", row " + std::to_string(tileY) + ", col " + std::to_string(tileX)); + } + } + } + + readTif.reset(); + flush(); +} + +void LibtiffTiffWriter::writePyramid() { + numLevels = calculateLevels(); + + // The base level (level 0) is already written, so we start from level 1 + for (int level = 1; level < numLevels; ++level) { + writeDownsampledResolutionPage(level); + flush(); + } +} + +PYBIND11_MODULE(_libtiff_tiff_writer, m) { + py::class_(m, "LibtiffTiffWriter") + .def(py::init([](py::object path, std::array size, std::array mpp, + std::array tileSize, py::object compression, int quality) { + fs::path cpp_path; + if (py::isinstance(path)) { + cpp_path = fs::path(path.cast()); + } else if (py::hasattr(path, "__fspath__")) { + cpp_path = fs::path(path.attr("__fspath__")().cast()); + } else { + throw py::type_error("Expected str or os.PathLike object"); + } + + CompressionType comp_type; + if (py::isinstance(compression)) { + comp_type = string_to_compression_type(compression.cast()); + } else if (py::isinstance(compression)) { + comp_type = compression.cast(); + } else { + throw py::type_error("Expected str or CompressionType for compression"); + } + + return new LibtiffTiffWriter(std::move(cpp_path), size, mpp, tileSize, comp_type, quality); + })) + .def("write_tile", &LibtiffTiffWriter::writeTile) + .def("write_pyramid", &LibtiffTiffWriter::writePyramid) + .def("finalize", &LibtiffTiffWriter::finalize); + + py::enum_(m, "CompressionType") + .value("NONE", CompressionType::NONE) + .value("JPEG", CompressionType::JPEG) + .value("LZW", CompressionType::LZW) + .value("DEFLATE", CompressionType::DEFLATE); + + py::register_exception(m, "TiffException"); + py::register_exception(m, "TiffOpenException"); + py::register_exception(m, "TiffReadException"); + py::register_exception(m, "TiffWriteException"); + py::register_exception(m, "TiffSetupException"); + py::register_exception(m, "TiffCompressionNotSupportedError"); +} diff --git a/tests/backends/test_openslide_backend.py b/tests/backends/test_openslide_backend.py index ce39b63f..f1ac3924 100644 --- a/tests/backends/test_openslide_backend.py +++ b/tests/backends/test_openslide_backend.py @@ -7,7 +7,7 @@ import pytest import pyvips -from dlup import UnsupportedSlideError +from dlup._exceptions import UnsupportedSlideError from dlup.backends.openslide_backend import ( TIFF_PROPERTY_NAME_RESOLUTION_UNIT, TIFF_PROPERTY_NAME_X_RESOLUTION, @@ -15,7 +15,7 @@ OpenSlideSlide, _get_mpp_from_tiff, ) -from dlup.types import PathLike +from dlup._types import PathLike from ..common import SlideConfig, get_sample_nonuniform_image diff --git a/tests/common.py b/tests/common.py index 007d43cc..8b231165 100644 --- a/tests/common.py +++ b/tests/common.py @@ -65,7 +65,7 @@ def get_sample_nonuniform_image(size: tuple[int, int] = (256, 256), divisions: i cell_height = height // y_divisions # Create an array to store the image - image_array = np.zeros((height, width, 4), dtype=np.uint8) + image_array = np.zeros((height, width, 4), dtype=float) # Define a set of distinct colors color_palette = [ @@ -105,4 +105,4 @@ def get_sample_nonuniform_image(size: tuple[int, int] = (256, 256), divisions: i for k in range(3): # Apply only to RGB channels, not alpha image_array[:, :, k] = image_array[:, :, k] * sine_wave - return pyvips.Image.new_from_array(image_array) + return pyvips.Image.new_from_array(image_array.astype(np.uint8)) diff --git a/tests/test_background.py b/tests/test_background.py index 4193d9e0..2da62f88 100644 --- a/tests/test_background.py +++ b/tests/test_background.py @@ -59,9 +59,7 @@ def test_wsiannotations(self, dlup_wsi, threshold): # Let's make a shapely polygon thats equal to # background_mask[14:20, 10:20] = True # background_mask[85:100, 50:80] = True - polygon0 = Polygon( - box(100, 140, 200, 200), AnnotationClass(annotation_type=AnnotationType.POLYGON, label="bg") - ) + polygon0 = Polygon(box(100, 140, 200, 200), AnnotationClass(annotation_type=AnnotationType.POLYGON, label="bg")) polygon1 = Polygon( box(500, 850, 800, 1000), AnnotationClass(annotation_type=AnnotationType.POLYGON, label="bg") ) diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 29e56b2e..00000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,39 +0,0 @@ -import pathlib -from argparse import ArgumentTypeError -from unittest.mock import patch - -import pytest - -from dlup.cli import dir_path, file_path, main - - -def test_dir_path_valid_directory(tmpdir): - path = tmpdir.mkdir("subdir") - assert dir_path(str(path)) == pathlib.Path(path) - - -def test_dir_path_invalid_directory(): - with pytest.raises(ArgumentTypeError): - dir_path("/path/which/does/not/exist") - - -def test_file_path_valid_file(tmpdir): - path = tmpdir.join("test_file.txt") - path.write("content") - assert file_path(str(path)) == pathlib.Path(path) - - -def test_file_path_invalid_file(): - with pytest.raises(ArgumentTypeError): - file_path("/path/which/does/not/exist.txt") - - -def test_file_path_no_need_exists(): - _path = "/path/which/does/not/need/to/exist.txt" - assert file_path(_path, need_exists=False) == pathlib.Path(_path) - - -def test_main_no_arguments(capsys): - with patch("sys.argv", ["dlup"]): - with pytest.raises(SystemExit): - main() diff --git a/tests/test_image.py b/tests/test_image.py index 46ba7376..3c131ecb 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -14,7 +14,8 @@ import pytest import pyvips -from dlup import SlideImage, UnsupportedSlideError +from dlup import SlideImage +from dlup._exceptions import UnsupportedSlideError from .backends.test_openslide_backend import SLIDE_CONFIGS, MockOpenSlideSlide, SlideConfig diff --git a/tests/test_writers.py b/tests/test_writers.py index a2760e4f..fb19eeea 100644 --- a/tests/test_writers.py +++ b/tests/test_writers.py @@ -1,18 +1,16 @@ # Copyright (c) dlup contributors import tempfile -import warnings import numpy as np -import openslide import pytest import pyvips -from packaging.version import Version from PIL import Image, ImageColor from dlup import SlideImage -from dlup.backends import ImageBackend, OpenSlideSlide, PyVipsSlide +from dlup.backends import OpenSlideSlide, PyVipsSlide from dlup.backends.pyvips_backend import open_slide as open_pyvips_slide -from dlup.writers import TiffCompression, TifffileImageWriter, _color_dict_to_color_lut +from dlup.utils.backends import ImageBackend +from dlup.writers import LibtiffImageWriter, TiffCompression, TifffileImageWriter, _color_dict_to_color_lut COLORMAP = { 1: "green", @@ -66,16 +64,14 @@ def test_tiff_writer(self, shape, target_mpp): # TODO, let's make a test like this with a mockup too # Let's force it to open with openslide - with PyVipsSlide(temp_tiff.name) as slide0, PyVipsSlide( - temp_tiff.name, load_with_openslide=True - ) as slide1: + with PyVipsSlide(temp_tiff.name) as slide0, PyVipsSlide(temp_tiff.name, load_with_openslide=True) as slide1: assert slide0._loader == "tiffload" assert slide1._loader == "openslideload" - if Version(openslide.__library_version__) < Version("4.0.0"): - warnings.warn("Openslide version is too old, skipping some tests.") - else: - assert np.allclose(slide0.spacing, slide1.spacing) + if not slide1.spacing: + slide1.spacing = slide0.spacing + + assert np.allclose(slide0.spacing, slide1.spacing) assert slide0.level_count == slide1.level_count assert slide0.dimensions == slide1.dimensions assert np.allclose(slide0.level_downsamples, slide1.level_downsamples) @@ -97,8 +93,9 @@ def test_tiff_writer(self, shape, target_mpp): slide_mpp = slide.mpp assert np.allclose(slide_mpp, target_mpp) + @pytest.mark.parametrize("writer_class", [TifffileImageWriter, LibtiffImageWriter]) @pytest.mark.parametrize("pyramid", [True, False]) - def test_tiff_writer_pyramid(self, pyramid): + def test_tiff_writer_pyramid(self, writer_class, pyramid): shape = (1010, 2173, 3) target_mpp = 1.0 tile_size = (128, 128) @@ -108,7 +105,7 @@ def test_tiff_writer_pyramid(self, pyramid): size = (*pil_image.size, 3) with tempfile.NamedTemporaryFile(suffix=".tiff") as temp_tiff: - writer = TifffileImageWriter( + writer = writer_class( temp_tiff.name, size=size, mpp=(target_mpp, target_mpp), @@ -122,7 +119,10 @@ def test_tiff_writer_pyramid(self, pyramid): n_pages = vips_image.get("n-pages") - assert n_pages == int(np.ceil(np.log2(np.asarray(size[:-1]) / np.asarray([tile_size]))).min()) + 1 + if writer_class == TifffileImageWriter: + assert n_pages == int(np.ceil(np.log2(np.asarray(size[:-1]) / np.asarray([tile_size]))).min()) + 1 + else: + assert n_pages == int(np.ceil(np.log2(np.asarray(size[:-1]) / np.asarray([tile_size]))).max()) assert vips_image.get("xres") == 1000.0 and vips_image.get("yres") == 1000.0 for page in range(1, n_pages): diff --git a/tox.ini b/tox.ini index e3fcb0ba..554cf1f1 100644 --- a/tox.ini +++ b/tox.ini @@ -4,12 +4,20 @@ isolated_build = True [testenv] deps = - numpy + meson + meson-python>=0.15.0 + numpy==1.26.4 Cython>=0.29 + spin + pybind11 + build extras = dev,darwin commands = - pip install -e . + sh -c 'meson setup builddir' + sh -c 'meson compile -C builddir' + sh -c 'cp builddir/*.so dlup' pytest allowlist_externals = + sh pytest pip