draft of multi-tiff multi-page imaging extractor

requirements-minimal.txt

@@ -6,3 +6,4 @@ dill>=0.3.2
 scipy>=1.5.2
 psutil>=5.8.0
 PyYAML
+parse>=1.20.0

src/roiextractors/__init__.py

@@ -2,10 +2,10 @@
 
 from importlib.metadata import version
 
-__version__ = version("roiextractors")
-
 from .example_datasets import toy_example
 from .extraction_tools import show_video
 from .extractorlist import *
 from .imagingextractor import ImagingExtractor
 from .segmentationextractor import SegmentationExtractor
+
+__version__ = version("roiextractors")

src/roiextractors/extractorlist.py

@@ -23,6 +23,8 @@
     BrukerTiffMultiPlaneImagingExtractor,
     BrukerTiffSinglePlaneImagingExtractor,
     MicroManagerTiffImagingExtractor,
+    MultiTiffImagingExtractor,
+    FolderTiffImagingExtractor,
 )
 from .extractors.sbximagingextractor import SbxImagingExtractor
 from .extractors.inscopixextractors import InscopixImagingExtractor
@@ -51,6 +53,8 @@
     NumpyMemmapImagingExtractor,
     MemmapImagingExtractor,
     VolumetricImagingExtractor,
+    MultiTiffImagingExtractor,
+    FolderTiffImagingExtractor,
     InscopixImagingExtractor,
 ]
 

src/roiextractors/extractors/tiffimagingextractors/__init__.py

@@ -33,7 +33,7 @@
     Specialized extractor for reading TIFF files produced via Micro-Manager.
 """
 
-from .tiffimagingextractor import TiffImagingExtractor
+from .tiffimagingextractor import TiffImagingExtractor, MultiTiffImagingExtractor, FolderTiffImagingExtractor
 from .scanimagetiffimagingextractor import (
     ScanImageTiffImagingExtractor,
     ScanImageTiffMultiPlaneImagingExtractor,

src/roiextractors/extractors/tiffimagingextractors/tiffimagingextractor.py

@@ -14,13 +14,15 @@
 import numpy as np
 from tqdm import tqdm
 
+from ...multiimagingextractor import MultiImagingExtractor
 from ...imagingextractor import ImagingExtractor
 from ...extraction_tools import (
     PathType,
     FloatType,
     raise_multi_channel_or_depth_not_implemented,
     get_package,
 )
+from ...utils import match_paths
 
 
 class TiffImagingExtractor(ImagingExtractor):
@@ -151,3 +153,56 @@ def write_imaging(imaging, save_path, overwrite: bool = False, chunk_size=None,
                     )
                     chunk_frames = np.squeeze(video)
                     tif.save(chunk_frames, contiguous=True, metadata=None)
+
+
+class MultiTiffImagingExtractor(MultiImagingExtractor):
+    """A ImagingExtractor for multiple TIFF files that each have multiple pages."""
+
+    extractor_name = "multi-tiff multi-page Imaging Extractor"
+    is_writable = False
+
+    def __init__(self, file_paths: list[str], sampling_frequency: float):
+        """Create a MultiTiffImagingExtractor instance.
+
+        Parameters
+        ----------
+        file_paths: list of str
+            List of paths to the TIFF files.
+        sampling_frequency : float
+            The frequency at which the frames were sampled, in Hz.
+        """
+        self.file_paths = file_paths
+        imaging_extractors = [
+            TiffImagingExtractor(file_path=x, sampling_frequency=sampling_frequency) for x in self.file_paths
+        ]
+        super().__init__(imaging_extractors=imaging_extractors)
+        self._kwargs.update({"file_paths": file_paths})
+
+
+class FolderTiffImagingExtractor(MultiTiffImagingExtractor):
+    """A ImagingExtractor for multiple TIFF files in a folder that each have multiple pages."""
+
+    extractor_name = "folder-tiff multi-page Imaging Extractor"
+    is_writable = False
+
+    def __init__(self, folder_path: PathType, pattern: str, sampling_frequency: float):
+        """Create a FolderTiffImagingExtractor instance.
+
+        Parameters
+        ----------
+        folder_path: PathType
+            Path to the folder containing the TIFF files.
+        pattern : str
+            The f-string pattern to match the TIFF files in the folder.
+        sampling_frequency : float
+            The frequency at which the frames were sampled, in Hz.
+        """
+        folder_path = Path(folder_path)
+        file_paths = match_paths(str(folder_path), pattern)
+        super().__init__(file_paths=file_paths, sampling_frequency=sampling_frequency)
+        self._kwargs.update(
+            {
+                "folder_path": str(folder_path.absolute()),
+                "pattern": pattern,
+            }
+        )

src/roiextractors/utils.py

@@ -0,0 +1,38 @@
+"""Utility functions for the ROIExtractors package."""
+
+import glob
+import os
+from typing import Dict, Any
+
+from parse import parse
+
+
+def match_paths(base: str, pattern: str, sort=True) -> Dict[str, Dict[str, Any]]:
+    """
+    Match paths in a directory to a pattern.
+
+    Parameters
+    ----------
+    base: str
+        The base directory to search in.
+    pattern: str
+        The f-string pattern to match the paths to.
+    sort: bool, default=True
+        Whether to sort the output by the values of the named groups in the pattern.
+
+    Returns
+    -------
+    dict
+    """
+    full_pattern = os.path.join(base, pattern)
+    paths = glob.glob(os.path.join(base, "*"))
+    out = {}
+    for path in paths:
+        parsed = parse(full_pattern, path)
+        if parsed is not None:
+            out[path] = parsed.named
+
+    if sort:
+        out = dict(sorted(out.items(), key=lambda item: tuple(item[1].values())))
+
+    return out

tests/test_internals/test_utils.py

@@ -0,0 +1,56 @@
+import os
+
+from roiextractors.utils import match_paths
+from tempfile import TemporaryDirectory
+
+
+def test_match_paths():
+    # create temporary directory
+    with TemporaryDirectory() as tmpdir:
+        # create temporary files
+        files = [
+            "split_1.tif",
+            "split_2.tif",
+            "split_3.tif",
+            "split_4.tif",
+            "split_5.tif",
+            "split_6.tif",
+            "split_7.tif",
+            "split_8.tif",
+            "split_9.tif",
+            "split_10.tif",
+        ]
+        for file in files:
+            with open(os.path.join(tmpdir, file), "w") as f:
+                f.write("")
+
+        # test match_paths
+        out = match_paths(tmpdir, "split_{split:d}.tif")
+        assert list(out.keys()) == [os.path.join(tmpdir, x) for x in files]
+        assert list([x["split"] for x in out.values()]) == list(range(1, 11))
+
+
+def test_match_paths_sub_select():
+    # create temporary directory
+    with TemporaryDirectory() as tmpdir:
+        # create temporary files
+        files = [
+            "chanA_split_1.tif",
+            "chanA_split_2.tif",
+            "chanA_split_3.tif",
+            "chanA_split_4.tif",
+            "chanA_split_5.tif",
+            "chanB_split_1.tif",
+            "chanB_split_2.tif",
+            "chanB_split_3.tif",
+            "chanB_split_4.tif",
+            "chanB_split_5.tif",
+        ]
+        for file in files:
+            with open(os.path.join(tmpdir, file), "w") as f:
+                f.write("")
+
+        # test match_paths
+        out = match_paths(tmpdir, "chanA_split_{split:d}.tif")
+        assert list(out.keys()) == [os.path.join(tmpdir, x) for x in files[:5]]
+        assert list([x["split"] for x in out.values()]) == list(range(1, 6))

tests/test_multitiff_multipage_imaging_extractor.py

@@ -0,0 +1,64 @@
+from roiextractors import MultiTiffImagingExtractor, FolderTiffImagingExtractor
+
+from tests.setup_paths import OPHYS_DATA_PATH
+
+
+def test_init_folder_tiff_imaging_extractor_multi_page():
+    extractor = FolderTiffImagingExtractor(
+        folder_path=OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits",
+        pattern="split_{split:d}.tif",
+        sampling_frequency=1.0,
+    )
+
+    assert extractor.get_num_channels() == 1
+    assert extractor.get_num_frames() == 2000
+    assert extractor.get_sampling_frequency() == 1.0
+    assert extractor.get_channel_names() is None
+    assert extractor.get_dtype() == "uint16"
+    assert extractor.get_image_size() == (60, 80)
+    assert extractor.get_video().shape == (2000, 60, 80)
+    assert list(extractor.file_paths) == [
+        str(OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x)
+        for x in (
+            "split_1.tif",
+            "split_2.tif",
+            "split_3.tif",
+            "split_4.tif",
+            "split_5.tif",
+            "split_6.tif",
+            "split_7.tif",
+            "split_8.tif",
+            "split_9.tif",
+            "split_10.tif",
+        )
+    ]
+
+
+def test_init_multitiff_imaging_extractor_multi_page():
+    extractor = MultiTiffImagingExtractor(
+        file_paths=[OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / f"split_{i}.tif" for i in range(1, 11)],
+        sampling_frequency=1.0,
+    )
+
+    assert extractor.get_num_channels() == 1
+    assert extractor.get_num_frames() == 2000
+    assert extractor.get_sampling_frequency() == 1.0
+    assert extractor.get_channel_names() is None
+    assert extractor.get_dtype() == "uint16"
+    assert extractor.get_image_size() == (60, 80)
+    assert extractor.get_video().shape == (2000, 60, 80)
+    assert list(extractor.file_paths) == [
+        OPHYS_DATA_PATH / "imaging_datasets" / "Tif" / "splits" / x
+        for x in (
+            "split_1.tif",
+            "split_2.tif",
+            "split_3.tif",
+            "split_4.tif",
+            "split_5.tif",
+            "split_6.tif",
+            "split_7.tif",
+            "split_8.tif",
+            "split_9.tif",
+            "split_10.tif",
+        )
+    ]