[ENH] Update config to support microscopy, qMRI, PET, ASL

.gitignore

@@ -1,6 +1,5 @@
 historical/
 
-#
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -86,6 +85,9 @@ celerybeat-schedule
 venv/
 ENV/
 
+# Pipenv
+Pipfile
+
 # Spyder project settings
 .spyderproject
 

bids/layout/config/bids.json

@@ -12,6 +12,10 @@
             "mandatory": false,
             "directory": "{subject}{session}"
         },
+        {
+            "name": "sample",
+            "pattern": "[_/\\\\]+sample-([a-zA-Z0-9]+)"
+        },
         {
             "name": "task",
             "pattern": "[_/\\\\]+task-([a-zA-Z0-9]+)"
@@ -24,6 +28,14 @@
             "name": "ceagent",
             "pattern": "[_/\\\\]+ce-([a-zA-Z0-9]+)"
         },
+        {
+            "name": "staining",
+            "pattern": "[_/\\\\]+stain-([a-zA-Z0-9]+)"
+        },
+        {
+            "name": "tracer",
+            "pattern": "[_/\\\\]+trc-([a-zA-Z0-9]+)"
+        },
         {
             "name": "reconstruction",
             "pattern": "[_/\\\\]+rec-([a-zA-Z0-9]+)"
@@ -73,6 +85,10 @@
             "name": "space",
             "pattern": "[_/\\\\]+space-([a-zA-Z0-9]+)"
         },
+        {
+            "name": "chunk",
+            "pattern": "[_/\\\\]+chunk-([0-9]+)"
+        },
         {
             "name": "suffix",
             "pattern": "[._]*([a-zA-Z0-9]*?)\\.[^/\\\\]+$"
@@ -87,7 +103,7 @@
         },
         {
             "name": "datatype",
-            "pattern": "[/\\\\]+(anat|beh|dwi|eeg|fmap|func|ieeg|meg|perf)[/\\\\]+"
+            "pattern": "[/\\\\]+(anat|beh|dwi|eeg|fmap|func|ieeg|meg|micr|perf|pet)[/\\\\]+"
         },
         {
             "name": "extension",
@@ -98,8 +114,8 @@
     "default_path_patterns": [
         "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_part-{part}]_{suffix<T1w|T2w|T1rho|T1map|T2map|T2star|FLAIR|FLASH|PDmap|PD|PDT2|inplaneT[12]|angio>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
         "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_mod-{modality}]_{suffix<defacemask>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
-        "sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_dir-{direction}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_part-{part}]_{suffix<bold|cbv|sbref>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
-        "sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_dir-{direction}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_{suffix<phase>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_dir-{direction}][_run-{run}][_echo-{echo}][_part-{part}]_{suffix<bold|cbv|sbref>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_dir-{direction}][_run-{run}][_echo-{echo}]_{suffix<phase>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
         "sub-{subject}[/ses-{session}]/{datatype<dwi>|dwi}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_part-{part}]_{suffix<dwi>}{extension<.bval|.bvec|.json|.nii.gz|.nii>|.nii.gz}",
         "sub-{subject}[/ses-{session}]/{datatype<fmap>|fmap}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_dir-{direction}][_run-{run}]_{fmap<phasediff|magnitude[12]|phase[12]|fieldmap>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
         "sub-{subject}[/ses-{session}]/{datatype<fmap>|fmap}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}]_dir-{direction}[_run-{run}]_{fmap<epi>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
@@ -109,13 +125,45 @@
         "sub-{subject}[/ses-{session}]/{datatype<meg>|meg}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_run-{run}][_proc-{proc}]_{suffix<channels>}{extension<.tsv|.json>|.tsv}",
         "sub-{subject}[/ses-{session}]/{datatype<meg>|meg}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}]_{suffix<coordsystem>}{extension<.json>|.json}",
         "sub-{subject}[/ses-{session}]/{datatype<meg>|meg}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}]_{suffix<photo>}{extension<.jpg>|.jpg}",
+        "sub-{subject}[/ses-{session}]/{datatype<micr>|micr}/sub-{subject}[_ses-{session}]_sample-{sample}[_acq-{acquisition}][_stain-{stain}][_run-{run}][_chunk-{chunk}]_{suffix<TEM|SEM|uCT|BF|DF|PC|DIC|FLUO|CONF|PLI|CARS|2PE|MPE|SR|NLO|OCT|SPIM>}{extension<.png|.tif|.ome.tif|.ome.btf|.json>}",
+        "sub-{subject}[/ses-{session}]/{datatype<micr>|micr}/sub-{subject}[_ses-{session}]_sample-{sample}[_acq-{acquisition}]_{suffix<photo>|photo}{extension<.jpg|.png|.tif|.json>}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_echo-{echo}[_part-{<mag|phase|real|imag>}]_{suffix<MEGRE|MESE>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}[_part-{<mag|phase|real|imag>}]_{suffix<VFA>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<IRT1>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_flip-{flip}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<MP2RAGE>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MPM|MTS>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MTR>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_{suffix<T1map|T2map|T2starmap|R1map|R2map|R2starmap|PDmap|MTRmap|MTsat|UNIT1|T1rho|MWFmap|MTVmap|PDT2map|Chimap|S0map|M0map>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}][_task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<pet>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}][_task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_recording-{recording}_{suffix<blood>}{extension<.tsv|.json>}",
+        "sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}]_task-{task}[trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<events>}{extension<.tsv|.json>}",
+        "sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}][_task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}][_recording-{recording}]_{suffix<physio|stim>}{extension<.tsv.gz|.json>}",      
+        "sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<asl|m0scan>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
+        "sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<aslcontext>}{extension<.tsv|.json>}",
+        "sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_run-{run}]_{suffix<asllabeling>}{extension<.jpg>}",
+        "sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}][_recording{recording}]_{suffix<physio|stim>}{extension<.tsv|.json>}",  
         "[acq-{acquisition}_][ce-{ceagent}_][rec-{reconstruction}_]{suffix<T1w|T2w|T1rho|T1map|T2map|T2star|FLAIR|FLASH|PDmap|PD|PDT2|inplaneT[12]|angio>}{extension<.json>|.json}",
         "[acq-{acquisition}_][ce-{ceagent}_][rec-{reconstruction}_][mod-{modality}_]{suffix<defacemask>}{extension<.json>|.json}",
         "task-{task}[_acq-{acquisition}][_ce-{ceagent}][_dir-{direction}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_{suffix<bold|cbv|phase|sbref>}{extension<.json>|.json}",
         "[acq-{acquisition}_]{suffix<dwi>}{extension<.json>|.json}",
         "[acq-{acquisition}_][dir-{direction}_][run-{run}_]{fmap<phasediff|magnitude[1-2]|phase[1-2]|fieldmap>}{extension<.json>|.json}",
         "[acq-{acquisition}_][ce-{ceagent}_]dir-{direction}[_run-{run}]_{fmap<epi>}{extension<.json>|.json}",
         "task-{task}[_acq-{acquisition}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_recording-{recording}]_{suffix<events>}{extension<.json>|.json}",
-        "task-{task}[_acq-{acquisition}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_recording-{recording}]_{suffix<physio|stim>}{extension<.json>}"
+        "task-{task}[_acq-{acquisition}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_recording-{recording}]_{suffix<physio|stim>}{extension<.json>}",
+        "sample-{sample}[_acq-{acquisition}][_stain-{stain}][_run-{run}][_chunk-{chunk}]_{suffix<TEM|SEM|uCT|BF|DF|PC|DIC|FLUO|CONF|PLI|CARS|2PE|MPE|SR|NLO|OCT|SPIM>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_echo-{echo}[_part-{<mag|phase|real|imag>}]_{suffix<MEGRE|MESE>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}[_part-{<mag|phase|real|imag>}]_{suffix<VFA>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<IRT1>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_flip-{flip}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<MP2RAGE>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MPM|MTS>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MTR>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_{suffix<T1map|T2map|T2starmap|R1map|R2map|R2starmap|PDmap|MTRmap|MTsat|UNIT1|T1rho|MWFmap|MTVmap|PDT2map|Chimap|S0map|M0map>}{extension<.json>|.json}",
+        "[task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<pet>}{extension<.json>|.json}",
+        "[task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_recording-{recording}_{suffix<blood>}{extension<.json>|.json}",
+        "task-{task}[trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<events>}{extension<.json>|.json}",
+        "[task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}][_recording-{recording}]_{suffix<physio|stim>}{extension<.json>|.json}",      
+        "[acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<asl|m0scan>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<aslcontext>}{extension<.json>|.json}",
+        "[acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}][_recording{recording}]_{suffix<physio|stim>}{extension<.json>|.json}"
     ]
 }

bids/layout/tests/test_layout_on_examples.py

@@ -0,0 +1,87 @@
+""" Tests runs layout on bids examples and make sure all files are caught"""
+
+""" TODO
+- add more 'vanilla' datasets
+- missing files in micr?
+"""
+
+from os.path import join
+
+import pytest
+
+from bids.layout import BIDSLayout
+
+# Values for the number of file got from a:
+#
+#   find ds_name -type f | wc -l
+#
+
+@pytest.mark.parametrize(
+    "dataset, nb_files",
+    [
+        ("qmri_irt1", 15),
+        ("qmri_mese", 73),
+        ("qmri_mp2rage", 14),
+        ("qmri_mp2rageme", 28),
+        ("qmri_mtsat", 23),
+        ("qmri_sa2rage", 9),
+        ("qmri_vfa", 17),
+    ],
+)
+def test_layout_on_examples_with_derivatives(dataset, nb_files, bids_examples):
+    ds = join(bids_examples, dataset)
+    layout = BIDSLayout(ds, derivatives=True)
+    files = layout.get()
+    assert len(files) == nb_files
+
+
+@pytest.mark.parametrize(
+    "dataset, nb_files",
+    [
+        ("micr_SEM", 12),
+        ("micr_SPIM", 22),
+        ("asl001", 8),
+        ("asl002", 10),
+        ("asl003", 10),
+        ("asl004", 12),
+        ("asl005", 10),
+        ("pet001", 12),
+        ("pet002", 20),
+        ("pet003", 9),
+        ("pet004", 10),
+        ("pet005", 14),
+        ("qmri_megre", 18),
+        ("qmri_tb1tfl", 6),
+    ],
+)
+def test_layout_on_examples_no_derivatives(dataset, nb_files, bids_examples):
+    ds = join(bids_examples, dataset)
+    layout = BIDSLayout(ds)
+    files = layout.get()
+    assert len(files) == nb_files
+
+
+@pytest.mark.parametrize(
+    "dataset, nb_files",
+    [
+        pytest.param(
+            "qmri_qsm",
+            8,
+            marks=pytest.mark.xfail(strict=True,
+                reason="https://github.com/bids-standard/bids-examples/issues/311"
+            ),
+        ),
+        pytest.param(
+            "qmri_mpm",
+            125,
+            marks=pytest.mark.xfail(strict=True,
+                reason="https://github.com/bids-standard/bids-examples/issues/310"
+            ),
+        ),
+    ],
+)
+def test_layout_on_examples_invalid_ds_description(dataset, nb_files, bids_examples):
+    ds = join(bids_examples, dataset)
+    layout = BIDSLayout(ds, derivatives=True)
+    files = layout.get()
+    assert len(files) == nb_files