Skip to content

Commit

Permalink
[ENH] Update config to support microscopy, qMRI, PET, ASL (#840)
Browse files Browse the repository at this point in the history
* add Pipfile to gitingore

* add missing datatypes to config

* add microscopy entities to config

* add draft path patterns microscopy and qMRI

* clean up and refactor

* add extension micr

* add suffixes micr

* add photo files for micr

* add parametric parts of qMRI

* add PET tracer entity to config

* add path patterns for PET

* add path patterns for ASL

* refactor some patterns

* add patterns for json files in root of dataset

* update bids nodot config and use {suffix<>} everywhere

* add tracer entity to no dot config

* fix entity order in filename pattern for no dot config

* fix typo

* remove problematic path pattern

* readd sample entity to micr filename pattern in root of dataset

* sample is actually required for micr

* update test layout on bids-example

* Update bids/layout/config/bids.json

Co-authored-by: Chris Markiewicz <[email protected]>

* Update bids/layout/config/bids.json

Co-authored-by: Chris Markiewicz <[email protected]>

* fix separator position for session

* add more bids example layout tests and flag MPM as xfail

* flag qmri_qsm as failure expected too

* make xfail strict to detect tests when they start passing

* Update bids/layout/config/bids.json

Co-authored-by: mariehbourget <[email protected]>

* Update bids/layout/config/bids.json

Co-authored-by: mariehbourget <[email protected]>

* fix more typos

* use bids-example submodule and fixture for testing

* remove makefile recipe for bids-examles

* MNT: Restore makefile

Co-authored-by: Chris Markiewicz <[email protected]>
Co-authored-by: mariehbourget <[email protected]>
Co-authored-by: Christopher J. Markiewicz <[email protected]>
  • Loading branch information
4 people authored Apr 24, 2022
1 parent c81e7f9 commit 79ee34a
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 5 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
historical/

#
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
Expand Down Expand Up @@ -86,6 +85,9 @@ celerybeat-schedule
venv/
ENV/

# Pipenv
Pipfile

# Spyder project settings
.spyderproject

Expand Down
56 changes: 52 additions & 4 deletions bids/layout/config/bids.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
"mandatory": false,
"directory": "{subject}{session}"
},
{
"name": "sample",
"pattern": "[_/\\\\]+sample-([a-zA-Z0-9]+)"
},
{
"name": "task",
"pattern": "[_/\\\\]+task-([a-zA-Z0-9]+)"
Expand All @@ -24,6 +28,14 @@
"name": "ceagent",
"pattern": "[_/\\\\]+ce-([a-zA-Z0-9]+)"
},
{
"name": "staining",
"pattern": "[_/\\\\]+stain-([a-zA-Z0-9]+)"
},
{
"name": "tracer",
"pattern": "[_/\\\\]+trc-([a-zA-Z0-9]+)"
},
{
"name": "reconstruction",
"pattern": "[_/\\\\]+rec-([a-zA-Z0-9]+)"
Expand Down Expand Up @@ -73,6 +85,10 @@
"name": "space",
"pattern": "[_/\\\\]+space-([a-zA-Z0-9]+)"
},
{
"name": "chunk",
"pattern": "[_/\\\\]+chunk-([0-9]+)"
},
{
"name": "suffix",
"pattern": "[._]*([a-zA-Z0-9]*?)\\.[^/\\\\]+$"
Expand All @@ -87,7 +103,7 @@
},
{
"name": "datatype",
"pattern": "[/\\\\]+(anat|beh|dwi|eeg|fmap|func|ieeg|meg|perf)[/\\\\]+"
"pattern": "[/\\\\]+(anat|beh|dwi|eeg|fmap|func|ieeg|meg|micr|perf|pet)[/\\\\]+"
},
{
"name": "extension",
Expand All @@ -98,8 +114,8 @@
"default_path_patterns": [
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_part-{part}]_{suffix<T1w|T2w|T1rho|T1map|T2map|T2star|FLAIR|FLASH|PDmap|PD|PDT2|inplaneT[12]|angio>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_mod-{modality}]_{suffix<defacemask>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_dir-{direction}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_part-{part}]_{suffix<bold|cbv|sbref>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_dir-{direction}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_{suffix<phase>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_dir-{direction}][_run-{run}][_echo-{echo}][_part-{part}]_{suffix<bold|cbv|sbref>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<func>|func}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_dir-{direction}][_run-{run}][_echo-{echo}]_{suffix<phase>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<dwi>|dwi}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_part-{part}]_{suffix<dwi>}{extension<.bval|.bvec|.json|.nii.gz|.nii>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<fmap>|fmap}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_dir-{direction}][_run-{run}]_{fmap<phasediff|magnitude[12]|phase[12]|fieldmap>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<fmap>|fmap}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}]_dir-{direction}[_run-{run}]_{fmap<epi>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
Expand All @@ -109,13 +125,45 @@
"sub-{subject}[/ses-{session}]/{datatype<meg>|meg}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}][_run-{run}][_proc-{proc}]_{suffix<channels>}{extension<.tsv|.json>|.tsv}",
"sub-{subject}[/ses-{session}]/{datatype<meg>|meg}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}]_{suffix<coordsystem>}{extension<.json>|.json}",
"sub-{subject}[/ses-{session}]/{datatype<meg>|meg}/sub-{subject}[_ses-{session}]_task-{task}[_acq-{acquisition}]_{suffix<photo>}{extension<.jpg>|.jpg}",
"sub-{subject}[/ses-{session}]/{datatype<micr>|micr}/sub-{subject}[_ses-{session}]_sample-{sample}[_acq-{acquisition}][_stain-{stain}][_run-{run}][_chunk-{chunk}]_{suffix<TEM|SEM|uCT|BF|DF|PC|DIC|FLUO|CONF|PLI|CARS|2PE|MPE|SR|NLO|OCT|SPIM>}{extension<.png|.tif|.ome.tif|.ome.btf|.json>}",
"sub-{subject}[/ses-{session}]/{datatype<micr>|micr}/sub-{subject}[_ses-{session}]_sample-{sample}[_acq-{acquisition}]_{suffix<photo>|photo}{extension<.jpg|.png|.tif|.json>}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_echo-{echo}[_part-{<mag|phase|real|imag>}]_{suffix<MEGRE|MESE>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}[_part-{<mag|phase|real|imag>}]_{suffix<VFA>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<IRT1>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_flip-{flip}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<MP2RAGE>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MPM|MTS>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MTR>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<anat>|anat}|anat/sub-{subject}[_ses-{session}][_acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_{suffix<T1map|T2map|T2starmap|R1map|R2map|R2starmap|PDmap|MTRmap|MTsat|UNIT1|T1rho|MWFmap|MTVmap|PDT2map|Chimap|S0map|M0map>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}][_task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<pet>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}][_task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_recording-{recording}_{suffix<blood>}{extension<.tsv|.json>}",
"sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}]_task-{task}[trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<events>}{extension<.tsv|.json>}",
"sub-{subject}[/ses-{session}]/{datatype<pet>|pet}/sub-{subject}[_ses-{session}][_task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}][_recording-{recording}]_{suffix<physio|stim>}{extension<.tsv.gz|.json>}",
"sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<asl|m0scan>}{extension<.nii|.nii.gz|.json>|.nii.gz}",
"sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<aslcontext>}{extension<.tsv|.json>}",
"sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_run-{run}]_{suffix<asllabeling>}{extension<.jpg>}",
"sub-{subject}[/ses-{session}]/{datatype<perf>|perf}/sub-{subject}[_ses-{session}][_acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}][_recording{recording}]_{suffix<physio|stim>}{extension<.tsv|.json>}",
"[acq-{acquisition}_][ce-{ceagent}_][rec-{reconstruction}_]{suffix<T1w|T2w|T1rho|T1map|T2map|T2star|FLAIR|FLASH|PDmap|PD|PDT2|inplaneT[12]|angio>}{extension<.json>|.json}",
"[acq-{acquisition}_][ce-{ceagent}_][rec-{reconstruction}_][mod-{modality}_]{suffix<defacemask>}{extension<.json>|.json}",
"task-{task}[_acq-{acquisition}][_ce-{ceagent}][_dir-{direction}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_{suffix<bold|cbv|phase|sbref>}{extension<.json>|.json}",
"[acq-{acquisition}_]{suffix<dwi>}{extension<.json>|.json}",
"[acq-{acquisition}_][dir-{direction}_][run-{run}_]{fmap<phasediff|magnitude[1-2]|phase[1-2]|fieldmap>}{extension<.json>|.json}",
"[acq-{acquisition}_][ce-{ceagent}_]dir-{direction}[_run-{run}]_{fmap<epi>}{extension<.json>|.json}",
"task-{task}[_acq-{acquisition}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_recording-{recording}]_{suffix<events>}{extension<.json>|.json}",
"task-{task}[_acq-{acquisition}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_recording-{recording}]_{suffix<physio|stim>}{extension<.json>}"
"task-{task}[_acq-{acquisition}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_recording-{recording}]_{suffix<physio|stim>}{extension<.json>}",
"sample-{sample}[_acq-{acquisition}][_stain-{stain}][_run-{run}][_chunk-{chunk}]_{suffix<TEM|SEM|uCT|BF|DF|PC|DIC|FLUO|CONF|PLI|CARS|2PE|MPE|SR|NLO|OCT|SPIM>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_echo-{echo}[_part-{<mag|phase|real|imag>}]_{suffix<MEGRE|MESE>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}[_part-{<mag|phase|real|imag>}]_{suffix<VFA>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<IRT1>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}][_flip-{flip}]_inv-{inv}[_part-{<mag|phase|real|imag>}]_{suffix<MP2RAGE>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}][_echo-{echo}]_flip-{flip}_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MPM|MTS>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_mt-{on|off}[_part-{<mag|phase|real|imag>}]_{suffix<MTR>}{extension<.json>|.json}",
"[acq-{acquisition}][_ce-{ceagent}][_rec-{reconstruction}][_run-{run}]_{suffix<T1map|T2map|T2starmap|R1map|R2map|R2starmap|PDmap|MTRmap|MTsat|UNIT1|T1rho|MWFmap|MTVmap|PDT2map|Chimap|S0map|M0map>}{extension<.json>|.json}",
"[task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<pet>}{extension<.json>|.json}",
"[task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}]_recording-{recording}_{suffix<blood>}{extension<.json>|.json}",
"task-{task}[trc-{tracer}][_rec-{reconstruction}][_run-{run}]_{suffix<events>}{extension<.json>|.json}",
"[task-{task}][trc-{tracer}][_rec-{reconstruction}][_run-{run}][_recording-{recording}]_{suffix<physio|stim>}{extension<.json>|.json}",
"[acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<asl|m0scan>}{extension<.json>|.json}",
"[acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}]_{suffix<aslcontext>}{extension<.json>|.json}",
"[acq-{acquisition}][_rec-{reconstruction}][_dir{direction}][_run-{run}][_recording{recording}]_{suffix<physio|stim>}{extension<.json>|.json}"
]
}
87 changes: 87 additions & 0 deletions bids/layout/tests/test_layout_on_examples.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
""" Tests runs layout on bids examples and make sure all files are caught"""

""" TODO
- add more 'vanilla' datasets
- missing files in micr?
"""

from os.path import join

import pytest

from bids.layout import BIDSLayout

# Values for the number of file got from a:
#
# find ds_name -type f | wc -l
#

@pytest.mark.parametrize(
"dataset, nb_files",
[
("qmri_irt1", 15),
("qmri_mese", 73),
("qmri_mp2rage", 14),
("qmri_mp2rageme", 28),
("qmri_mtsat", 23),
("qmri_sa2rage", 9),
("qmri_vfa", 17),
],
)
def test_layout_on_examples_with_derivatives(dataset, nb_files, bids_examples):
ds = join(bids_examples, dataset)
layout = BIDSLayout(ds, derivatives=True)
files = layout.get()
assert len(files) == nb_files


@pytest.mark.parametrize(
"dataset, nb_files",
[
("micr_SEM", 12),
("micr_SPIM", 22),
("asl001", 8),
("asl002", 10),
("asl003", 10),
("asl004", 12),
("asl005", 10),
("pet001", 12),
("pet002", 20),
("pet003", 9),
("pet004", 10),
("pet005", 14),
("qmri_megre", 18),
("qmri_tb1tfl", 6),
],
)
def test_layout_on_examples_no_derivatives(dataset, nb_files, bids_examples):
ds = join(bids_examples, dataset)
layout = BIDSLayout(ds)
files = layout.get()
assert len(files) == nb_files


@pytest.mark.parametrize(
"dataset, nb_files",
[
pytest.param(
"qmri_qsm",
8,
marks=pytest.mark.xfail(strict=True,
reason="https://github.com/bids-standard/bids-examples/issues/311"
),
),
pytest.param(
"qmri_mpm",
125,
marks=pytest.mark.xfail(strict=True,
reason="https://github.com/bids-standard/bids-examples/issues/310"
),
),
],
)
def test_layout_on_examples_invalid_ds_description(dataset, nb_files, bids_examples):
ds = join(bids_examples, dataset)
layout = BIDSLayout(ds, derivatives=True)
files = layout.get()
assert len(files) == nb_files

0 comments on commit 79ee34a

Please sign in to comment.