Merge branch 'master' into dwi_scanner_derivatives

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,env,venv,ds*,asl*,qmr*,genetics*,fnirs*,eeg*,meg*,pet*,ieeg*,motion*,synthetic*
 builtin = clear,rare
-ignore-words-list = nd
+ignore-words-list = nd,te

.github/workflows/validate_datasets.yml

@@ -98,6 +98,11 @@ jobs:
         fi
       shell: bash
 
+    - name: Skip MRS validation for legacy validator
+      run: for DS in mrs_*; do touch $DS/.SKIP_VALIDATION; done
+      if: "matrix.bids-validator != 'master-deno'"
+      shell: bash
+
     - name: Validate all BIDS datasets using bids-validator
       run: |
         cat ./run_tests.sh

dataset_listing.tsv

@@ -2,7 +2,7 @@ name	description	link to full data	maintained by	datatypes	suffixes
 eeg_matchingpennies	Offline data of BCI experiment decoding left vs. right hand movement. BrainVision data format (.eeg, .vhdr, .vmrk)	[link](https://doi.org/10.17605/OSF.IO/CJ2DR)	[@sappelhoff](https://github.com/sappelhoff)	eeg	channels, eeg, events
 eeg_rishikesh	Mind wandering experiment. EEG data in Biosemi (.bdf) format	[link](https://openneuro.org/datasets/ds001787)	[@arnodelorme](https://github.com/arnodelorme)	eeg	channels, eeg, events
 eeg_face13	Deconstructing the early visual electrocortical response to face and house stimuli. EDF format		[@andesha](https://github.com/andesha)	eeg	channels, coordsystem, eeg, electrodes, events
-eeg_ds003645s_hed_demo	Shows usage of Hierarchical Event Descriptor (HED) in .tsv files	[link](https://openneuro.org/datasets/ds003645)	[@VisLab](https://github.com/VisLab)	anat, beh, eeg, micr, motion	KSSSleep, SPIM, beh, channels, coordsystem, defacemask, eeg, electrodes, events, headshape, motion, photo, samples, scans
+eeg_ds003645s_hed_demo	Shows usage of Hierarchical Event Descriptor (HED) in .tsv files	[link](https://openneuro.org/datasets/ds003645)	[@VisLab](https://github.com/VisLab)	anat, beh, eeg, micr, motion	KSSSleep, SPIM, beh, channels, coordsystem, defacemask, eeg, electrodes, events, motion, photo, samples, scans
 eeg_ds003645s_hed_library	HED annotation using HED library vocabularies (schema).	[link](https://openneuro.org/datasets/ds003645)	[@VisLab](https://github.com/VisLab)	eeg	channels, eeg, events
 eeg_cbm	Rest EEG. European Data Format (.edf)		[@cpernet](https://github.com/cpernet)	eeg	channels, eeg, events, scans
 ieeg_filtered_speech	recordings of three seizures		[@choldgraf](https://github.com/choldgraf)	ieeg	channels, coordsystem, electrodes, events, ieeg, photo
@@ -59,12 +59,13 @@ pet005	T1w, PET		[@mnoergaard](https://github.com/mnoergaard)	anat, pet	T1w, eve
 micr_SEM	Example SEM dataset in PNG format with 1 sample imaged over 2 sessions	[link](https://doi.org/10.5281/zenodo.5498378)	[@jcohenadad](https://github.com/jcohenadad)	micr	SEM, photo, samples, sessions
 micr_SEMzarr	Example SEM dataset in PNG and OME-ZARR format with 1 sample imaged over 2 sessions		[@TheChymera](https://github.com/TheChymera)	micr	SEM, SPIM, samples, sessions
 micr_SPIM	Example SPIM dataset in OME-TIFF format with 2 samples from the same subject with 4 chunks each	[link](https://doi.org/10.5281/zenodo.5517223)	[@jcohenadad](https://github.com/jcohenadad)	micr	SPIM, photo, samples
+micr_XPCTzarr	Example XPCT dataset in OME-ZARR format with 1 sample imaged	[link](https://human-organ-atlas.esrf.eu/datasets/572252538)	[@chourroutm](https://github.com/chourroutm)	micr	XPCT, photo, samples, sessions
 fnirs_tapping	Example fNIRS measurement with three conditions from five subjects	[link](https://doi.org/10.5281/zenodo.5529797)	[@rob-luke](https://github.com/rob-luke)	nirs	channels, coordsystem, events, nirs, optodes, scans
 fnirs_automaticity	24 subjects performing (non-)automatic finger tapping and foot stepping	[link](https://doi.org/10.34973/vesb-mh30)	[@robertoostenveld](https://github.com/robertoostenveld)	nirs	channels, coordsystem, events, nirs, optodes, practicelogbook, scans
 motion_systemvalidation	Example dataset of two different motion captured system recorded almost simultaneously, but no brain data	[link](https://doi.org/10.6084/m9.figshare.20238006.v2)	[@JuliusWelzel](https://github.com/JuliusWelzel)	motion	channels, motion, scans
-ds000117	A multi-subject, multi-modal human neuroimaging dataset of 19 subjects on a MEG visual task	[link](https://openneuro.org/datasets/ds000117/)	[@RikHenson](https://github.com/RikHenson)	anat, beh, dwi, fmap, func, meg	FLASH, T1w, beh, bold, channels, coordsystem, dwi, events, headshape, magnitude1, magnitude2, meg, phasediff, scans
-ds000246	Auditory dataset used for Brainstorm’s general online tutorial	[link](https://openneuro.org/datasets/ds000246/versions/00001)	[@guiomar](https://github.com/guiomar)	anat, meg	ChannelGroupSet, ClassFile, MarkerFile, T1w, channels, coordsystem, default, headshape, meg, params, photo, processing, scans
-ds000247	Five minutes, eyes-open, resting-state MEG data from 5 subjects. This is a sample from The Open MEG Archive (OMEGA).	[link](https://openneuro.org/datasets/ds000247/versions/00001)	[@guiomar](https://github.com/guiomar)	anat, meg	ClassFile, T1w, bad, channels, coordsystem, default, headshape, meg, params, processing, scans
+ds000117	A multi-subject, multi-modal human neuroimaging dataset of 19 subjects on a MEG visual task	[link](https://openneuro.org/datasets/ds000117/)	[@RikHenson](https://github.com/RikHenson)	anat, beh, dwi, fmap, func, meg	FLASH, T1w, bold, channels, coordsystem, dwi, events, headshape, magnitude1, magnitude2, meg, phasediff, scans
+ds000246	Auditory dataset used for Brainstorm’s general online tutorial	[link](https://openneuro.org/datasets/ds000246/versions/00001)	[@guiomar](https://github.com/guiomar)	anat, meg	T1w, channels, coordsystem, headshape, meg, photo, scans
+ds000247	Five minutes, eyes-open, resting-state MEG data from 5 subjects. This is a sample from The Open MEG Archive (OMEGA).	[link](https://openneuro.org/datasets/ds000247/versions/00001)	[@guiomar](https://github.com/guiomar)	anat, meg	T1w, channels, coordsystem, headshape, meg, scans
 ds000248	MNE sample data: Data with visual and auditory stimuli	[link](https://openneuro.org/datasets/ds000248/versions/00001)	[@agramfort](https://github.com/agramfort)	anat, meg	FLASH, T1w, channels, coordsystem, events, meg, scans
 eeg_ds000117	Multimodal (fMRI, MEG, EEG) stripped down to EEG with MRI anatomical scan and electrode coordinates. EEGLAB data format (.set, .fdt)	[link](https://openneuro.org/datasets/ds000117/)	[@robertoostenveld](https://github.com/robertoostenveld)	anat, eeg	T1w, channels, coordsystem, eeg, electrodes, events
 eeg_rest_fmri	Resting state with simultaneous fMRI. BrainVision data format (.eeg, .vhdr, .vmrk)		[@cpernet](https://github.com/cpernet)	anat, dwi, eeg, func	T1w, bold, dwi, eeg
@@ -75,3 +76,6 @@ ieeg_visual_multimodal			[@irisgroen](https://github.com/irisgroen)	anat, fmap,
 genetics_ukbb	multiple tasks, T1w, DTI, BOLD, genetic info		[@cpernet](https://github.com/cpernet)	anat, dwi, func	FLAIR, T1w, bold, dwi, events, info
 motion_dualtask	older and younger participants walking while performing discrimination task		[@sjeung](https://github.com/sjeung)	eeg, motion	channels, eeg, events, motion, scans
 motion_spotrotation	participants rotated heading using full-body motion or joystick	[link](https://openneuro.org/datasets/ds004460)	[@sjeung](https://github.com/sjeung)	eeg, motion	channels, coordsystem, eeg, electrodes, events, motion, scans
+mrs_2dmrsi	2D sLASER MRSI data from 8 subjects	[link](https://zenodo.org/records/7701228)	[@markmikkelsen](https://github.com/markmikkelsen)	anat, mrs	T1w, mrsi
+mrs_biggaba	MEGA-PRESS and PRESS MRS data from 12 subjects from one site from the Big GABA project	[link](https://www.nitrc.org/projects/biggaba)	[@markmikkelsen](https://github.com/markmikkelsen)	anat, mrs	T1w, mrsref, svs
+mrs_fmrs	Functional MRS data involving a pain stimulus task from 15 subjects	[link](https://www.nitrc.org/projects/fmrs_2020)	[@markmikkelsen](https://github.com/markmikkelsen)	anat, mrs	T1w, events, mrsref, svs

micr_XPCTzarr/README.md

@@ -0,0 +1,3 @@
+1 human brain sample imaged over 1 session.
+Example dataset with empty images containing X-ray phase-contrast tomography (XPCT) data to illustrate BIDS convention.
+Using NGFF nested directory format example

micr_XPCTzarr/code/data_retrieval.py

@@ -0,0 +1,77 @@
+from pathlib import Path
+import zarr.convenience
+import dask.array
+import ome_zarr, ome_zarr.io, ome_zarr.writer
+from numcodecs import Blosc, Delta
+import time
+
+"""
+You may use the following command to prepare a Python 3.8+ environment for the download of the dataset: `pip install -r data_retrieval_reqs.txt`
+"""
+
+data_path = Path("../sub-LADAF-2020-31/ses-01/micr/") # path to session
+data_path.mkdir(exist_ok=True) # create directory
+
+# short utility function
+def zarr_array(url,selector):
+    print('Retrieving data from', url)
+    n5_store = zarr.N5FSStore(url)
+    root = zarr.group(store=n5_store)
+    return dask.array.from_zarr(root[selector])
+
+dataset_full = zarr_array("gs://ucl-hip-ct-35a68e99feaae8932b1d44da0358940b/LADAF-2020-31/brain/25.08um_complete-organ_bm05/","s0") # get a dask.array.Array that points to the whole N5 dataset
+
+# default filters and compressors made the script crash; the ones below work:
+filters = [Delta(dtype='i4')]
+compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.SHUFFLE)
+
+path_roi = data_path / "sub-LADAF-2020-31_ses-01_sample-brain_XPCT.ome.zarr" # full name of the dataset following BIDS specification
+
+print('Writing in', str(path_roi))
+
+tic = time.time()
+
+store = ome_zarr.io.parse_url(path_roi,mode="a").store # NB: `mode="a"` should allow overwrite but it does not at the moment, see https://github.com/ome/ome-zarr-py/issues/376
+root = zarr.group(store=store)
+ome_zarr.writer.write_image(image=dataset_full,
+                            group=root,
+                            scaler=None,
+                            axes=[
+                                {
+                                    "name": "z",
+                                    "type": "space",
+                                    "units": "micrometer" # voxel size is an isotropic 25.08 um
+                                },
+                                {
+                                    "name": "y",
+                                    "type": "space",
+                                    "units": "micrometer" # voxel size is an isotropic 25.08 um
+                                },
+                                {
+                                    "name": "x",
+                                    "type": "space",
+                                    "units": "micrometer" # voxel size is an isotropic 25.08 um
+                                }
+                            ], # axis order of the dataset following BIDS specification
+                            coordinate_transformations=[
+                                [
+                                    {
+                                        "scale": [
+                                            25.08,
+                                            25.08,
+                                            25.08 
+                                        ], # voxel size is an isotropic 25.08 um
+                                        "type": "scale"
+                                    }
+                                ] 
+                            ],
+                            storage_options=dict(
+                                chunks=(512,512,512), # this chunk size may be altered depending on someone's needs
+                                filters=filters, # default filters made the script crash
+                                compressor=compressor # default compressors made the script crash
+                            )
+                        )
+
+toc = time.time()
+
+print('Writing completed in', (toc - tic)/60, 'min!')

micr_XPCTzarr/code/data_retrieval_reqs.txt

@@ -0,0 +1,5 @@
+dask[complete]
+zarr
+ome-zarr
+numcodecs
+gcsfs

micr_XPCTzarr/code/metadata_formatter.py

@@ -0,0 +1,39 @@
+import urllib.request
+import json
+
+"""
+You may use the following command to prepare a Python 3.8+ environment for the download of the dataset: `pip install -r metadata_formatter_reqs.txt`
+"""
+
+# The following metadata file can be downloaded from https://human-organ-atlas.esrf.eu/datasets/572252538
+metadata_txt_file_uri = 'https://ids.esrf.fr/ids/getData?sessionId=182d0a3b-de3b-4602-8caf-9bd91dc5b0e5&datafileIds=572252539' # This URI expires quite often
+
+req = urllib.request.urlopen(metadata_txt_file_uri)
+
+json_dict = {}
+
+for line in urllib.request.urlopen(metadata_txt_file_uri):
+    text = line.decode('utf-8')
+    if text.startswith('#') or text == '\r\n':
+        pass
+    else:
+        s = text.replace('\t','').replace('\r\n','').replace('"', '').replace('N.A.', 'n/a').split('=')
+
+        try:
+            s[1] = int(s[1])
+        except:
+            try:
+                s[1] = float(s[1])
+            except:
+                pass
+        json_dict[s[0]] = s[1]
+json_obj = json.dumps(json_dict)
+print(json_obj)
+
+"""
+The metadata has been split into several files:
+  - micr_XPCTzarr/samples.json
+  - micr_XPCTzarr/sub-LADAF-2020-31/sub-LADAF-2020-31_sessions.tsv
+  - micr_XPCTzarr/sub-LADAF-2020-31/ses-01/micr/sub-LADAF-2020-31_ses-01_sample-brain_XPCT.json
+Note that some fields are redundant. 
+"""

micr_XPCTzarr/code/metadata_formatter_reqs.txt

@@ -0,0 +1 @@
+urllib3

micr_XPCTzarr/code/truncate_data.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Bash script to truncate all the files containing the binary chunked data as per ../../CONTRIBUTING.md
+# Modified from ../../CONTRIBUTING.md
+
+find ../sub-LADAF-2020-31/ses-01/micr/sub-LADAF-2020-31_ses-01_sample-brain_XPCT.ome.zarr/ -type f -regex ".*/[0-9]*" -exec truncate -s 0 {} +

micr_XPCTzarr/dataset_description.json

@@ -0,0 +1,15 @@
+{
+	"Name": "micr_XPCTzarr",
+	"Authors": [
+		"Matthieu Chourrout",
+		"David Stansby",
+		"Guillaume Gaisne",
+		"Claire L. Walsh",
+		"Peter D. Lee"
+	],
+	"BIDSVersion": "1.10.0",
+	"License": "CC-BY-4.0",
+	"DatasetDOI": "doi:10.15151/ESRF-DC-572252655",
+	"DatasetType": "raw",
+	"HowToAcknowledge": "Please cite this dataset as: Tafforeau, P., Walsh, C., Wagner, W. L.,  Daniyal J. Jafree, Bellier, A., Werlein, C., Kühnel, M. P., Boller, E., Walker-Samuel, S., Robertus, J. L., Long, D. A., Jacob, J., Marussi, S.,  Eeline Brown, Holroyd, N., Jonigk, D. D., Ackermann, M., & Lee, P. D. (2021). Complete brain from the body donor LADAF-2020-31 (Version 1) [dataset]. European Synchrotron Radiation Facility. https://doi.org/10.15151/ESRF-DC-572252655"
+}

micr_XPCTzarr/participants.json

@@ -0,0 +1,36 @@
+{
+    "participant_id": {
+        "Description": "Unique alphanumeric participant ID starting with sub-"
+    },
+    "participant_name": {
+        "Description": "full name or pseudo of the participant which can contain non-alphanumeric characters"
+    },
+    "sex": {
+        "Description": "sex of the participant",
+        "Levels": {
+            "M": "male",
+            "F": "female"
+        }
+    },
+    "age": {
+        "Description": "age of the participant",
+        "Units": "year"
+    },
+    "weight": {
+        "Description": "weight of the participant",
+        "Units": "kg"
+    },
+    "height": {
+        "Description": "height of the participant",
+        "Units": "cm"
+    },
+    "species": {
+        "Description": "binomial species name from the NCBI Taxonomy (https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi)"
+    },
+    "medical_information": {
+        "Description": "medical information as provided by the biobank"
+    },
+    "institute": {
+        "Description": "name of the biobank"
+    }
+}

micr_XPCTzarr/participants.tsv

@@ -0,0 +1,2 @@
+participant_id	participant_name	sex	age	species	weight	height	institute	medical_information
+sub-01	LADAF-2020-31	F	69	homo sapiens	40	145	Laboratoire d'Anatomie des Alpes Françaises	type 2 diabetes, pelvic radiation to treat cancer of the uterus, right colectomy (benign lesion on histopathology), bilateral nephrostomy for acute obstructive renal failure, cystectomy, omentectomy and peritoneal carcinoma with occlusive syndrome

micr_XPCTzarr/samples.json

@@ -0,0 +1,17 @@
+{
+    "sample_id": {
+        "Description": "Sample ID"
+    },
+    "participant_id": {
+        "Description": "Participant ID from whom tissue samples have been acquired"
+    },
+    "sample_type": {
+        "Description": "Type of sample from ENCODE Biosample Type (https://www.encodeproject.org/profiles/biosample_type)"
+    },
+    "sample_info": {
+        "Description": "One-line title of the sample"
+    },
+    "sample_preparation": {
+        "Description": "Specific preparation of the sample"
+    }
+}

micr_XPCTzarr/samples.tsv

@@ -0,0 +1,2 @@
+sample_id	participant_id	sample_type	sample_info 	sample_preparation
+sample-brain	sub-01	tissue	complete brain from the body donor program of the Laboratoire d'Anatomie des Alpes Francaise (LADAF)	formalin fixed, progressive transfer to ethanol 70% with gentle vacuum degassing at each step, mounted with mixed agar gel at 70% ethanol, n.b. some damages due to the too rapid vacuum degassing

micr_XPCTzarr/sub-01/ses-01/micr/sub-01_ses-01_sample-brain_XPCT.json

@@ -0,0 +1,57 @@
+{
+        "InstitutionName": "European Synchrotron Radiation Facility",
+        "StationName": "BM05 EBS dipole wiggler 0.85T",
+        "BodyPart": "BRAIN",
+        "SampleEnvironment": "ex vivo",
+        "SampleFixation": "formalin",
+        "SampleEmbedding": "mixed agar gel at 70% ethanol",
+        "PixelSize": [
+                25.08,
+                25.08,
+                25.08
+        ],
+        "PixelSizeUnits": "um",
+        "AcquisitionParameters": {
+                "XStep": "n/a",
+                "XStages": 1,
+                "YStep": "n/a",
+                "YStages": 1,
+                "ZStep ": "2.2 mm",
+                "ZStages ": "2x79",
+                "Projections": 9990,
+                "RefN": "n/a",
+                "DarkN": 400,
+                "RefOn": "n/a",
+                "ScanningMode ": "continuous",
+                "ExposureTime": "0.036 s",
+                "AccExposureTime ": "0.006 s",
+                "AccFramesCount": 6,
+                "PropDistance": "3475 mm",
+                "Filters": "Mo 0.1mm Al 2mm SiO2 bars 12*5mm diameter",
+                "DetAvgEnergy": "93 keV",
+                "ScanGeometry": "quarter-acquisition, one scan in half-acquisition plus one annular scan",
+                "ScanRange": "360 deg",
+                "SensorName": "sCMOS PCO edge 4.2 CLHS",
+                "SensorMode": "rolling shutter",
+                "SensorPixelSize": "6.5 um",
+                "Magnification": 0.24,
+                "XPixelNum": 2048,
+                "YPixelNum": 176,
+                "OpticsType": "dzoom optic from BM05 based on Hasselblad 120mm F/4 macro objective",
+                "Scintillator": "LuAG:Ce 2000 um",
+                "SurfDoseRate": "10.5 Gy/s",
+                "VoiDoseRate": "10.5 Gy/s",
+                "VoiIntegDose": "2.48 kGy",
+                "ScanTime": "7.88 min",
+                "SeriesTime": "22 h"
+        },
+        "ProcessingParameters": {
+                "RefApproach": "reference jar with 70% ethanol, single reference",
+                "VolumeX": 5965,
+                "VolumeY": 5965,
+                "VolumeZ": 6990,
+                "32to16BitsMin": -0.04,
+                "32to16BitsMax": 0.1,
+                "Jp2ComprRatio": 10
+        }
+}

micr_XPCTzarr/sub-01/ses-01/micr/sub-01_ses-01_sample-brain_XPCT.ome.zarr/.zattrs

@@ -0,0 +1,40 @@
+{
+    "multiscales": [
+        {
+            "axes": [
+                {
+                    "name": "z",
+                    "type": "space",
+                    "units": "micrometer"
+                },
+                {
+                    "name": "y",
+                    "type": "space",
+                    "units": "micrometer"
+                },
+                {
+                    "name": "x",
+                    "type": "space",
+                    "units": "micrometer"
+                }
+            ],
+            "datasets": [
+                {
+                    "coordinateTransformations": [
+                        {
+                            "scale": [
+                                25.08,
+                                25.08,
+                                25.08
+                            ],
+                            "type": "scale"
+                        }
+                    ],
+                    "path": "0"
+                }
+            ],
+            "name": "/",
+            "version": "0.4"
+        }
+    ]
+}