From e6b96730bd6d3cf56d06067389f942704cdd9bbb Mon Sep 17 00:00:00 2001 From: maestroque Date: Thu, 11 Jul 2024 14:24:10 +0300 Subject: [PATCH 01/16] Add preliminary load_from_bids function --- physutils/io.py | 116 ++++++++++++++++++++++++++++++++++++++++++++ physutils/physio.py | 6 ++- setup.cfg | 1 + 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index b6d460c..dc608dd 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -8,6 +8,7 @@ import os.path as op import numpy as np +from bids import BIDSLayout, BIDSValidator from loguru import logger from physutils import physio @@ -15,6 +16,121 @@ EXPECTED = ["data", "fs", "history", "metadata"] +def load_from_bids( + bids_path, + subject, + session=None, + task=None, + run=None, + extension="tsv.gz", + suffix="physio", +): + """ + Load physiological data from BIDS-formatted directory + + Parameters + ---------- + bids_path : str + Path to BIDS-formatted directory + subject : str + Subject identifier + session : str + Session identifier + task : str + Task identifier + run : str + Run identifier + suffix : str + Suffix of file to load + + Returns + ------- + data : :class:`physutils.Physio` + Loaded physiological data + """ + _supported_columns = [ + "cardiac", + "respiratory", + "trigger", + "rsp", + "ppg", + "tr", + "time", + ] + validator = BIDSValidator() + + # check if file exists and is in BIDS format + if not op.exists(bids_path): + raise FileNotFoundError(f"Provided path {bids_path} does not exist") + if not validator.is_bids(bids_path): + raise ValueError(f"Provided path {bids_path} is not a BIDS directory") + + layout = BIDSLayout(bids_path) + bids_file = layout.get( + subject=subject, + session=session, + task=task, + run=run, + suffix=suffix, + extension=extension, + )[0] + if len(bids_file) == 0: + raise FileNotFoundError( + f"No files found for subject {subject}, session {session}, task {task}, run {run}" + ) + if len(bids_file) > 1: + raise ValueError( + f"Multiple files found for subject {subject}, session {session}, task {task}, run {run}" + ) + + config_file = bids_file.get_metadata() + fs = config_file["SamplingFrequency"] + t_start = config_file["StartTime"] # noqa + columns = config_file["Columns"] + + physio_objects = {} + data = np.loadtxt(op.join(bids_file.dirname, bids_file.path)) + + if "time" in columns: + idx_0 = np.argmax(data[:, columns.index("time")] >= 0) + else: + idx_0 = 0 + logger.warning( + "No time column found in file. Assuming data starts at the beginning of the file" + ) + + for col in columns: + if col not in _supported_columns: + raise ValueError( + f"Column {col} is not supported. Supported columns are {_supported_columns}" + ) + if col in ["cardiac", "ppg", "ecg"]: + physio_type = "cardiac" + if col in ["respiratory", "rsp"]: + physio_type = "respiratory" + if col in ["trigger", "tr"]: + physio_type = "trigger" + if col in ["time"]: + continue + + if physio_type == "cardiac" or "respiratory": + physio_objects[physio_type] = physio.Physio( + data[idx_0:][columns.index(col)], + fs=fs, + history=[physio._get_call(exclude=[])], + ) + physio_objects[physio_type]._physio_type = physio_type + physio_objects[physio_type].label = bids_file.filename.split(".")[ + 0 + ].replace("_physio", "") + + if physio_type == "trigger": + # TODO: Implement trigger loading using the MRI data object + logger.warning("Trigger loading not yet implemented") + + return physio_objects + + def load_physio(data, *, fs=None, dtype=None, history=None, allow_pickle=False): """ Returns `Physio` object with provided data diff --git a/physutils/physio.py b/physutils/physio.py index a0dc577..31b6a39 100644 --- a/physutils/physio.py +++ b/physutils/physio.py @@ -220,10 +220,12 @@ def new_physio_like( if suppdata is None: suppdata = ref_physio._suppdata if copy_suppdata else None - + label = ref_physio.label if copy_label else None physio_type = ref_physio.physio_type if copy_physio_type else None - computed_metrics = list(ref_physio.computed_metrics) if copy_computed_metrics else [] + computed_metrics = ( + list(ref_physio.computed_metrics) if copy_computed_metrics else [] + ) # make new class out = ref_physio.__class__( diff --git a/setup.cfg b/setup.cfg index 8a74e06..ed17a18 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ install_requires = numpy >=1.9.3 scipy loguru + pybids tests_require = pytest >=3.6 test_suite = pytest From ecb73be7bd6308b6d5141045d4479c9360ce084a Mon Sep 17 00:00:00 2001 From: maestroque Date: Tue, 16 Jul 2024 15:30:34 +0300 Subject: [PATCH 02/16] Add MRIConfig class --- physutils/physio.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/physutils/physio.py b/physutils/physio.py index 31b6a39..b43556e 100644 --- a/physutils/physio.py +++ b/physutils/physio.py @@ -544,3 +544,55 @@ def neurokit2phys( metadata = dict(peaks=peaks) return cls(data, fs=fs, metadata=metadata, **kwargs) + + +class MRIConfig: + """ + Class to hold MRI configuration information + + Parameters + ---------- + slice_timings : 1D array_like + Slice timings in seconds + n_scans : int + Number of volumes in the MRI scan + tr : float + Repetition time in seconds + """ + + def __init__(self, slice_timings=None, n_scans=None, tr=None): + if np.ndim(slice_timings) > 1: + raise ValueError("Slice timings must be a 1-dimensional array.") + if np.size(slice_timings) != n_scans: + raise ValueError( + "Number of slice timings ({}) must match number of scans ({}).".format( + np.size(slice_timings), n_scans + ) + ) + + self._slice_timings = np.asarray(slice_timings) + self._n_scans = int(n_scans) + self._tr = float(tr) + logger.debug(f"Initializing new MRIConfig object: {self}") + + def __str__(self): + return "{name}(n_scans={n_scans}, tr={tr})".format( + name=self.__class__.__name__, + n_scans=self._n_scans, + tr=self._tr, + ) + + @property + def slice_timings(self): + """Slice timings in seconds""" + return self._slice_timings + + @property + def n_scans(self): + """Number of volumes in the MRI scan""" + return self._n_scans + + @property + def tr(self): + """Repetition time in seconds""" + return self._tr From 3440fe84e63584aa5f65499d3e9837eb19fff0b8 Mon Sep 17 00:00:00 2001 From: maestroque Date: Tue, 16 Jul 2024 18:18:47 +0300 Subject: [PATCH 03/16] Fixes for working load_from_bids function --- physutils/io.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index dc608dd..90a297a 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -8,7 +8,7 @@ import os.path as op import numpy as np -from bids import BIDSLayout, BIDSValidator +from bids import BIDSLayout from loguru import logger from physutils import physio @@ -57,13 +57,10 @@ def load_from_bids( "tr", "time", ] - validator = BIDSValidator() # check if file exists and is in BIDS format if not op.exists(bids_path): raise FileNotFoundError(f"Provided path {bids_path} does not exist") - if not validator.is_bids(bids_path): - raise ValueError(f"Provided path {bids_path} is not a BIDS directory") layout = BIDSLayout(bids_path) bids_file = layout.get( @@ -73,7 +70,7 @@ def load_from_bids( run=run, suffix=suffix, extension=extension, - )[0] + ) if len(bids_file) == 0: raise FileNotFoundError( f"No files found for subject {subject}, session {session}, task {task}, run {run}" @@ -83,13 +80,14 @@ def load_from_bids( f"Multiple files found for subject {subject}, session {session}, task {task}, run {run}" ) - config_file = bids_file.get_metadata() + config_file = bids_file[0].get_metadata() fs = config_file["SamplingFrequency"] t_start = config_file["StartTime"] # noqa columns = config_file["Columns"] + logger.debug(f"Loaded structure contains columns: {columns}") physio_objects = {} - data = np.loadtxt(op.join(bids_file.dirname, bids_file.path)) + data = np.loadtxt(bids_file[0].path) if "time" in columns: idx_0 = np.argmax(data[:, columns.index("time")] >= 0) @@ -100,31 +98,30 @@ def load_from_bids( ) for col in columns: + col_physio_type = None if col not in _supported_columns: - raise ValueError( - f"Column {col} is not supported. Supported columns are {_supported_columns}" - ) + logger.warning(f"Column {col} is not supported. Skipping") if col in ["cardiac", "ppg", "ecg"]: - physio_type = "cardiac" + col_physio_type = "cardiac" if col in ["respiratory", "rsp"]: - physio_type = "respiratory" + col_physio_type = "respiratory" if col in ["trigger", "tr"]: - physio_type = "trigger" + col_physio_type = "trigger" if col in ["time"]: continue - if physio_type == "cardiac" or "respiratory": - physio_objects[physio_type] = physio.Physio( - data[idx_0:][columns.index(col)], + if col_physio_type == "cardiac" or "respiratory": + physio_objects[col_physio_type] = physio.Physio( + data[idx_0:, columns.index(col)], fs=fs, history=[physio._get_call(exclude=[])], ) - physio_objects[physio_type]._physio_type = physio_type - physio_objects[physio_type].label = bids_file.filename.split(".")[ - 0 - ].replace("_physio", "") + physio_objects[col_physio_type]._physio_type = col_physio_type + physio_objects[col_physio_type]._label = ( + bids_file[0].filename.split(".")[0].replace("_physio", "") + ) - if physio_type == "trigger": + if col_physio_type == "trigger": # TODO: Implement trigger loading using the MRI data object logger.warning("Trigger loading not yet implemented") From 344ce3d7ef06615677f7aa37980e0b749345186f Mon Sep 17 00:00:00 2001 From: maestroque Date: Tue, 16 Jul 2024 18:41:45 +0300 Subject: [PATCH 04/16] Remove supported columns and add detection for known columns --- physutils/io.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index 90a297a..11f10b8 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -48,15 +48,6 @@ def load_from_bids( data : :class:`physutils.Physio` Loaded physiological data """ - _supported_columns = [ - "cardiac", - "respiratory", - "trigger", - "rsp", - "ppg", - "tr", - "time", - ] # check if file exists and is in BIDS format if not op.exists(bids_path): @@ -99,16 +90,18 @@ def load_from_bids( for col in columns: col_physio_type = None - if col not in _supported_columns: - logger.warning(f"Column {col} is not supported. Skipping") - if col in ["cardiac", "ppg", "ecg"]: + if any([x in col for x in ["cardiac", "ppg", "ecg", "card"]]): col_physio_type = "cardiac" - if col in ["respiratory", "rsp"]: + elif any([x in col for x in ["respiratory", "rsp", "resp"]]): col_physio_type = "respiratory" - if col in ["trigger", "tr"]: + elif any([x in col for x in ["trigger", "tr"]]): col_physio_type = "trigger" - if col in ["time"]: + elif any([x in col for x in ["time"]]): continue + else: + logger.warning( + f"Column {col}'s type cannot be determined. Additional features may be missing." + ) if col_physio_type == "cardiac" or "respiratory": physio_objects[col_physio_type] = physio.Physio( From f6b3d9f369b3a07f4c56454d9a189cb15bfd8da4 Mon Sep 17 00:00:00 2001 From: maestroque Date: Tue, 16 Jul 2024 18:47:12 +0300 Subject: [PATCH 05/16] load_from_bids: Dictionary keys named after column names --- physutils/io.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index 11f10b8..b5e4eda 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -103,14 +103,14 @@ def load_from_bids( f"Column {col}'s type cannot be determined. Additional features may be missing." ) - if col_physio_type == "cardiac" or "respiratory": - physio_objects[col_physio_type] = physio.Physio( + if col_physio_type in ["cardiac", "respiratory"]: + physio_objects[col] = physio.Physio( data[idx_0:, columns.index(col)], fs=fs, history=[physio._get_call(exclude=[])], ) - physio_objects[col_physio_type]._physio_type = col_physio_type - physio_objects[col_physio_type]._label = ( + physio_objects[col]._physio_type = col_physio_type + physio_objects[col]._label = ( bids_file[0].filename.split(".")[0].replace("_physio", "") ) From efeaccc4ed4e8b5e2fdf9743067d1f6a2c770b69 Mon Sep 17 00:00:00 2001 From: maestroque Date: Wed, 17 Jul 2024 10:21:48 +0300 Subject: [PATCH 06/16] load_from_bids: return trigger as Physio --- physutils/io.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/physutils/io.py b/physutils/io.py index b5e4eda..7ce89d3 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -116,7 +116,12 @@ def load_from_bids( if col_physio_type == "trigger": # TODO: Implement trigger loading using the MRI data object - logger.warning("Trigger loading not yet implemented") + logger.warning("MRI trigger characteristics extraction not yet implemented") + physio_objects[col] = physio.Physio( + data[idx_0:, columns.index(col)], + fs=fs, + history=[physio._get_call(exclude=[])], + ) return physio_objects From 6bafa700d8993fa6f73cf124460b899fcb272e4c Mon Sep 17 00:00:00 2001 From: maestroque Date: Fri, 9 Aug 2024 02:53:32 +0300 Subject: [PATCH 07/16] Add BIDS structure creation test utils function --- physutils/tests/utils.py | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/physutils/tests/utils.py b/physutils/tests/utils.py index 7a6998f..aa3b9fe 100644 --- a/physutils/tests/utils.py +++ b/physutils/tests/utils.py @@ -2,9 +2,12 @@ Utilities for testing """ +import json +from os import makedirs from os.path import join as pjoin import numpy as np +import pandas as pd from pkg_resources import resource_filename from scipy import signal @@ -77,3 +80,82 @@ def filter_physio(data, cutoffs, method, *, order=3): filtered = physio.new_physio_like(data, signal.filtfilt(b, a, data)) return filtered + + +def create_random_bids_structure(data_dir): + + dataset_description = { + "Name": "Example BIDS Dataset", + "BIDSVersion": "1.7.0", + "License": "", + "Authors": ["Author1", "Author2"], + "Acknowledgements": "", + "HowToAcknowledge": "", + "Funding": "", + "ReferencesAndLinks": "", + "DatasetDOI": "", + } + + physio_json = { + "SamplingFrequency": 10000.0, + "StartTime": -3, + "Columns": [ + "time", + "respiratory_chest", + "trigger", + "cardiac", + "respiratory_CO2", + "respiratory_O2", + ], + } + + # Create BIDS structure directory + subject_id = "01" + session_id = "01" + task_id = "rest" + run_id = "01" + + bids_dir = pjoin( + data_dir, "bids-dir", f"sub-{subject_id}", f"ses-{session_id}", "func" + ) + makedirs(bids_dir, exist_ok=True) + + # Create dataset_description.json + with open(pjoin(data_dir, "bids-dir", "dataset_description.json"), "w") as f: + json.dump(dataset_description, f, indent=4) + + # Create physio.json + with open( + pjoin( + bids_dir, + f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_physio.json", + ), + "w", + ) as f: + json.dump(physio_json, f, indent=4) + + # Initialize tsv file with random data columns and a time column + num_rows = 100000 + num_cols = 5 + time = ( + np.arange(num_rows) / physio_json["SamplingFrequency"] + + physio_json["StartTime"] + ) + data = np.column_stack((time, np.random.rand(num_rows, num_cols - 1).round(8))) + df = pd.DataFrame(data) + tsv_file = pjoin( + bids_dir, + f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_physio.tsv", + ) + df.to_csv(tsv_file, sep="\t", index=False, header=False, float_format="%.8e") + + # Compress tsv file into tsv.gz + tsv_gz_file = pjoin( + bids_dir, + f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_physio.tsv.gz", + ) + pd.read_csv(tsv_file, sep="\t").to_csv( + tsv_gz_file, sep="\t", index=False, compression="gzip" + ) + + return bids_dir From c281917707d0eba7122ab81e197475fc0c11541a Mon Sep 17 00:00:00 2001 From: maestroque Date: Fri, 9 Aug 2024 22:53:57 +0300 Subject: [PATCH 08/16] Add load_from_bids unit test --- physutils/tests/test_io.py | 24 +++++++++++++++++++++++- physutils/tests/utils.py | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/physutils/tests/test_io.py b/physutils/tests/test_io.py index b420243..383acc7 100644 --- a/physutils/tests/test_io.py +++ b/physutils/tests/test_io.py @@ -7,7 +7,11 @@ import pytest from physutils import io, physio -from physutils.tests.utils import filter_physio, get_test_data_path +from physutils.tests.utils import ( + create_random_bids_structure, + filter_physio, + get_test_data_path, +) def test_load_physio(caplog): @@ -46,6 +50,24 @@ def test_load_physio(caplog): io.load_physio([1, 2, 3]) +def test_load_from_bids(): + create_random_bids_structure("physutils/tests/data") + phys_array = io.load_from_bids( + "physutils/tests/data/bids-dir", + subject="01", + session="01", + task="rest", + run="01", + ) + + for col in phys_array.keys(): + assert isinstance(phys_array[col], physio.Physio) + # The data saved are the ones after t_0 = 0s + assert phys_array[col].data.size == 70000 + assert phys_array[col].fs == 10000.0 + assert phys_array[col].history[0][0] == "physutils.io.load_from_bids" + + def test_save_physio(tmpdir): pckl = io.load_physio(get_test_data_path("ECG.phys"), allow_pickle=True) out = io.save_physio(tmpdir.join("tmp").purebasename, pckl) diff --git a/physutils/tests/utils.py b/physutils/tests/utils.py index aa3b9fe..188ea83 100644 --- a/physutils/tests/utils.py +++ b/physutils/tests/utils.py @@ -136,7 +136,7 @@ def create_random_bids_structure(data_dir): # Initialize tsv file with random data columns and a time column num_rows = 100000 - num_cols = 5 + num_cols = 6 time = ( np.arange(num_rows) / physio_json["SamplingFrequency"] + physio_json["StartTime"] From 1b0deaadd7e4fa49a00ec0ed4f73ffa27df70a22 Mon Sep 17 00:00:00 2001 From: maestroque Date: Fri, 9 Aug 2024 22:59:56 +0300 Subject: [PATCH 09/16] Review fixes --- physutils/io.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index 7ce89d3..0d1b59d 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -22,6 +22,7 @@ def load_from_bids( session=None, task=None, run=None, + recording=None, extension="tsv.gz", suffix="physio", ): @@ -53,7 +54,7 @@ def load_from_bids( if not op.exists(bids_path): raise FileNotFoundError(f"Provided path {bids_path} does not exist") - layout = BIDSLayout(bids_path) + layout = BIDSLayout(bids_path, validate=False) bids_file = layout.get( subject=subject, session=session, @@ -61,6 +62,7 @@ def load_from_bids( run=run, suffix=suffix, extension=extension, + recording=recording, ) if len(bids_file) == 0: raise FileNotFoundError( @@ -90,13 +92,13 @@ def load_from_bids( for col in columns: col_physio_type = None - if any([x in col for x in ["cardiac", "ppg", "ecg", "card"]]): + if any([x in col.lower() for x in ["cardiac", "ppg", "ecg", "card"]]): col_physio_type = "cardiac" - elif any([x in col for x in ["respiratory", "rsp", "resp"]]): + elif any([x in col.lower() for x in ["respiratory", "rsp", "resp"]]): col_physio_type = "respiratory" - elif any([x in col for x in ["trigger", "tr"]]): + elif any([x in col.lower() for x in ["trigger", "tr"]]): col_physio_type = "trigger" - elif any([x in col for x in ["time"]]): + elif any([x in col.lower() for x in ["time"]]): continue else: logger.warning( From 0c324c5c04f4142160bc9a0705110cafe67d9473 Mon Sep 17 00:00:00 2001 From: maestroque Date: Sat, 10 Aug 2024 19:10:58 +0300 Subject: [PATCH 10/16] Fix computed_metrics attribute initialization --- physutils/physio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/physutils/physio.py b/physutils/physio.py index b43556e..d61a56e 100644 --- a/physutils/physio.py +++ b/physutils/physio.py @@ -342,7 +342,7 @@ def __init__( reject=np.empty(0, dtype=int), ) self._suppdata = None if suppdata is None else np.asarray(suppdata).squeeze() - self._computed_metrics = [] + self._computed_metrics = dict() def __array__(self): return self.data From 2b0deb935f9d615e5d0110f229836ca1778bccc0 Mon Sep 17 00:00:00 2001 From: maestroque Date: Sat, 10 Aug 2024 20:34:04 +0300 Subject: [PATCH 11/16] Fix new_physio_like computed_metrics copying bug --- physutils/physio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/physutils/physio.py b/physutils/physio.py index d61a56e..33dd678 100644 --- a/physutils/physio.py +++ b/physutils/physio.py @@ -224,7 +224,7 @@ def new_physio_like( label = ref_physio.label if copy_label else None physio_type = ref_physio.physio_type if copy_physio_type else None computed_metrics = ( - list(ref_physio.computed_metrics) if copy_computed_metrics else [] + dict(ref_physio.computed_metrics) if copy_computed_metrics else [] ) # make new class From 038be8659dd96cf10c6316d1e8a8a7de1759b529 Mon Sep 17 00:00:00 2001 From: maestroque Date: Mon, 19 Aug 2024 17:55:40 +0300 Subject: [PATCH 12/16] Minor improvements --- .gitignore | 4 ++++ physutils/io.py | 14 ++++++++++---- physutils/physio.py | 2 +- physutils/tests/test_io.py | 1 + physutils/tests/utils.py | 7 ++++--- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 7c01937..9073dc3 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,7 @@ dmypy.json .pyre/ .vscode/ + +# Test Data +physutils/tests/data/bids-dir +tmp.* diff --git a/physutils/io.py b/physutils/io.py index 0d1b59d..9baea37 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -64,13 +64,14 @@ def load_from_bids( extension=extension, recording=recording, ) + logger.debug(f"BIDS file found: {bids_file}") if len(bids_file) == 0: raise FileNotFoundError( - f"No files found for subject {subject}, session {session}, task {task}, run {run}" + f"No files found for subject {subject}, session {session}, task {task}, run {run}, recording {recording}" ) if len(bids_file) > 1: raise ValueError( - f"Multiple files found for subject {subject}, session {session}, task {task}, run {run}" + f"Multiple files found for subject {subject}, session {session}, task {task}, run {run}, recording {recording}" ) config_file = bids_file[0].get_metadata() @@ -92,9 +93,14 @@ def load_from_bids( for col in columns: col_physio_type = None - if any([x in col.lower() for x in ["cardiac", "ppg", "ecg", "card"]]): + if any([x in col.lower() for x in ["cardiac", "ppg", "ecg", "card", "pulse"]]): col_physio_type = "cardiac" - elif any([x in col.lower() for x in ["respiratory", "rsp", "resp"]]): + elif any( + [ + x in col.lower() + for x in ["respiratory", "rsp", "resp", "breath", "co2", "o2"] + ] + ): col_physio_type = "respiratory" elif any([x in col.lower() for x in ["trigger", "tr"]]): col_physio_type = "trigger" diff --git a/physutils/physio.py b/physutils/physio.py index 33dd678..3da5235 100644 --- a/physutils/physio.py +++ b/physutils/physio.py @@ -224,7 +224,7 @@ def new_physio_like( label = ref_physio.label if copy_label else None physio_type = ref_physio.physio_type if copy_physio_type else None computed_metrics = ( - dict(ref_physio.computed_metrics) if copy_computed_metrics else [] + dict(ref_physio.computed_metrics) if copy_computed_metrics else {} ) # make new class diff --git a/physutils/tests/test_io.py b/physutils/tests/test_io.py index 383acc7..6555d3d 100644 --- a/physutils/tests/test_io.py +++ b/physutils/tests/test_io.py @@ -58,6 +58,7 @@ def test_load_from_bids(): session="01", task="rest", run="01", + recording="cardiac", ) for col in phys_array.keys(): diff --git a/physutils/tests/utils.py b/physutils/tests/utils.py index 188ea83..b4df1d8 100644 --- a/physutils/tests/utils.py +++ b/physutils/tests/utils.py @@ -114,6 +114,7 @@ def create_random_bids_structure(data_dir): session_id = "01" task_id = "rest" run_id = "01" + recording_id = "cardiac" bids_dir = pjoin( data_dir, "bids-dir", f"sub-{subject_id}", f"ses-{session_id}", "func" @@ -128,7 +129,7 @@ def create_random_bids_structure(data_dir): with open( pjoin( bids_dir, - f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_physio.json", + f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}_physio.json", ), "w", ) as f: @@ -145,14 +146,14 @@ def create_random_bids_structure(data_dir): df = pd.DataFrame(data) tsv_file = pjoin( bids_dir, - f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_physio.tsv", + f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}_physio.tsv", ) df.to_csv(tsv_file, sep="\t", index=False, header=False, float_format="%.8e") # Compress tsv file into tsv.gz tsv_gz_file = pjoin( bids_dir, - f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_physio.tsv.gz", + f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}_physio.tsv.gz", ) pd.read_csv(tsv_file, sep="\t").to_csv( tsv_gz_file, sep="\t", index=False, compression="gzip" From 82d6ee9d8dbb2e401fcead146ca627de4e39f1f6 Mon Sep 17 00:00:00 2001 From: maestroque Date: Mon, 19 Aug 2024 17:56:09 +0300 Subject: [PATCH 13/16] matplotlib dependency constraint removal --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ed17a18..c2a6475 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,7 @@ provides = [options] python_requires = >=3.6.1 install_requires = - matplotlib >=3.9 + matplotlib numpy >=1.9.3 scipy loguru From bbe49a6b10eb7d7001321804ee629c2ee0f81a04 Mon Sep 17 00:00:00 2001 From: maestroque Date: Mon, 19 Aug 2024 18:39:48 +0300 Subject: [PATCH 14/16] Integrate StartTime BIDS physio parameter --- physutils/io.py | 4 ++-- physutils/physio.py | 6 ------ physutils/tests/test_io.py | 4 ++-- physutils/tests/utils.py | 2 ++ 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index 9baea37..dc455c7 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -76,7 +76,7 @@ def load_from_bids( config_file = bids_file[0].get_metadata() fs = config_file["SamplingFrequency"] - t_start = config_file["StartTime"] # noqa + t_start = config_file["StartTime"] if "StartTime" in config_file else 0 columns = config_file["Columns"] logger.debug(f"Loaded structure contains columns: {columns}") @@ -84,7 +84,7 @@ def load_from_bids( data = np.loadtxt(bids_file[0].path) if "time" in columns: - idx_0 = np.argmax(data[:, columns.index("time")] >= 0) + idx_0 = np.argmax(data[:, columns.index("time")] >= t_start) else: idx_0 = 0 logger.warning( diff --git a/physutils/physio.py b/physutils/physio.py index 3da5235..8c9ce8f 100644 --- a/physutils/physio.py +++ b/physutils/physio.py @@ -563,12 +563,6 @@ class MRIConfig: def __init__(self, slice_timings=None, n_scans=None, tr=None): if np.ndim(slice_timings) > 1: raise ValueError("Slice timings must be a 1-dimensional array.") - if np.size(slice_timings) != n_scans: - raise ValueError( - "Number of slice timings ({}) must match number of scans ({}).".format( - np.size(slice_timings), n_scans - ) - ) self._slice_timings = np.asarray(slice_timings) self._n_scans = int(n_scans) diff --git a/physutils/tests/test_io.py b/physutils/tests/test_io.py index 6555d3d..bd4fd24 100644 --- a/physutils/tests/test_io.py +++ b/physutils/tests/test_io.py @@ -63,8 +63,8 @@ def test_load_from_bids(): for col in phys_array.keys(): assert isinstance(phys_array[col], physio.Physio) - # The data saved are the ones after t_0 = 0s - assert phys_array[col].data.size == 70000 + # The data saved are the ones after t_0 = -3s + assert phys_array[col].data.size == 80000 assert phys_array[col].fs == 10000.0 assert phys_array[col].history[0][0] == "physutils.io.load_from_bids" diff --git a/physutils/tests/utils.py b/physutils/tests/utils.py index b4df1d8..68dee50 100644 --- a/physutils/tests/utils.py +++ b/physutils/tests/utils.py @@ -138,9 +138,11 @@ def create_random_bids_structure(data_dir): # Initialize tsv file with random data columns and a time column num_rows = 100000 num_cols = 6 + time_offset = 2 time = ( np.arange(num_rows) / physio_json["SamplingFrequency"] + physio_json["StartTime"] + - time_offset ) data = np.column_stack((time, np.random.rand(num_rows, num_cols - 1).round(8))) df = pd.DataFrame(data) From d82ef72ab1d3361052399cb4604671c9dbaed129 Mon Sep 17 00:00:00 2001 From: maestroque Date: Mon, 19 Aug 2024 19:08:18 +0300 Subject: [PATCH 15/16] Minor fix --- physutils/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/physutils/io.py b/physutils/io.py index dc455c7..76a7f75 100644 --- a/physutils/io.py +++ b/physutils/io.py @@ -5,6 +5,7 @@ import importlib import json +import os import os.path as op import numpy as np @@ -140,7 +141,7 @@ def load_physio(data, *, fs=None, dtype=None, history=None, allow_pickle=False): Parameters ---------- - data : str or array_like or Physio_like + data : str, os.path.PathLike or array_like or Physio_like Input physiological data. If array_like, should be one-dimensional fs : float, optional Sampling rate of `data`. Default: None @@ -165,7 +166,7 @@ def load_physio(data, *, fs=None, dtype=None, history=None, allow_pickle=False): # first check if the file was made with `save_physio`; otherwise, try to # load it as a plain text file and instantiate a history - if isinstance(data, str): + if isinstance(data, str) or isinstance(data, os.PathLike): try: inp = dict(np.load(data, allow_pickle=allow_pickle)) for attr in EXPECTED: From ffa8e7152c52f3177966de25adedc5392a8ce732 Mon Sep 17 00:00:00 2001 From: maestroque Date: Wed, 21 Aug 2024 16:26:36 +0200 Subject: [PATCH 16/16] [test] BIDS structure creation utility function fix --- physutils/tests/test_io.py | 20 +++++++++++++++++++- physutils/tests/utils.py | 30 ++++++++++++++++++------------ 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/physutils/tests/test_io.py b/physutils/tests/test_io.py index bd4fd24..b64078a 100644 --- a/physutils/tests/test_io.py +++ b/physutils/tests/test_io.py @@ -51,7 +51,7 @@ def test_load_physio(caplog): def test_load_from_bids(): - create_random_bids_structure("physutils/tests/data") + create_random_bids_structure("physutils/tests/data", recording_id="cardiac") phys_array = io.load_from_bids( "physutils/tests/data/bids-dir", subject="01", @@ -69,6 +69,24 @@ def test_load_from_bids(): assert phys_array[col].history[0][0] == "physutils.io.load_from_bids" +def test_load_from_bids_no_rec(): + create_random_bids_structure("physutils/tests/data") + phys_array = io.load_from_bids( + "physutils/tests/data/bids-dir", + subject="01", + session="01", + task="rest", + run="01", + ) + + for col in phys_array.keys(): + assert isinstance(phys_array[col], physio.Physio) + # The data saved are the ones after t_0 = -3s + assert phys_array[col].data.size == 80000 + assert phys_array[col].fs == 10000.0 + assert phys_array[col].history[0][0] == "physutils.io.load_from_bids" + + def test_save_physio(tmpdir): pckl = io.load_physio(get_test_data_path("ECG.phys"), allow_pickle=True) out = io.save_physio(tmpdir.join("tmp").purebasename, pckl) diff --git a/physutils/tests/utils.py b/physutils/tests/utils.py index 68dee50..fe9c30f 100644 --- a/physutils/tests/utils.py +++ b/physutils/tests/utils.py @@ -82,7 +82,7 @@ def filter_physio(data, cutoffs, method, *, order=3): return filtered -def create_random_bids_structure(data_dir): +def create_random_bids_structure(data_dir, recording_id=None): dataset_description = { "Name": "Example BIDS Dataset", @@ -114,7 +114,7 @@ def create_random_bids_structure(data_dir): session_id = "01" task_id = "rest" run_id = "01" - recording_id = "cardiac" + recording_id = recording_id bids_dir = pjoin( data_dir, "bids-dir", f"sub-{subject_id}", f"ses-{session_id}", "func" @@ -125,11 +125,16 @@ def create_random_bids_structure(data_dir): with open(pjoin(data_dir, "bids-dir", "dataset_description.json"), "w") as f: json.dump(dataset_description, f, indent=4) + if recording_id is not None: + filename_body = f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}" + else: + filename_body = f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}" + # Create physio.json with open( pjoin( bids_dir, - f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}_physio.json", + f"{filename_body}_physio.json", ), "w", ) as f: @@ -146,19 +151,20 @@ def create_random_bids_structure(data_dir): ) data = np.column_stack((time, np.random.rand(num_rows, num_cols - 1).round(8))) df = pd.DataFrame(data) - tsv_file = pjoin( - bids_dir, - f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}_physio.tsv", - ) - df.to_csv(tsv_file, sep="\t", index=False, header=False, float_format="%.8e") - # Compress tsv file into tsv.gz + # Compress dataframe into tsv.gz tsv_gz_file = pjoin( bids_dir, - f"sub-{subject_id}_ses-{session_id}_task-{task_id}_run-{run_id}_recording-{recording_id}_physio.tsv.gz", + f"{filename_body}_physio.tsv.gz", ) - pd.read_csv(tsv_file, sep="\t").to_csv( - tsv_gz_file, sep="\t", index=False, compression="gzip" + + df.to_csv( + tsv_gz_file, + sep="\t", + index=False, + header=False, + float_format="%.8e", + compression="gzip", ) return bids_dir