diff --git a/iblrig/base_choice_world.py b/iblrig/base_choice_world.py index 82cc6c47..c0abc164 100644 --- a/iblrig/base_choice_world.py +++ b/iblrig/base_choice_world.py @@ -833,7 +833,7 @@ def __init__(self, **kwargs): def new_block(self): """ - if block_init_5050 + If block_init_5050 First block has 50/50 probability of leftward stim is 90 trials long """ diff --git a/iblrig/test/test_transfers.py b/iblrig/test/test_transfers.py index 2bb2a67c..76999f60 100644 --- a/iblrig/test/test_transfers.py +++ b/iblrig/test/test_transfers.py @@ -145,7 +145,7 @@ def create_fake_data(self): raw_photometry_df.to_csv(folder_neurophotometrics / 'raw_photometry.csv', index=False) def test_copier(self): - session = _create_behavior_session(ntrials=50, kwargs=self.session_kwargs) + # session = _create_behavior_session(ntrials=50, kwargs=self.session_kwargs) self.create_fake_data() # the workaround to find the settings.yaml @@ -153,11 +153,13 @@ def test_copier(self): mocker.side_effect = self.side_effect # the actual code to test iblrig.neurophotometrics.init_neurophotometrics_subject( - session_stub=session.paths['SESSION_FOLDER'], + session_stub=f'test_subject/{datetime.today().strftime("%Y-%m-%d")}/001', rois=['Region00', 'Region01'], locations=['VTA', 'SNc'], ) - iblrig.neurophotometrics.copy_photometry_subject(session.paths['SESSION_FOLDER']) + # iblrig.neurophotometrics.copy_photometry_subject(session.paths['SESSION_FOLDER']) + (sc,) = iblrig.commands.transfer_data(tag='neurophotometrics') + self.assertEqual(sc.state, 2) class TestIntegrationTransferExperiments(TestIntegrationTransferExperimentsBase): diff --git a/iblrig/transfer_experiments.py b/iblrig/transfer_experiments.py index 52361941..a5872079 100644 --- a/iblrig/transfer_experiments.py +++ b/iblrig/transfer_experiments.py @@ -12,10 +12,9 @@ from pathlib import Path import numpy as np -import pandas as pd -import pandera import ibllib.pipes.misc +import iblphotometry.io as fpio import iblrig import one.alf.path as alfiles from ibllib.io import raw_data_loaders, session_params @@ -679,11 +678,17 @@ def neurophotometrics_description( description['fibers'] = {roi: {'location': location} for roi, location in zip(rois, locations, strict=False)} return {'neurophotometrics': description} - def _copy_collections(self, folder_neurophotometric: Path) -> bool: + def _copy_collections(self, folder_neurophotometric: Path | None = None) -> bool: ed = self.experiment_description['neurophotometrics'] dt = datetime.datetime.fromisoformat(ed['datetime']) # Here we find the first photometry folder after the start_time. In case this is failing # we can feed a custom start_time to go to the desired folder, or just rename the folder + # FIXME TODO + folder_neurophotometric = ( + self.session_path.parents[4].joinpath('neurophotometrics') + if folder_neurophotometric is None + else folder_neurophotometric + ) folder_day = next(folder_neurophotometric.glob(ed['datetime'][:10]), None) assert folder_day is not None, f"Neurophotometrics folder {folder_neurophotometric} doesn't contain data" folder_times = list(folder_day.glob('T*')) @@ -694,32 +699,14 @@ def _copy_collections(self, folder_neurophotometric: Path) -> bool: csv_digital_inputs = folder_day.joinpath(f'T{hhmmss[i]}', 'digital_inputs.csv') assert csv_raw_photometry.exists(), f'Raw photometry file {csv_raw_photometry} not found' assert csv_digital_inputs.exists(), f'Digital inputs file {csv_digital_inputs} not found' + # Copy the raw and digital inputs files to the server - # TODO move this into a data loader ? Especially the schemas will apply to both the csv and parquet format - df_raw_photometry = pd.read_csv(csv_raw_photometry) - df_digital_inputs = pd.read_csv(csv_digital_inputs, header=None) - df_digital_inputs.columns = ['ChannelName', 'Channel', 'AlwaysTrue', 'SystemTimestamp', 'ComputerTimestamp'] - # this will ensure the columns are present, and that there was no magic new format on a new Bonsai version - schema_raw_data = pandera.DataFrameSchema( - columns=dict( - FrameCounter=pandera.Column(pandera.Int64), - SystemTimestamp=pandera.Column(pandera.Float64), - LedState=pandera.Column(pandera.Int16, coerce=True), - ComputerTimestamp=pandera.Column(pandera.Float64), - **{k: pandera.Column(pandera.Float64) for k in ed['fibers']}, - ) - ) - schema_digital_inputs = pandera.DataFrameSchema( - columns=dict( - ChannelName=pandera.Column(str, coerce=True), - Channel=pandera.Column(pandera.Int8, coerce=True), - AlwaysTrue=pandera.Column(bool, coerce=True), - SystemTimestamp=pandera.Column(pandera.Float64), - ComputerTimestamp=pandera.Column(pandera.Float64), - ) - ) - df_raw_photometry = schema_raw_data.validate(df_raw_photometry) - df_digital_inputs = schema_digital_inputs.validate(df_digital_inputs) + # read in and + df_raw_photometry = fpio.from_raw_neurophotometrics_file_to_raw_df(csv_raw_photometry, validate=False) + # explicitly explicitly with the data from the experiment description file + cols = ed['fibers'].keys() + df_raw_photometry = fpio.validate_neurophotometrics_df(df_raw_photometry, data_columns=cols) + df_digital_inputs = fpio.read_digital_inputs_csv(csv_digital_inputs, validate=True) remote_photometry_path = self.remote_session_path.joinpath(ed['collection']) remote_photometry_path.mkdir(parents=True, exist_ok=True) df_raw_photometry.to_parquet(remote_photometry_path.joinpath('_neurophotometrics_fpData.raw.pqt')) diff --git a/pdm.lock b/pdm.lock index 2ea5974d..a9449fa6 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "ci", "dev", "doc", "project-extraction", "test", "typing"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:929627e3cbf02e2d0e6981063bcea0ca78239f62528cfd70c906d929786bbb98" +content_hash = "sha256:53d0e1a0887da72fdb394bc536560b1456d42f9f417d9cf7316960f85fdfea21" [[metadata.targets]] requires_python = "==3.10.*" @@ -783,6 +783,9 @@ files = [ name = "ibllib" version = "3.1.0" requires_python = ">=3.10" +git = "https://github.com/int-brain-lab/ibllib" +ref = "develop" +revision = "3c82b302118408c6ec1228ad2e76ba1939f7e278" summary = "IBL libraries" groups = ["default"] dependencies = [ @@ -819,9 +822,23 @@ dependencies = [ "sparse", "tqdm>=4.32.1", ] -files = [ - {file = "ibllib-3.1.0-py3-none-any.whl", hash = "sha256:489adba69fd5b87fa92aacc11c053a462848e1df54eccb1bfe42baebd1157d89"}, - {file = "ibllib-3.1.0.tar.gz", hash = "sha256:c099e1e9062dd72915e86da6bbe2621b043022c62656a96490fbacd06c41d21f"}, + +[[package]] +name = "iblphotometry" +version = "1.0.0" +requires_python = ">=3.10" +git = "https://github.com/int-brain-lab/ibl-photometry" +ref = "main" +revision = "09983f65394a6eb914e4a1e7c18a8ec7e0913454" +summary = "IBL photometry module" +groups = ["default"] +dependencies = [ + "ibllib @ git+https://github.com/int-brain-lab/ibllib@develop", + "matplotlib", + "numpy", + "pandera", + "pytest", + "scipy", ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 6e1bbf90..0bace38a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "iblqt>=0.3.1", "ONE-api>=2.11.1", "tycmd-wrapper>=0.2.1", + "ibl-photometry", # # Everything else "annotated-types>=0.7.0",