Merge pull request #513 from int-brain-lab/iblrigv8dev

8.10.2

CHANGELOG.md

@@ -3,6 +3,14 @@ Changelog
 
 -------------------------------
 
+8.10.2
+------
+* hot-fix parsing of path args in transfer_data
+* add install_spinnaker command for ... installing spinnaker
+* fixed CI warnings about ports that haven't been closed
+* draw subject weight for adaptive reward from previous session
+* format reward with 1 decimal on online plot
+
 8.10.1
 ------
 * more reliable way to check for dirty repository

iblrig/__init__.py

@@ -4,7 +4,7 @@
 # 3) Check CI and eventually wet lab test
 # 4) Pull request to iblrigv8
 # 5) git tag the release in accordance to the version number below (after merge!)
-__version__ = '8.10.1'
+__version__ = '8.10.2'
 
 # The following method call will try to get post-release information (i.e. the number of commits since the last tagged
 # release corresponding to the one above), plus information about the state of the local repository (dirty/broken)

iblrig/base_choice_world.py

@@ -748,7 +748,6 @@ def get_subject_training_info(self):
         try:
             training_phase, adaptive_reward, _ = choiceworld.get_subject_training_info(
                 subject_name=self.session_info.SUBJECT_NAME,
-                subject_weight_grams=self.session_info['SUBJECT_WEIGHT'],
                 default_reward=self.task_params.REWARD_AMOUNT_UL,
                 local_path=self.iblrig_settings['iblrig_local_data_path'],
                 remote_path=self.iblrig_settings['iblrig_remote_data_path'],

iblrig/base_tasks.py

@@ -39,6 +39,9 @@
 import ibllib.io.session_params as ses_params
 from iblrig.transfer_experiments import BehaviorCopier
 
+# if HAS_PYSPIN:
+#     import PySpin
+
 OSC_CLIENT_IP = "127.0.0.1"
 
 
@@ -446,6 +449,9 @@ class OSCClient(udp_client.SimpleUDPClient):
     def __init__(self, port, ip="127.0.0.1"):
         super(OSCClient, self).__init__(ip, port)
 
+    def __del__(self):
+        self._sock.close()
+
     def send2bonsai(self, **kwargs):
         """
         :param see list of keys in OSC_PROTOCOL
@@ -518,11 +524,18 @@ def start_mixin_bonsai_cameras(self):
         desired borders of rig features, the actual triggering of the  cameras is done in the trigger_bonsai_cameras method.
         """
 
-        # TODO: spinnaker SDK
-
         if self._camera_mixin_bonsai_get_workflow_file(self.hardware_settings.get('device_cameras', None)) is None:
             return
 
+        # # TODO
+        # # enable trigger mode - if PySpin is available
+        # if HAS_PYSPIN:
+        #     pyspin_system = PySpin.System.GetInstance()
+        #     pyspin_cameras = pyspin_system.GetCameras()
+        #     for cam in pyspin_cameras:
+        #         cam.Init()
+        #         cam.TriggerMode.SetValue(True)
+
         bonsai_camera_file = self.paths.IBLRIG_FOLDER.joinpath('devices', 'camera_setup', 'setup_video.bonsai')
         # this locks until Bonsai closes
         cmd = [str(self.paths.BONSAI), str(bonsai_camera_file), "--start-no-debug", "--no-boot"]

iblrig/camera.py

@@ -0,0 +1,83 @@
+import sys
+import os
+import subprocess
+from importlib.util import find_spec
+import zipfile
+from pathlib import Path
+from shutil import which
+
+from one.webclient import AlyxClient, http_download_file
+from iblutil.io import hashfile
+
+
+def pyspin_installed() -> bool:
+    return find_spec('PySpin') is not None
+
+
+def spinnaker_sdk_installed() -> bool:
+    if os.name != 'nt':
+        return False
+    spin_exe = which('SpinUpdateConsole_v140')
+    return spin_exe and Path(spin_exe).parents[2].joinpath('src').exists()
+
+
+def install_spinnaker_sdk():
+    def download(asset: int, filename: str, target_md5: str):
+        print(f'Downloading {filename} ...')
+        out_dir = Path.home().joinpath('Downloads')
+        out_file = out_dir.joinpath(filename)
+        options = {'target_dir': out_dir, 'clobber': True, 'return_md5': True}
+        if out_file.exists() and hashfile.md5(out_file) == target_md5:
+            return out_file
+        try:
+            tmp_file, md5_sum = AlyxClient().download_file(f'resources/spinnaker/{filename}', **options)
+        except OSError as e1:
+            try:
+                url = f'https://flir.nsetx.net/file/asset/{asset}/original/attachment'
+                tmp_file, md5_sum = http_download_file(url, **options)
+            except OSError as e2:
+                raise e2 from e1
+        os.rename(tmp_file, out_file)
+        if md5_sum != target_md5:
+            raise Exception(f'`{filename}` does not match the expected MD5 - please try running the script again or')
+        return out_file
+
+    # Check prerequisites
+    if os.name != 'nt':
+        raise Exception(f'{Path(__file__).name} can only be run on Windows.')
+    if sys.base_prefix == sys.prefix:
+        raise Exception(f'{Path(__file__).name} needs to be started in the IBLRIG venv.')
+
+    # Display some information
+    print('This script will try to automatically\n'
+          '   1) Download & install Spinnaker SDK for Windows, and\n'
+          '   2) Download & install PySpin to the IBLRIG Python environment.')
+    input('Press [ENTER] to continue.\n')
+
+    # Download & install Spinnaker SDK
+    if spinnaker_sdk_installed():
+        print('Spinnaker SDK for Windows is already installed.')
+    else:
+        file_winsdk = download(54386, 'SpinnakerSDK_FULL_3.1.0.79_x64.exe', 'd9d83772f852e5369da2fbcc248c9c81')
+        print('Installing Spinnaker SDK for Windows ...')
+        input('Please select the "Application Development" Installation Profile. Everything else can be left at '
+              'default values. Press [ENTER] to continue.')
+        return_code = subprocess.check_call(file_winsdk)
+        if return_code == 0 and spinnaker_sdk_installed():
+            print('Installation of Spinnaker SDK was successful.')
+        os.unlink(file_winsdk)
+
+    # Download & install PySpin
+    if pyspin_installed():
+        print('PySpin is already installed.')
+    else:
+        file_zip = download(54396, 'spinnaker_python-3.1.0.79-cp310-cp310-win_amd64.zip',
+                            'e00148800757d0ed7171348d850947ac')
+        print('Installing PySpin ...')
+        with zipfile.ZipFile(file_zip, 'r') as f:
+            file_whl = f.extract(file_zip.stem + '.whl', file_zip.parent)
+        return_code = subprocess.check_call([sys.executable, "-m", "pip", "install", file_whl])
+        if return_code == 0:
+            print('Installation of PySpin was successful.')
+        os.unlink(file_whl)
+        file_zip.unlink()

iblrig/choiceworld.py

@@ -37,7 +37,7 @@ def compute_adaptive_reward_volume(subject_weight_g, reward_volume_ul, delivered
 
 
 def get_subject_training_info(
-        subject_name, subject_weight_grams=None, task_name='_iblrig_tasks_trainingChoiceWorld',
+        subject_name, task_name='_iblrig_tasks_trainingChoiceWorld',
         default_reward=DEFAULT_REWARD_VOLUME, mode='silent', **kwargs):
     """
     Goes through the history of a subject and gets the latest
@@ -50,14 +50,14 @@ def get_subject_training_info(
     :param mode: 'defaults' or 'raise': if 'defaults' returns default values if no history is found, if 'raise' raises ValueError
     :param **kwargs: optional arguments to be passed to iblrig.path_helper.get_local_and_remote_paths
     if not used, will use the arguments from iblrig/settings/iblrig_settings.yaml
-    :return: training_phase (int), default_reward uL (float between 1.5 and 3) and status (True if previous was found,
-    False if unable and default values were returned)
+    :return: training_phase (int), default_reward uL (float between 1.5 and 3) and a
+    session_info dictionary with keys: session_path, experiment_description, task_settings, file_task_data
     """
     session_info = iterate_previous_sessions(subject_name, task_name=task_name, n=1, **kwargs)
     if len(session_info) == 0:
         if mode == 'silent':
             logger.warning("The training status could not be determined returning default values")
-            return DEFAULT_TRAINING_PHASE, default_reward, False
+            return DEFAULT_TRAINING_PHASE, default_reward, None
         elif mode == 'raise':
             raise ValueError("The training status could not be determined as no previous sessions were found")
     else:
@@ -66,15 +66,15 @@ def get_subject_training_info(
     previous_reward_volume = (session_info.task_settings.get('ADAPTIVE_REWARD_AMOUNT_UL') or
                               session_info.task_settings.get('REWARD_AMOUNT_UL'))
     adaptive_reward = compute_adaptive_reward_volume(
-        subject_weight_g=subject_weight_grams or session_info.task_settings['SUBJECT_WEIGHT'],
+        subject_weight_g=session_info.task_settings['SUBJECT_WEIGHT'],
         reward_volume_ul=previous_reward_volume,
         delivered_volume_ul=trials_data['reward_amount'].sum(),
         ntrials=trials_data.shape[0])
     if 'training_phase' in trials_data:
         training_phase = trials_data['training_phase'].values[-1]
     else:
         training_phase = DEFAULT_TRAINING_PHASE
-    return training_phase, adaptive_reward, True
+    return training_phase, adaptive_reward, session_info
 
 
 def training_contrasts_probabilities(phase=1):

iblrig/commands.py

@@ -35,12 +35,17 @@ def transfer_data(local_path=None, remote_path=None, dry=False):
     """
     Copies the behavior data from the rig to the local server if the session has more than 42 trials
     If the hardware settings file contains MAIN_SYNC=True, the number of expected devices is set to 1
-    :param local_path: local path to the subjects folder
-    :param weeks:
+    :param local_path: local path to the subjects folder, otherwise uses the local_data_folder key in
+    the iblrig_settings.yaml file, or the iblrig_data directory in the home path.
     :param dry:
     :return:
     """
+    # If paths not passed, uses those defined in the iblrig_settings.yaml file
     rig_paths = get_local_and_remote_paths(local_path=local_path, remote_path=remote_path)
+    local_path = rig_paths.local_subjects_folder
+    remote_path = rig_paths.remote_subjects_folder
+    assert isinstance(local_path, Path)  # get_local_and_remote_paths should always return Path obj
+
     hardware_settings = load_settings_yaml('hardware_settings.yaml')
     number_of_expected_devices = 1 if hardware_settings.get('MAIN_SYNC', True) else None
 

iblrig/constants.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 from shutil import which
+from iblrig.camera import spinnaker_sdk_installed, pyspin_installed
 
 BASE_DIR = str(Path(__file__).parents[1])
 IS_GIT = Path(BASE_DIR).joinpath('.git').exists() and which('git') is not None
+HAS_PYSPIN = spinnaker_sdk_installed() and pyspin_installed()

iblrig/gui/wizard.py

@@ -371,11 +371,12 @@ def controls_for_extra_parameters(self):
                     widget.setSpecialValueText('automatic')
                     widget.setMaximum(3)
                     widget.setSingleStep(0.1)
-                    widget.setMinimum(-1)
+                    widget.setMinimum(1.4)
                     widget.setValue(widget.minimum())
                     widget.valueChanged.connect(
-                        lambda val, a=arg:
-                        self._set_task_arg(a.option_strings[0], str(val if val > widget.minimum() else -1)))
+                        lambda val, a=arg, m=widget.minimum():
+                        self._set_task_arg(a.option_strings[0], str(val if val > m else -1)))
+                    widget.valueChanged.emit(widget.value())
 
             layout.addRow(self.tr(label), widget)
 
@@ -420,12 +421,16 @@ def start_stop(self):
         match self.uiPushStart.text():
             case 'Start':
                 self.uiPushStart.setText('Stop')
+                self.uiPushStart.setIcon(self.style().standardIcon(QStyle.SP_MediaStop))
                 self.enable_UI_elements()
 
                 dlg = QtWidgets.QInputDialog()
                 weight, ok = dlg.getDouble(self, 'Subject Weight', 'Subject Weight (g):', value=0, min=0,
                                            flags=dlg.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint)
                 if not ok or weight == 0:
+                    self.uiPushStart.setText('Start')
+                    self.uiPushStart.setIcon(self.style().standardIcon(QStyle.SP_MediaPlay))
+                    self.enable_UI_elements()
                     return
 
                 self.controller2model()
@@ -460,10 +465,9 @@ def start_stop(self):
                 self.checkSubProcessTimer.start(1000)
             case 'Stop':
                 self.uiPushStart.setText('Stop')
-                self.uiPushStart.setEnabled(False)
                 self.checkSubProcessTimer.stop()
                 # if the process crashed catastrophically, the session folder might not exist
-                if self.model.session_folder.exists():
+                if self.model.session_folder and self.model.session_folder.exists():
                     self.model.session_folder.joinpath('.stop').touch()
 
                 # this will wait for the process to finish, usually the time for the trial to end
@@ -474,6 +478,7 @@ def start_stop(self):
                     msgBox.setText("The task was terminated with an error.\nPlease check the command-line output for details.")
                     msgBox.setIcon(QtWidgets.QMessageBox().Critical)
                     msgBox.exec_()
+
                 self.running_task_process = None
 
                 # manage poop count

iblrig/online_plots.py

@@ -255,7 +255,7 @@ def __init__(self, task_file=None):
     def update_titles(self):
         self.h.fig_title.set_text(
             f"{self._session_string} time elapsed: {str(datetime.timedelta(seconds=int(self.data.time_elapsed)))}")
-        self.h.ax_water.title.set_text(f"water \n {int(self.data.water_delivered)} (uL)")
+        self.h.ax_water.title.set_text(f"water \n {self.data.water_delivered:.2f} (uL)")
         self.h.ax_performance.title.set_text(f" correct/tot \n {self.data.ntrials_correct} / {self.data.ntrials}")
 
     def update_trial(self, trial_data, bpod_data):

iblrig/test/test_choice_world.py

@@ -84,11 +84,14 @@ def test_adaptive_training_level(self):
         """
         self.mock_jsonable(self.sesb.paths.DATA_FILE_PATH, training_phase=2, reward_amount=1050)
         self.sesb.session_info['ADAPTIVE_REWARD_AMOUNT_UL'] = 2.1
+        self.sesb.session_info['SUBJECT_WEIGHT'] = 17
+
         self.sesb.save_task_parameters_to_json_file()
         # test the function entry point
-        result = iblrig.choiceworld.get_subject_training_info(
-            self.kwargs['subject'], subject_weight_grams=17, local_path=Path(self.root_path), lab='cortexlab', mode='raise')
-        self.assertEqual((2, 2.1, True), result)
+        a, b, info = iblrig.choiceworld.get_subject_training_info(
+            self.kwargs['subject'], local_path=Path(self.root_path), lab='cortexlab', mode='raise')
+        self.assertEqual((2, 2.1), (a, b))
+        self.assertIsInstance(info, dict)
 
         # test the task instantiation, should be the same as above
         t = TrainingChoiceWorldSession(**self.kwargs, training_phase=4, adaptive_reward=2.9)

iblrig/test/test_transfers.py

@@ -3,6 +3,7 @@
 import random
 import tempfile
 import unittest
+from unittest import mock
 
 from ibllib.io import session_params
 
@@ -63,6 +64,16 @@ def test_behavior_copy_complete_session(self):
                                     remote_subjects_folder=session.paths.REMOTE_SUBJECT_FOLDER)
                 self.assertEqual(sc.state, 3)
 
+        # Check that the settings file is used when no path passed
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td:
+            session = _create_behavior_session(td, ntrials=50, hard_crash=hard_crash)
+            session.paths.SESSION_FOLDER.joinpath('transfer_me.flag').touch()
+            with mock.patch('iblrig.path_helper.load_settings_yaml', return_value=session.iblrig_settings):
+                iblrig.commands.transfer_data()
+            sc = BehaviorCopier(session_path=session.paths.SESSION_FOLDER,
+                                remote_subjects_folder=session.paths.REMOTE_SUBJECT_FOLDER)
+            self.assertEqual(sc.state, 3)
+
     def test_behavior_do_not_copy_dummy_sessions(self):
         """
         Here we test the case when an aborted session or a session with less than 42 trials attempts to be copied

iblrig/version_management.py

@@ -132,14 +132,14 @@ def get_detailed_version_string(v_basic: str) -> str:
     # get details through `git describe`
     try:
         get_remote_tags()
-        v_detailed = check_output(["git", "describe", "--dirty", "--broken", "--match", v_sanitized, "--tags", "--long"],
-                                  cwd=BASE_DIR, text=True, timeout=1, stderr=STDOUT)
-    except (SubprocessError, CalledProcessError):
-        log.error('Error calling `git describe`')
+        v_detailed = check_output(["git", "describe", "--dirty", "--broken", "--match", v_sanitized,
+                                   "--tags", "--long"], cwd=BASE_DIR, text=True, timeout=1, stderr=STDOUT)
+    except (SubprocessError, CalledProcessError) as e:
+        log.debug(e, exc_info=True)
         return v_basic
 
     # apply a bit of regex magic for formatting & return the detailed version string
-    v_detailed = re.sub(r'^((?:[\d+\.])+)(-[1-9]{1}\d*)?(?:-0\d*)?(?:-\w+)(-dirty|-broken)?\n?$', r'\1\2\3', v_detailed)
+    v_detailed = re.sub(r'^((?:[\d+\.])+)(-[1-9]\d*)?(?:-0\d*)?(?:-\w+)(-dirty|-broken)?\n?$', r'\1\2\3', v_detailed)
     v_detailed = re.sub(r'-(\d+)', r'.post\1', v_detailed)
     v_detailed = re.sub(r'\-(dirty|broken)', r'+\1', v_detailed)
     return v_detailed

pyproject.toml

@@ -55,6 +55,7 @@ flush = "iblrig.commands:flush"
 remove-old-sessions = "iblrig.commands:remove_local_sessions"
 iblrig = "iblrig.gui.wizard:main"
 upgrade_iblrig = "iblrig.version_management:upgrade"
+install_spinnaker = "iblrig.video:install_spinnaker_sdk"
 
 [tool.setuptools.dynamic]
 readme = {file = "README.md", content-type = "text/markdown"}