diff --git a/src/auspex/experiment.py b/src/auspex/experiment.py index 9a68a5458..d79000594 100644 --- a/src/auspex/experiment.py +++ b/src/auspex/experiment.py @@ -384,16 +384,33 @@ def sweep(self): def connect_instruments(self): # Connect the instruments to their resources - if not self.instrs_connected: + if self.instrs_connected == False: + connected_list = [] for instrument in self._instruments.values(): - instrument.connect() + try: + instrument.connect() + connected_list.append(instrument) + except: + logger.error(f"Failed to connect to instrument {instrument.name}") + logger.error("Disconnecting from other connected instruments") + for instr in connected_list: + try: + instr.disconnect() + except: + logger.error(f"Failed to disconnect from {instr.name}") + raise Exception(f"Failed to connect to all instruments; disconnected as best as possible") self.instrs_connected = True def disconnect_instruments(self): # Connect the instruments to their resources - for instrument in self._instruments.values(): - instrument.disconnect() - self.instrs_connected = False + if self.instrs_connected == True: + for instrument in self._instruments.values(): + try: + instrument.disconnect() + except: + logger.error(f"Failed to disconnect from {instrument.name}") + #This probably should have a fail flag or something to throw a higher error after it's done trying to disconnect + self.instrs_connected = False def init_dashboard(self): from bqplot import DateScale, LinearScale, DateScale, Axis, Lines, Figure, Tooltip @@ -549,6 +566,7 @@ def run_sweeps(self): time.sleep(0.1) #connect all instruments self.connect_instruments() + assert self.instrs_connected == True, "Instruments were not connected successfully." try: #initialize instruments diff --git a/src/auspex/qubit/qubit_exp.py b/src/auspex/qubit/qubit_exp.py index 18bfd0c29..fbac3b4d3 100644 --- a/src/auspex/qubit/qubit_exp.py +++ b/src/auspex/qubit/qubit_exp.py @@ -569,22 +569,44 @@ def shutdown_instruments(self): logger.debug("Shutting down instruments") try: for awg in self.awgs: - awg.stop() + try: + awg.stop() + except: + logger.error(f"Could not stop AWG {awg.name}") + raise Exception(f"Could not stop AWG {awg.name}") for dig in self.digitizers: - dig.stop() + try: + dig.stop() + except: + logger.error(f"Could not stop digitizer {dig.name}") + raise Exception(f"Could not stop digitizer {dig.name}") for gen_proxy in self.generators: - gen_proxy.instr.output = False + try: + gen_proxy.instr.output = False + except: + logger.error(f"Could not set {gen_proxy.name} output to false") + raise Exception(f"Could not set {gen_proxy.name} output to false") except: - logger.error('Could Not Stop AWGs or Digitizers; Reset Experiment') + logger.error('Could Not Stop AWGs or Digitizers; Reset Experiment') + failflag = False for instr in self.instruments: - instr.disconnect() - self.dig_exit.set() - for listener in self.dig_listeners: - listener.join(2) - if listener.is_alive(): - logger.debug(f"Terminating listener {listener} aggressively") - listener.terminate() - del listener + try: + instr.disconnect() + except: + logger.error(f"Could not disconnect instrument {instr.name}") + failflag = True + if failflag is True: + logger.error('Could not disconnect from some number of instruments, they may need to be reset.') + + #Ensure that the digitizer-related attributes were created, since they aren't in certain failure conditions. + if hasattr(self,"dig_exit") and hasattr(self, "dig_listeners"): + self.dig_exit.set() + for listener in self.dig_listeners: + listener.join(2) + if listener.is_alive(): + logger.debug(f"Terminating listener {listener} aggressively") + listener.terminate() + del listener import gc gc.collect()