Improvements to experiment error handling

src/auspex/experiment.py

@@ -384,16 +384,33 @@ def sweep(self):
 
     def connect_instruments(self):
         # Connect the instruments to their resources
-        if not self.instrs_connected:
+        if self.instrs_connected == False:
+            connected_list = []
             for instrument in self._instruments.values():
-                instrument.connect()
+                try:
+                    instrument.connect()
+                    connected_list.append(instrument)
+                except:
+                    logger.error(f"Failed to connect to instrument {instrument.name}")
+                    logger.error("Disconnecting from other connected instruments")
+                    for instr in connected_list:
+                        try:
+                            instr.disconnect()
+                        except:
+                            logger.error(f"Failed to disconnect from {instr.name}")
+                    raise Exception(f"Failed to connect to all instruments; disconnected as best as possible")
             self.instrs_connected = True
 
     def disconnect_instruments(self):
         # Connect the instruments to their resources
-        for instrument in self._instruments.values():
-            instrument.disconnect()
-        self.instrs_connected = False
+        if self.instrs_connected == True:
+            for instrument in self._instruments.values():
+                try:
+                    instrument.disconnect()
+                except:
+                    logger.error(f"Failed to disconnect from {instrument.name}")
+                    #This probably should have a fail flag or something to throw a higher error after it's done trying to disconnect
+            self.instrs_connected = False
 
     def init_dashboard(self):
         from bqplot import DateScale, LinearScale, DateScale, Axis, Lines, Figure, Tooltip
@@ -549,6 +566,7 @@ def run_sweeps(self):
         time.sleep(0.1)
         #connect all instruments
         self.connect_instruments()
+        assert self.instrs_connected == True, "Instruments were not connected successfully."
 
         try:
             #initialize instruments

src/auspex/qubit/qubit_exp.py

@@ -569,22 +569,44 @@ def shutdown_instruments(self):
         logger.debug("Shutting down instruments")
         try:
             for awg in self.awgs:
-                awg.stop()
+                try:
+                    awg.stop()
+                except:
+                    logger.error(f"Could not stop AWG {awg.name}")
+                    raise Exception(f"Could not stop AWG {awg.name}")
             for dig in self.digitizers:
-                dig.stop()
+                try:
+                    dig.stop()
+                except:
+                    logger.error(f"Could not stop digitizer {dig.name}")
+                    raise Exception(f"Could not stop digitizer {dig.name}")
             for gen_proxy in self.generators:
-                gen_proxy.instr.output = False
+                try:
+                    gen_proxy.instr.output = False
+                except:
+                    logger.error(f"Could not set {gen_proxy.name} output to false")
+                    raise Exception(f"Could not set {gen_proxy.name} output to false")
         except:
-            logger.error('Could Not Stop AWGs or Digitizers; Reset Experiment')
+            logger.error('Could Not Stop AWGs or Digitizers; Reset Experiment') 
+        failflag = False
         for instr in self.instruments:
-            instr.disconnect()
-        self.dig_exit.set()
-        for listener in self.dig_listeners:
-            listener.join(2)
-            if listener.is_alive():
-                logger.debug(f"Terminating listener {listener} aggressively")
-                listener.terminate()
-            del listener
+            try:
+                instr.disconnect()
+            except:
+                logger.error(f"Could not disconnect instrument {instr.name}")
+                failflag = True
+        if failflag is True:
+            logger.error('Could not disconnect from some number of instruments, they may need to be reset.')
+
+        #Ensure that the digitizer-related attributes were created, since they aren't in certain failure conditions.
+        if hasattr(self,"dig_exit") and hasattr(self, "dig_listeners"):
+            self.dig_exit.set()
+            for listener in self.dig_listeners:
+                listener.join(2)
+                if listener.is_alive():
+                    logger.debug(f"Terminating listener {listener} aggressively")
+                    listener.terminate()
+                del listener
 
         import gc
         gc.collect()