Merge pull request #55 from scriptorron/53-imx290-crashes-or-hangs-af…

…ter-first-exposure

53 imx290 crashes or hangs after first exposure

CHANGELOG

@@ -1,3 +1,7 @@
+2.4.0
+- added INI switch "force_Restart" and implemented camera restart (solves crashes for IMX290 and IMX519)
+- forwarding log messages to client
+
 2.3.0
 - update FITS header formatting and timestamps
 - use lxml for construction of indi_pylibcamera.xml

README.md

@@ -12,6 +12,19 @@ The "indi_pylibcamera" may support all cameras supported by "libcamera". But not
 in the required formats (raw Bayer or at least RGB). So it is not guaranteed that the driver will work with all
 cameras you can connect to a Raspberry Pi.
 
+The 'indi_pylibcamera' is one layer in a stack of software:
+```
+    INDI client (for instance KStars, PHD2, CCDciel, ...)
+        --> INDI server
+            --> indi_pylibcamera
+                --> picamera2
+                    --> libcamera library
+                        --> kernel driver
+```
+It can not work when the versions of `libcamera` and `picamera2` are too old (both are in a dynamic development).
+And it can not work when the libcamera-tools (like `libcamera-hello` and `libcamera-still`) have issues with your
+camera.
+
 ## Requirements and installation
 Some packages need to be installed with apt-get:
 - `libcamera` (if not already installed). You can test libcamera and the support
@@ -94,6 +107,14 @@ at the beginning of the driver initialization. This can be done here in the INI
 indi_pylibcamera driver uses this feature to get observer location, telescope information and telescope direction
 from the mount driver. It writes these information as metadata in the FITS images. This function got newly implemented
 and may make trouble in some setups. With the `DoSnooping` you can disable this function.
+- `force_Restart` (`yes`, `no`, `auto`): Some cameras crash after the first exposure. Restarting the camera before
+every frame exposure can solve this issue. Valid values of this switch are:
+  * `no`: Do not restart if not needed to reconfigure camera.
+  * `yes`: Always restart. Can lead to longer time between frames.
+  * `auto`: Automatically choose based on list of known critical cameras. 
+
+  Default (if not otherwise set in INI file) is `auto`.
+
 
 There are more settings, mostly to support debugging.
 
@@ -287,9 +308,10 @@ exposure time was successful.
 and has therefore well-defined restrictions. It is not clear if the reported higher maximum analogue gain is correct.
 
 ## Credits
-Many thanks to all who helped o improve this software. Contributions came from:
+Many thanks to all who helped to improve this software. Contributions came from:
 - Simon Šander
 - Aaron W Morris
 - Caden Gobat
+- anjok
 
 I hope I did not forget someone. If so please do not be angry and tell me.

src/indi_pylibcamera/CameraControl.py

@@ -1,7 +1,6 @@
 """
 indi_pylibcamera: CameraControl class
 """
-import logging
 import os.path
 import numpy as np
 import io
@@ -164,8 +163,8 @@ def is_ReconfigurationNeeded(self, NewCameraSettings):
         return is_ReconfigurationNeeded
 
     def __str__(self):
-        return f'CameraSettings: FastExposure={self.DoFastExposure}, DoRaw={self.DoRaw}, ProcSize={self.ProcSize}, ` +\
-        `RawMode={self.RawMode}, CameraControls={self.camera_controls}'
+        return f'CameraSettings: FastExposure={self.DoFastExposure}, DoRaw={self.DoRaw}, ProcSize={self.ProcSize}, ' \
+               f'RawMode={self.RawMode}, CameraControls={self.camera_controls}'
 
     def __repr__(self):
         return str(self)
@@ -226,6 +225,7 @@ def __init__(self, parent, config):
         self.min_AnalogueGain = None
         self.max_AnalogueGain = None
         self.camera_controls = dict()
+        self.needs_Restarts = False
         # exposure loop control
         self.ExposureTime = 0.0
         self.Sig_Do = threading.Event() # do an action
@@ -244,7 +244,7 @@ def __init__(self, parent, config):
     def closeCamera(self):
         """close camera
         """
-        logging.info('closing camera')
+        logger.info('closing camera')
         # stop exposure loop
         if self.ExposureThread is not None:
             if self.ExposureThread.is_alive():
@@ -283,11 +283,11 @@ def getRawCameraModes(self):
                 sensor_format = sensor_mode["unpacked"]
             # packed data formats are not supported
             if sensor_format.endswith("_CSI2P"):
-                logging.warning(f'raw mode not supported: {sensor_mode}')
+                logger.warning(f'raw mode not supported: {sensor_mode}')
                 continue
             # only Bayer pattern formats are supported
             if not re.match("S[RGB]{4}[0-9]+", sensor_format):
-                logging.warning(f'raw mode not supported: {sensor_mode}')
+                logger.warning(f'raw mode not supported: {sensor_mode}')
                 continue
             #
             size = sensor_mode["size"]
@@ -310,7 +310,7 @@ def getRawCameraModes(self):
                     elif size == (4056, 3040):
                         true_size = (4056, 3040)
                     else:
-                        logging.warning(f'Unsupported frame size {size} for imx477!')
+                        logger.warning(f'Unsupported frame size {size} for imx477!')
                 elif self.CamProps["Model"] == 'ov5647':
                     if size == (640, 480):
                         binning = (4, 4)
@@ -321,7 +321,7 @@ def getRawCameraModes(self):
                     elif size == (2592, 1944):
                         pass
                     else:
-                        logging.warning(f'Unsupported frame size {size} for ov5647!')
+                        logger.warning(f'Unsupported frame size {size} for ov5647!')
                 elif self.CamProps["Model"].startswith("imx708"):
                     if size == (1536, 864):
                         binning = (2, 2)
@@ -330,7 +330,7 @@ def getRawCameraModes(self):
                     elif size == (4608, 2592):
                         pass
                     else:
-                        logging.warning(f'Unsupported frame size {size} for imx708!')
+                        logger.warning(f'Unsupported frame size {size} for imx708!')
             # add to list of raw formats
             raw_mode = {
                 "label": f'{size[0]}x{size[1]} {sensor_format[1:5]} {sensor_mode["bit_depth"]}bit',
@@ -348,15 +348,14 @@ def getRawCameraModes(self):
     def openCamera(self, idx: int):
         """open camera with given index idx
         """
-        self.closeCamera()
-        logging.info("opening camera")
+        logger.info("opening camera")
         self.picam2 = Picamera2(idx)
         # read camera properties
         self.CamProps = self.picam2.camera_properties
-        logging.info(f'camera properties: {self.CamProps}')
+        logger.info(f'camera properties: {self.CamProps}')
         # force properties with values from config file
         if "UnitCellSize" not in self.CamProps:
-            logging.warning("Camera properties do not have UnitCellSize value. Need to force from config file!")
+            logger.warning("Camera properties do not have UnitCellSize value. Need to force from config file!")
         self.CamProps["UnitCellSize"] = (
             self.config.getint(
                 "driver", "force_UnitCellSize_X",
@@ -380,6 +379,18 @@ def openCamera(self, idx: int):
         # exposure time range
         self.min_ExposureTime, self.max_ExposureTime, default_exp = self.camera_controls["ExposureTime"]
         self.min_AnalogueGain, self.max_AnalogueGain, default_again = self.camera_controls["AnalogueGain"]
+        # TODO
+        force_Restart = self.config.get("driver", "force_Restart", fallback="auto").lower()
+        if force_Restart == "yes":
+            logger.info("INI setting forces camera restart")
+            self.needs_Restarts = True
+        elif force_Restart == "no":
+            logger.info("INI setting for camera restarts as needed")
+            self.needs_Restarts = False
+        else:
+            if force_Restart != "auto":
+                logger.warning(f'unknown INI value for camera restart: force_Restart={force_Restart}')
+            self.needs_Restarts = self.CamProps["Model"] in ["imx290", "imx519"]
         # start exposure loop
         self.Sig_ActionExit.clear()
         self.Sig_ActionExpose.clear()
@@ -481,7 +492,7 @@ def snooped_FitsHeader(self):
             "EQUINOX": (2000, "[yr] Equinox"),
             "DATE-END": (datetime.datetime.utcnow().isoformat(timespec="milliseconds"), "UTC time at end of observation"),
         })
-        logging.info("Finished collecting snooped data.")
+        logger.debug("Finished collecting snooped data.")
         ####
         return FitsHeader
 
@@ -673,15 +684,15 @@ def __ExposureLoop(self):
                     advertised_camera_controls=self.camera_controls,
                     has_RawModes=has_RawModes,
                 )
-            logging.info(f'exposure settings: {NewCameraSettings}')
+            logger.info(f'exposure settings: {NewCameraSettings}')
             # need a camera stop/start when something has changed on exposure controls
-            IsRestartNeeded = self.present_CameraSettings.is_RestartNeeded(NewCameraSettings)
+            IsRestartNeeded = self.present_CameraSettings.is_RestartNeeded(NewCameraSettings) or self.needs_Restarts
             if self.picam2.started and IsRestartNeeded:
-                logging.info(f'stopping camera for deeper reconfiguration')
+                logger.info(f'stopping camera for deeper reconfiguration')
                 self.picam2.stop_()
             # change of DoFastExposure needs a configuration change
-            if self.present_CameraSettings.is_ReconfigurationNeeded(NewCameraSettings):
-                logging.info(f'reconfiguring camera')
+            if self.present_CameraSettings.is_ReconfigurationNeeded(NewCameraSettings) or self.needs_Restarts:
+                logger.info(f'reconfiguring camera')
                 # need a new camera configuration
                 config = self.picam2.create_still_configuration(
                     queue=NewCameraSettings.DoFastExposure,
@@ -713,7 +724,7 @@ def __ExposureLoop(self):
             # start camera if not already running in Fast Exposure mode
             if not self.picam2.started:
                 self.picam2.start()
-                logging.info(f'camera started')
+                logger.debug(f'camera started')
             # camera runs now with new parameter
             self.present_CameraSettings = NewCameraSettings
             # last chance to exit or abort before doing exposure
@@ -759,7 +770,7 @@ def __ExposureLoop(self):
                 # get frame and its metadata
                 if not Abort:
                     (array, ), metadata = self.picam2.wait(job)
-                    logging.info("got exposed frame")
+                    logger.info("got exposed frame")
                     # at least HQ camera reports CCD temperature in meta data
                     self.parent.setVector("CCD_TEMPERATURE", "CCD_TEMPERATURE_VALUE",
                                           value=metadata.get('SensorTemperature', 0))
@@ -802,18 +813,18 @@ def __ExposureLoop(self):
                         # requested to save locally
                         local_filename = getLocalFileName(dir=upload_dir, prefix=upload_prefix, suffix=".fits")
                         bstream.seek(0)
-                        logging.info(f"saving image to file {local_filename}")
+                        logger.info(f"saving image to file {local_filename}")
                         with open(local_filename, 'wb') as fh:
                             fh.write(bstream.getbuffer())
                     if upload_mode[0] in ["UPLOAD_CLIENT", "UPLOAD_BOTH"]:
                         # send blob to client
                         bstream.seek(0)
                         # make BLOB
-                        logging.info(f"preparing frame as BLOB: {size} bytes")
+                        logger.info(f"preparing frame as BLOB: {size} bytes")
                         bv = self.parent.knownVectors["CCD1"]
                         compress = self.parent.knownVectors["CCD_COMPRESSION"]["CCD_COMPRESS"].value == ISwitchState.ON
                         bv["CCD1"].set_data(data=bstream.getbuffer(), format=".fits", compress=compress)
-                        logging.info(f"sending BLOB")
+                        logger.info(f"sending BLOB")
                         bv.send_setVector()
                     # tell client that we finished exposure
                     if DoFastExposure:

src/indi_pylibcamera/SnoopingManager.py

@@ -1,11 +1,11 @@
 """
 Snooping manager
 """
-import logging
 
 
 class SnoopingManager:
-    def __init__(self, parent, to_server_func):
+    def __init__(self, parent, to_server_func, logger):
+        self.logger = logger
         self.to_server = to_server_func
         self.parent = parent
         # snooped values: dict(device->dict(name->dict(elements)))
@@ -54,7 +54,7 @@ def catching(self, device: str, name: str, values: dict):
         if device in self.snoopedValues:
             if name in self.snoopedValues[device]:
                 self.snoopedValues[device][name] = values
-                logging.debug(f'snooped "{device}" - "{name}": {values}')
+                self.logger.debug(f'snooped "{device}" - "{name}": {values}')
                 if ("DO_SNOOPING" in self.parent.knownVectors) and ("SNOOP" in self.parent.knownVectors["DO_SNOOPING"].get_OnSwitches()):
                     if name in self.parent.knownVectors:
                         self.parent.knownVectors[name].set_byClient(values)

src/indi_pylibcamera/__init__.py

@@ -2,4 +2,4 @@
 INDI driver for libcamera supported cameras
 """
 
-__version__ = "2.3.0"
+__version__ = "2.4.0"

src/indi_pylibcamera/indi_pylibcamera.ini

@@ -26,6 +26,16 @@ LoggingLevel=Info
 # This feature requires the system time on your Raspberry Pi to be correct!
 DoSnooping=yes
 
+# Some cameras crash after the first exposure. Restarting the camera before every frame exposure can solve this issue.
+# Valid values are:
+#   no   - Do not restart if not needed to reconfigure camera.
+#   yes  - Always restart. Can lead to longer time between frames.
+#   auto - automatically choose based on list of critical cameras
+# Default if not otherwise set in INI file is "auto".
+#force_Restart=auto
+#force_Restart=no
+#force_Restart=yes
+
 #####################################
 # The following settings are to help debugging. Don't change them unasked!
 #