proper handling/restarting of devices if clock was missing and recove…

…red from this.

node/services/backends/cameras/picamera2backend.py

@@ -2,7 +2,7 @@
 import time
 from datetime import datetime
 from pathlib import Path
-from queue import Full, Queue
+from queue import Empty, Full, Queue
 from threading import Event, Thread
 
 from libcamera import Transform, controls
@@ -220,9 +220,16 @@ def _camera_fun(self):
                 logger.info(f"####### capture end, took {round((time.time() - tms), 2)}s #######")
             else:
                 job = self._picamera2.capture_request(wait=False)
-                # TODO: error checking, recovering from timeouts
-                capture_time_assigned_timestamp_ns = self._queue_timestamp_monotonic_ns.get(block=True, timeout=2.0)
-                request = self._picamera2.wait(job, timeout=2.0)
+
+                try:
+                    capture_time_assigned_timestamp_ns = self._queue_timestamp_monotonic_ns.get(block=True, timeout=2.0)
+                    request = self._picamera2.wait(job, timeout=2.0)
+                except (Empty, TimeoutError):  # no information in exc avail so omitted
+                    logger.warning("timeout while waiting for clock/camera")
+                    # continue so .is_running is checked. if supervisor detected already, var is false and effective aborted the thread.
+                    # if it is still true, maybe it was restarted already and we can try again?
+                    continue
+
                 picam_metadata = request.get_metadata()
                 request.release()
 

node/services/sync_acquisition_service.py

@@ -157,10 +157,12 @@ def _sync_fun(self):
         while self._device_is_running:
             try:
                 timestamp_ns = self._gpio_backend.wait_for_clock_rise_signal(timeout=1)
-                self._camera_backend.sync_tick(timestamp_ns)
             except TimeoutError:
                 # stop devices when no clock is avail, supervisor enables again after clock is received, derives new framerate ans starts backends
+                logger.error("no clock signal received within timeout! stopping devices.")
                 self._device_stop()
+            else:
+                self._camera_backend.sync_tick(timestamp_ns)
 
     def _capture_fun(self):
         while self._device_is_running: