serialqueue: Eventually time out if unable to write CANbus messages

Klipper logs an error on a failed CANbus write.  Unfortunately, if the
bus becomes permanently disabled (eg, due to a user removing power to
devices on the CANbus) then it can result in the logs filling with
error messages.

Permanently disable the low-level processing of messages if CANbus
writes continually fail for at least 10 seconds.  This avoids filling
the log with redundant messages.

Signed-off-by: Kevin O'Connor <[email protected]>

klippy/chelper/serialqueue.c

@@ -62,6 +62,7 @@ struct serialqueue {
     int ready_bytes, upcoming_bytes, need_ack_bytes, last_ack_bytes;
     uint64_t need_kick_clock;
     struct list_head notify_queue;
+    double last_write_fail_time;
     // Received messages
     struct list_head receive_queue;
     // Fastreader support
@@ -376,8 +377,16 @@ do_write(struct serialqueue *sq, void *buf, int buflen)
         int ret = write(sq->serial_fd, &cf, sizeof(cf));
         if (ret < 0) {
             report_errno("can write", ret);
+            double curtime = get_monotonic();
+            if (!sq->last_write_fail_time) {
+                sq->last_write_fail_time = curtime;
+            } else if (curtime > sq->last_write_fail_time + 10.0) {
+                errorf("Halting reads due to CAN write errors.");
+                pollreactor_do_exit(sq->pr);
+            }
             return;
         }
+        sq->last_write_fail_time = 0.0;
         buf += size;
         buflen -= size;
     }