add backoff to heartbeats and detect unresponsive nodes

README.md

@@ -32,6 +32,10 @@ ferroflow-vcan down vcan0
 
 ## Development
 
+### Runtime Configuration
+
+Ferroflow reads `config.yml` on startup. An example can be found at `config.example.yml`.
+
 ### Running CI Checks
 
 The repository includes a CI script (`ci-rust.sh`) that runs all quality checks on the Rust implementation. This script is used both locally and in GitHub Actions

config.example.yml

@@ -0,0 +1,11 @@
+can_bus_interfaces:
+  - can0
+
+heartbeat:
+  period: 1
+  backoff_multiplier: 2
+  max_period: 60
+  max_unanswered: 5
+
+# Leave empty to disable database logging.
+database_url: ""

src/config/mod.rs

@@ -4,10 +4,25 @@ use anyhow::{Context, Result};
 use config as config_builder;
 use serde::{Deserialize, Serialize};
 
+const DEFAULT_HEARTBEAT_BACKOFF_MULTIPLIER: u32 = 2;
+const DEFAULT_HEARTBEAT_MAX_PERIOD: u32 = 60;
+const DEFAULT_HEARTBEAT_MAX_UNANSWERED: u32 = 5;
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct HeartbeatConfig {
+    pub period: u32,
+    #[serde(default = "default_heartbeat_backoff_multiplier")]
+    pub backoff_multiplier: u32,
+    #[serde(default = "default_heartbeat_max_period")]
+    pub max_period: u32,
+    #[serde(default = "default_heartbeat_max_unanswered")]
+    pub max_unanswered: u32,
+}
+
 #[derive(Deserialize, Serialize, Debug)]
 pub struct Config {
     pub can_bus_interfaces: Vec<String>,
-    pub heartbeat_period: u64,
+    pub heartbeat: HeartbeatConfig,
     pub database_url: String,
 }
 
@@ -20,3 +35,15 @@ pub fn load_config(path: &str) -> Result<Config> {
         .try_deserialize()
         .with_context(|| format!("Failed to deserialize config from {}", path))
 }
+
+fn default_heartbeat_backoff_multiplier() -> u32 {
+    DEFAULT_HEARTBEAT_BACKOFF_MULTIPLIER
+}
+
+fn default_heartbeat_max_period() -> u32 {
+    DEFAULT_HEARTBEAT_MAX_PERIOD
+}
+
+fn default_heartbeat_max_unanswered() -> u32 {
+    DEFAULT_HEARTBEAT_MAX_UNANSWERED
+}

src/lib.rs

@@ -38,12 +38,7 @@ pub fn run_with_dependencies(
         println!("Starting node registration");
 
         nodes::spawn_can_msg_handler_thread(node_manager, event_dispatcher, scope);
-        nodes::spawn_heartbeat_thread(
-            node_manager,
-            std::time::Duration::from_secs(config.heartbeat_period),
-            event_dispatcher,
-            scope,
-        );
+        nodes::spawn_heartbeat_thread(node_manager, &config.heartbeat, event_dispatcher, scope);
 
         node_manager.start_node_registration();