fix debounce link config changes

backend/src/config.py

@@ -53,6 +53,7 @@ class BackendSettings:
     sniffer_thread_join_timeout_seconds: float
     bridge_bpf_build_dir: str
     bridge_link_state_thread_join_timeout_seconds: float
+    bridge_link_state_failure_holdoff_seconds: float
     bridge_link_state_recovery_holdoff_seconds: float
     bridge_link_state_degraded_recheck_seconds: float
     telemetry_process_stop_timeout_seconds: float
@@ -93,6 +94,10 @@ def load_settings() -> BackendSettings:
             "BACKEND_BRIDGE_LINK_STATE_THREAD_JOIN_TIMEOUT_SECONDS",
             2.0,
         ),
+        bridge_link_state_failure_holdoff_seconds=_env_float(
+            "BACKEND_BRIDGE_LINK_STATE_FAILURE_HOLDOFF_SECONDS",
+            0.75,
+        ),
         bridge_link_state_recovery_holdoff_seconds=_env_float(
             "BACKEND_BRIDGE_LINK_STATE_RECOVERY_HOLDOFF_SECONDS",
             1.0,

backend/src/utilities/bridge_link_state_manager.py

@@ -99,6 +99,7 @@ class BridgeLinkStateWatcher:
         )
         self._running = False
         self._suppressed_members: dict[str, bool] = {}
+        self._failing_since: dict[str, float] = {}
         self._settle_deadlines: dict[str, float] = {}
         self._managed_event_deadlines: dict[str, float] = {}
         self._all_clear_since: Optional[float] = None
@@ -152,6 +153,7 @@ class BridgeLinkStateWatcher:
                 "last_error": self._last_error,
                 "last_action": self._last_action,
                 "suppressed_members": sorted(self._suppressed_members),
+                "failure_holdoff_seconds": settings.bridge_link_state_failure_holdoff_seconds,
                 "recovery_holdoff_seconds": self.recovery_holdoff_seconds,
                 "degraded_recheck_seconds": settings.bridge_link_state_degraded_recheck_seconds,
                 "members": members,
@@ -209,6 +211,11 @@ class BridgeLinkStateWatcher:
                 for iface, restore_up in self._suppressed_members.items()
                 if iface in states
             }
+            self._failing_since = {
+                iface: first_seen
+                for iface, first_seen in self._failing_since.items()
+                if iface in states
+            }
             self._settle_deadlines = {
                 iface: deadline
                 for iface, deadline in self._settle_deadlines.items()
@@ -223,6 +230,7 @@ class BridgeLinkStateWatcher:
         if len(states) < 2:
             with self._lock:
                 self._degraded = False
+                self._failing_since = {}
                 self._all_clear_since = None
             self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members")
             return
@@ -238,13 +246,22 @@ class BridgeLinkStateWatcher:
         )
 
         if failing_members:
+            matured_failing_members = self._track_failing_members(failing_members, now)
+            if not matured_failing_members:
+                self._set_last_action(
+                    f"waiting {settings.bridge_link_state_failure_holdoff_seconds:.2f}s before suppressing due to failures={failing_members}"
+                )
+                return
+
             with self._lock:
                 self._degraded = True
                 self._all_clear_since = None
-            self._suppress_other_members(states, failing_members)
+            self._suppress_other_members(states, matured_failing_members)
             return
 
         if suppressed_snapshot:
+            with self._lock:
+                self._failing_since = {}
             should_restore = False
             with self._lock:
                 self._degraded = True
@@ -261,6 +278,7 @@ class BridgeLinkStateWatcher:
 
         with self._lock:
             self._degraded = False
+            self._failing_since = {}
             self._all_clear_since = None
 
         if source_ifname in states and self._is_source_eligible(source_ifname, states):
@@ -293,12 +311,35 @@ class BridgeLinkStateWatcher:
         state = states.get(ifname)
         return state is not None and state.link_ready
 
+    def _track_failing_members(self, failing_members: list[str], now: float) -> list[str]:
+        """Record first-seen timestamps and return failures that exceeded the holdoff."""
+        with self._lock:
+            self._degraded = True
+            tracked = {
+                ifname: self._failing_since.get(ifname, now)
+                for ifname in failing_members
+            }
+            self._failing_since = tracked
+            matured = [
+                ifname
+                for ifname, first_seen in tracked.items()
+                if now - first_seen >= settings.bridge_link_state_failure_holdoff_seconds
+            ]
+
+        return sorted(matured)
+
     def _next_wait_timeout(self) -> Optional[float]:
         """Return how long the watcher may sleep before the next restore deadline."""
         with self._lock:
             if self._suppressed_members and self._all_clear_since is not None:
                 deadline = self._all_clear_since + self.recovery_holdoff_seconds
                 return max(0.0, min(deadline - time.time(), settings.bridge_link_state_degraded_recheck_seconds))
+            if self._failing_since:
+                earliest_deadline = min(
+                    first_seen + settings.bridge_link_state_failure_holdoff_seconds
+                    for first_seen in self._failing_since.values()
+                )
+                return max(0.0, min(earliest_deadline - time.time(), settings.bridge_link_state_degraded_recheck_seconds))
             if self._degraded:
                 return settings.bridge_link_state_degraded_recheck_seconds
         return None