diff --git a/backend/src/config.py b/backend/src/config.py index cceb770..0122ed0 100644 --- a/backend/src/config.py +++ b/backend/src/config.py @@ -53,6 +53,7 @@ class BackendSettings: sniffer_thread_join_timeout_seconds: float bridge_bpf_build_dir: str bridge_link_state_thread_join_timeout_seconds: float + bridge_link_state_failure_holdoff_seconds: float bridge_link_state_recovery_holdoff_seconds: float bridge_link_state_degraded_recheck_seconds: float telemetry_process_stop_timeout_seconds: float @@ -93,6 +94,10 @@ def load_settings() -> BackendSettings: "BACKEND_BRIDGE_LINK_STATE_THREAD_JOIN_TIMEOUT_SECONDS", 2.0, ), + bridge_link_state_failure_holdoff_seconds=_env_float( + "BACKEND_BRIDGE_LINK_STATE_FAILURE_HOLDOFF_SECONDS", + 0.75, + ), bridge_link_state_recovery_holdoff_seconds=_env_float( "BACKEND_BRIDGE_LINK_STATE_RECOVERY_HOLDOFF_SECONDS", 1.0, diff --git a/backend/src/utilities/bridge_link_state_manager.py b/backend/src/utilities/bridge_link_state_manager.py index e48dd08..bf894e6 100644 --- a/backend/src/utilities/bridge_link_state_manager.py +++ b/backend/src/utilities/bridge_link_state_manager.py @@ -99,6 +99,7 @@ class BridgeLinkStateWatcher: ) self._running = False self._suppressed_members: dict[str, bool] = {} + self._failing_since: dict[str, float] = {} self._settle_deadlines: dict[str, float] = {} self._managed_event_deadlines: dict[str, float] = {} self._all_clear_since: Optional[float] = None @@ -152,6 +153,7 @@ class BridgeLinkStateWatcher: "last_error": self._last_error, "last_action": self._last_action, "suppressed_members": sorted(self._suppressed_members), + "failure_holdoff_seconds": settings.bridge_link_state_failure_holdoff_seconds, "recovery_holdoff_seconds": self.recovery_holdoff_seconds, "degraded_recheck_seconds": settings.bridge_link_state_degraded_recheck_seconds, "members": members, @@ -209,6 +211,11 @@ class BridgeLinkStateWatcher: for iface, restore_up in self._suppressed_members.items() if iface in states } + self._failing_since = { + iface: first_seen + for iface, first_seen in self._failing_since.items() + if iface in states + } self._settle_deadlines = { iface: deadline for iface, deadline in self._settle_deadlines.items() @@ -223,6 +230,7 @@ class BridgeLinkStateWatcher: if len(states) < 2: with self._lock: self._degraded = False + self._failing_since = {} self._all_clear_since = None self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members") return @@ -238,13 +246,22 @@ class BridgeLinkStateWatcher: ) if failing_members: + matured_failing_members = self._track_failing_members(failing_members, now) + if not matured_failing_members: + self._set_last_action( + f"waiting {settings.bridge_link_state_failure_holdoff_seconds:.2f}s before suppressing due to failures={failing_members}" + ) + return + with self._lock: self._degraded = True self._all_clear_since = None - self._suppress_other_members(states, failing_members) + self._suppress_other_members(states, matured_failing_members) return if suppressed_snapshot: + with self._lock: + self._failing_since = {} should_restore = False with self._lock: self._degraded = True @@ -261,6 +278,7 @@ class BridgeLinkStateWatcher: with self._lock: self._degraded = False + self._failing_since = {} self._all_clear_since = None if source_ifname in states and self._is_source_eligible(source_ifname, states): @@ -293,12 +311,35 @@ class BridgeLinkStateWatcher: state = states.get(ifname) return state is not None and state.link_ready + def _track_failing_members(self, failing_members: list[str], now: float) -> list[str]: + """Record first-seen timestamps and return failures that exceeded the holdoff.""" + with self._lock: + self._degraded = True + tracked = { + ifname: self._failing_since.get(ifname, now) + for ifname in failing_members + } + self._failing_since = tracked + matured = [ + ifname + for ifname, first_seen in tracked.items() + if now - first_seen >= settings.bridge_link_state_failure_holdoff_seconds + ] + + return sorted(matured) + def _next_wait_timeout(self) -> Optional[float]: """Return how long the watcher may sleep before the next restore deadline.""" with self._lock: if self._suppressed_members and self._all_clear_since is not None: deadline = self._all_clear_since + self.recovery_holdoff_seconds return max(0.0, min(deadline - time.time(), settings.bridge_link_state_degraded_recheck_seconds)) + if self._failing_since: + earliest_deadline = min( + first_seen + settings.bridge_link_state_failure_holdoff_seconds + for first_seen in self._failing_since.values() + ) + return max(0.0, min(earliest_deadline - time.time(), settings.bridge_link_state_degraded_recheck_seconds)) if self._degraded: return settings.bridge_link_state_degraded_recheck_seconds return None