diff --git a/backend/src/config.py b/backend/src/config.py index cbe10ed..4ada9a1 100644 --- a/backend/src/config.py +++ b/backend/src/config.py @@ -54,6 +54,7 @@ class BackendSettings: bridge_bpf_build_dir: str bridge_link_state_poll_interval_seconds: float bridge_link_state_thread_join_timeout_seconds: float + bridge_link_state_recovery_holdoff_seconds: float telemetry_process_stop_timeout_seconds: float telemetry_reader_join_timeout_seconds: float tshark_enabled: bool @@ -96,6 +97,10 @@ def load_settings() -> BackendSettings: "BACKEND_BRIDGE_LINK_STATE_THREAD_JOIN_TIMEOUT_SECONDS", 2.0, ), + bridge_link_state_recovery_holdoff_seconds=_env_float( + "BACKEND_BRIDGE_LINK_STATE_RECOVERY_HOLDOFF_SECONDS", + 1.0, + ), telemetry_process_stop_timeout_seconds=_env_float("BACKEND_TELEMETRY_PROCESS_STOP_TIMEOUT_SECONDS", 3.0), telemetry_reader_join_timeout_seconds=_env_float("BACKEND_TELEMETRY_READER_JOIN_TIMEOUT_SECONDS", 2.0), tshark_enabled=_env_bool("BACKEND_TSHARK_ENABLED", True), diff --git a/backend/src/utilities/bridge_link_state_manager.py b/backend/src/utilities/bridge_link_state_manager.py index 9e80db5..437bdc3 100644 --- a/backend/src/utilities/bridge_link_state_manager.py +++ b/backend/src/utilities/bridge_link_state_manager.py @@ -69,6 +69,8 @@ class BridgeLinkStateWatcher: ) self._running = False self._suppressed_members: dict[str, bool] = {} + self._settle_deadlines: dict[str, float] = {} + self._all_clear_since: Optional[float] = None self._last_poll_ts: Optional[float] = None self._last_error: Optional[str] = None self._last_action: Optional[str] = None @@ -111,6 +113,7 @@ class BridgeLinkStateWatcher: "last_error": self._last_error, "last_action": self._last_action, "suppressed_members": sorted(self._suppressed_members), + "recovery_holdoff_seconds": settings.bridge_link_state_recovery_holdoff_seconds, "members": members, } @@ -139,22 +142,55 @@ class BridgeLinkStateWatcher: for iface, restore_up in self._suppressed_members.items() if iface in states } + self._settle_deadlines = { + iface: deadline + for iface, deadline in self._settle_deadlines.items() + if iface in states and deadline > now + } if len(states) < 2: + self._all_clear_since = None self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members") return - suppressed_snapshot = set(self._suppressed_members) + with self._lock: + suppressed_snapshot = set(self._suppressed_members) + settling_snapshot = { + iface for iface, deadline in self._settle_deadlines.items() if deadline > now + } + failing_members = sorted( ifname for ifname, state in states.items() - if ifname not in suppressed_snapshot and not state.link_ready + if ifname not in suppressed_snapshot and ifname not in settling_snapshot and not state.link_ready ) if failing_members: + with self._lock: + self._all_clear_since = None self._suppress_other_members(states, failing_members) return + if suppressed_snapshot: + should_restore = False + with self._lock: + if self._all_clear_since is None: + self._all_clear_since = now + should_restore = now - self._all_clear_since >= settings.bridge_link_state_recovery_holdoff_seconds + if not should_restore: + remaining = max( + 0.0, + settings.bridge_link_state_recovery_holdoff_seconds - (now - self._all_clear_since), + ) + self._last_action = f"waiting {remaining:.2f}s before restoring suppressed members" + + if should_restore: + self._restore_suppressed_members(reason="all_members_recovered") + return + + with self._lock: + self._all_clear_since = None + self._restore_suppressed_members(reason="all_members_recovered") def _read_member_state(self, ifname: str) -> MemberLinkState: @@ -209,6 +245,7 @@ class BridgeLinkStateWatcher: with self._lock: self._suppressed_members = {} + self._all_clear_since = None self._last_action = f"restored {restored_members} ({reason})" def _set_interface_admin_state(self, ifname: str, target_up: bool) -> None: @@ -222,6 +259,10 @@ class BridgeLinkStateWatcher: raise RuntimeError(f"Interface {ifname} not found while propagating link state") ipr.link("set", index=indices[0], state=state_name) + if target_up: + with self._lock: + self._settle_deadlines[ifname] = time.time() + settings.bridge_link_state_recovery_holdoff_seconds + class BridgeLinkStateManager: """Track bridge link-state watchers keyed by bridge name."""