diff --git a/backend/src/utilities/bridge_link_state_manager.py b/backend/src/utilities/bridge_link_state_manager.py index d4bca5b..89dfc66 100644 --- a/backend/src/utilities/bridge_link_state_manager.py +++ b/backend/src/utilities/bridge_link_state_manager.py @@ -103,6 +103,8 @@ class BridgeLinkStateWatcher: self._failing_since: dict[str, float] = {} self._settle_deadlines: dict[str, float] = {} self._managed_event_deadlines: dict[str, float] = {} + self._config_change_deadlines: dict[str, float] = {} + self._sync_attempt_deadlines: dict[tuple[str, str], tuple[str, float]] = {} self._all_clear_since: Optional[float] = None self._degraded = False self._last_event_ts: Optional[float] = None @@ -233,6 +235,16 @@ class BridgeLinkStateWatcher: for iface, deadline in self._managed_event_deadlines.items() if iface in states and deadline > now } + self._config_change_deadlines = { + iface: deadline + for iface, deadline in self._config_change_deadlines.items() + if iface in states and deadline > now + } + self._sync_attempt_deadlines = { + key: value + for key, value in self._sync_attempt_deadlines.items() + if key[0] in states and key[1] in states and value[1] > now + } if len(states) < 2: with self._lock: @@ -251,6 +263,13 @@ class BridgeLinkStateWatcher: for ifname, state in states.items() if ifname not in suppressed_snapshot and ifname not in settling_snapshot and not state.link_ready ) + changed_members = sorted( + ifname + for ifname in states + if self._config_changed(ifname, states, previous_states) + ) + if changed_members: + self._mark_config_changes(changed_members, now) if failing_members: config_source = self._find_config_sync_source(states, previous_states, preferred_ifname=source_ifname) @@ -260,6 +279,21 @@ class BridgeLinkStateWatcher: self._sync_member_configuration(config_source, states) return + transition_members = sorted( + { + *changed_members, + *( [source_ifname] if source_ifname is not None else [] ), + *failing_members, + } + ) + if self._config_transition_active(transition_members, now): + with self._lock: + self._failing_since = {} + self._set_last_action( + f"waiting for config transition to settle on {transition_members} before suppressing" + ) + return + matured_failing_members = self._track_failing_members(failing_members, now) if not matured_failing_members: self._set_last_action(f"waiting before suppressing transient failures on {failing_members}") @@ -444,6 +478,21 @@ class BridgeLinkStateWatcher: return sorted(matured) + def _mark_config_changes(self, ifnames: list[str], now: float) -> None: + """Keep short-lived link flaps from config changes from being treated as failures.""" + holdoff = max( + settings.bridge_link_state_failure_holdoff_seconds, + self.recovery_holdoff_seconds, + ) + with self._lock: + for ifname in ifnames: + self._config_change_deadlines[ifname] = now + holdoff + + def _config_transition_active(self, ifnames: list[str], now: float) -> bool: + """Return whether any listed member is still inside the config-change grace window.""" + with self._lock: + return any(self._config_change_deadlines.get(ifname, 0.0) > now for ifname in ifnames) + def _next_wait_timeout(self) -> Optional[float]: """Return how long the watcher may sleep before the next restore deadline.""" with self._lock: @@ -659,6 +708,8 @@ class BridgeLinkStateWatcher: target_profile = target.ethernet_profile if source_profile is None or target_profile is None or source_profile == target_profile: return None + if not self._should_attempt_sync(source_ifname, target_ifname, source_profile): + return None change_summary: Optional[str] = None commands: list[tuple[list[str], str]] = [] @@ -738,11 +789,13 @@ class BridgeLinkStateWatcher: if change_summary is None: if last_error is not None: + self._record_sync_attempt(source_ifname, target_ifname, source_profile) self._set_last_action( f"failed to sync link profile from {source_ifname} to {target_ifname}: {last_error}" ) return None + self._record_sync_attempt(source_ifname, target_ifname, source_profile) self._mark_managed_change(target_ifname) logger.info( "Synchronized ethtool profile from %s to %s on bridge=%s: %s", @@ -753,6 +806,41 @@ class BridgeLinkStateWatcher: ) return change_summary + def _sync_attempt_signature(self, source_profile: EthernetProfile) -> str: + """Serialize the desired mirrored link state for retry deduplication.""" + return f"{source_profile.speed_mbps}:{source_profile.duplex}:{source_profile.autoneg}" + + def _should_attempt_sync( + self, + source_ifname: str, + target_ifname: str, + source_profile: EthernetProfile, + ) -> bool: + """Avoid replaying the same sync on every degraded-state recheck.""" + signature = self._sync_attempt_signature(source_profile) + now = time.time() + with self._lock: + cached = self._sync_attempt_deadlines.get((source_ifname, target_ifname)) + if cached is None: + return True + cached_signature, deadline = cached + return cached_signature != signature or deadline <= now + + def _record_sync_attempt( + self, + source_ifname: str, + target_ifname: str, + source_profile: EthernetProfile, + ) -> None: + """Rate-limit repeated sync attempts for the same desired link profile.""" + signature = self._sync_attempt_signature(source_profile) + cooldown = max( + settings.bridge_link_state_degraded_recheck_seconds * 4, + self.recovery_holdoff_seconds, + ) + with self._lock: + self._sync_attempt_deadlines[(source_ifname, target_ifname)] = (signature, time.time() + cooldown) + def _suppress_other_members(self, states: dict[str, MemberLinkState], failing_members: list[str]) -> None: desired_suppressed = set(states) - set(failing_members)