try fix endless loop again
This commit is contained in:
@@ -103,6 +103,8 @@ class BridgeLinkStateWatcher:
|
|||||||
self._failing_since: dict[str, float] = {}
|
self._failing_since: dict[str, float] = {}
|
||||||
self._settle_deadlines: dict[str, float] = {}
|
self._settle_deadlines: dict[str, float] = {}
|
||||||
self._managed_event_deadlines: dict[str, float] = {}
|
self._managed_event_deadlines: dict[str, float] = {}
|
||||||
|
self._config_change_deadlines: dict[str, float] = {}
|
||||||
|
self._sync_attempt_deadlines: dict[tuple[str, str], tuple[str, float]] = {}
|
||||||
self._all_clear_since: Optional[float] = None
|
self._all_clear_since: Optional[float] = None
|
||||||
self._degraded = False
|
self._degraded = False
|
||||||
self._last_event_ts: Optional[float] = None
|
self._last_event_ts: Optional[float] = None
|
||||||
@@ -233,6 +235,16 @@ class BridgeLinkStateWatcher:
|
|||||||
for iface, deadline in self._managed_event_deadlines.items()
|
for iface, deadline in self._managed_event_deadlines.items()
|
||||||
if iface in states and deadline > now
|
if iface in states and deadline > now
|
||||||
}
|
}
|
||||||
|
self._config_change_deadlines = {
|
||||||
|
iface: deadline
|
||||||
|
for iface, deadline in self._config_change_deadlines.items()
|
||||||
|
if iface in states and deadline > now
|
||||||
|
}
|
||||||
|
self._sync_attempt_deadlines = {
|
||||||
|
key: value
|
||||||
|
for key, value in self._sync_attempt_deadlines.items()
|
||||||
|
if key[0] in states and key[1] in states and value[1] > now
|
||||||
|
}
|
||||||
|
|
||||||
if len(states) < 2:
|
if len(states) < 2:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -251,6 +263,13 @@ class BridgeLinkStateWatcher:
|
|||||||
for ifname, state in states.items()
|
for ifname, state in states.items()
|
||||||
if ifname not in suppressed_snapshot and ifname not in settling_snapshot and not state.link_ready
|
if ifname not in suppressed_snapshot and ifname not in settling_snapshot and not state.link_ready
|
||||||
)
|
)
|
||||||
|
changed_members = sorted(
|
||||||
|
ifname
|
||||||
|
for ifname in states
|
||||||
|
if self._config_changed(ifname, states, previous_states)
|
||||||
|
)
|
||||||
|
if changed_members:
|
||||||
|
self._mark_config_changes(changed_members, now)
|
||||||
|
|
||||||
if failing_members:
|
if failing_members:
|
||||||
config_source = self._find_config_sync_source(states, previous_states, preferred_ifname=source_ifname)
|
config_source = self._find_config_sync_source(states, previous_states, preferred_ifname=source_ifname)
|
||||||
@@ -260,6 +279,21 @@ class BridgeLinkStateWatcher:
|
|||||||
self._sync_member_configuration(config_source, states)
|
self._sync_member_configuration(config_source, states)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
transition_members = sorted(
|
||||||
|
{
|
||||||
|
*changed_members,
|
||||||
|
*( [source_ifname] if source_ifname is not None else [] ),
|
||||||
|
*failing_members,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if self._config_transition_active(transition_members, now):
|
||||||
|
with self._lock:
|
||||||
|
self._failing_since = {}
|
||||||
|
self._set_last_action(
|
||||||
|
f"waiting for config transition to settle on {transition_members} before suppressing"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
matured_failing_members = self._track_failing_members(failing_members, now)
|
matured_failing_members = self._track_failing_members(failing_members, now)
|
||||||
if not matured_failing_members:
|
if not matured_failing_members:
|
||||||
self._set_last_action(f"waiting before suppressing transient failures on {failing_members}")
|
self._set_last_action(f"waiting before suppressing transient failures on {failing_members}")
|
||||||
@@ -444,6 +478,21 @@ class BridgeLinkStateWatcher:
|
|||||||
|
|
||||||
return sorted(matured)
|
return sorted(matured)
|
||||||
|
|
||||||
|
def _mark_config_changes(self, ifnames: list[str], now: float) -> None:
|
||||||
|
"""Keep short-lived link flaps from config changes from being treated as failures."""
|
||||||
|
holdoff = max(
|
||||||
|
settings.bridge_link_state_failure_holdoff_seconds,
|
||||||
|
self.recovery_holdoff_seconds,
|
||||||
|
)
|
||||||
|
with self._lock:
|
||||||
|
for ifname in ifnames:
|
||||||
|
self._config_change_deadlines[ifname] = now + holdoff
|
||||||
|
|
||||||
|
def _config_transition_active(self, ifnames: list[str], now: float) -> bool:
|
||||||
|
"""Return whether any listed member is still inside the config-change grace window."""
|
||||||
|
with self._lock:
|
||||||
|
return any(self._config_change_deadlines.get(ifname, 0.0) > now for ifname in ifnames)
|
||||||
|
|
||||||
def _next_wait_timeout(self) -> Optional[float]:
|
def _next_wait_timeout(self) -> Optional[float]:
|
||||||
"""Return how long the watcher may sleep before the next restore deadline."""
|
"""Return how long the watcher may sleep before the next restore deadline."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -659,6 +708,8 @@ class BridgeLinkStateWatcher:
|
|||||||
target_profile = target.ethernet_profile
|
target_profile = target.ethernet_profile
|
||||||
if source_profile is None or target_profile is None or source_profile == target_profile:
|
if source_profile is None or target_profile is None or source_profile == target_profile:
|
||||||
return None
|
return None
|
||||||
|
if not self._should_attempt_sync(source_ifname, target_ifname, source_profile):
|
||||||
|
return None
|
||||||
|
|
||||||
change_summary: Optional[str] = None
|
change_summary: Optional[str] = None
|
||||||
commands: list[tuple[list[str], str]] = []
|
commands: list[tuple[list[str], str]] = []
|
||||||
@@ -738,11 +789,13 @@ class BridgeLinkStateWatcher:
|
|||||||
|
|
||||||
if change_summary is None:
|
if change_summary is None:
|
||||||
if last_error is not None:
|
if last_error is not None:
|
||||||
|
self._record_sync_attempt(source_ifname, target_ifname, source_profile)
|
||||||
self._set_last_action(
|
self._set_last_action(
|
||||||
f"failed to sync link profile from {source_ifname} to {target_ifname}: {last_error}"
|
f"failed to sync link profile from {source_ifname} to {target_ifname}: {last_error}"
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
self._record_sync_attempt(source_ifname, target_ifname, source_profile)
|
||||||
self._mark_managed_change(target_ifname)
|
self._mark_managed_change(target_ifname)
|
||||||
logger.info(
|
logger.info(
|
||||||
"Synchronized ethtool profile from %s to %s on bridge=%s: %s",
|
"Synchronized ethtool profile from %s to %s on bridge=%s: %s",
|
||||||
@@ -753,6 +806,41 @@ class BridgeLinkStateWatcher:
|
|||||||
)
|
)
|
||||||
return change_summary
|
return change_summary
|
||||||
|
|
||||||
|
def _sync_attempt_signature(self, source_profile: EthernetProfile) -> str:
|
||||||
|
"""Serialize the desired mirrored link state for retry deduplication."""
|
||||||
|
return f"{source_profile.speed_mbps}:{source_profile.duplex}:{source_profile.autoneg}"
|
||||||
|
|
||||||
|
def _should_attempt_sync(
|
||||||
|
self,
|
||||||
|
source_ifname: str,
|
||||||
|
target_ifname: str,
|
||||||
|
source_profile: EthernetProfile,
|
||||||
|
) -> bool:
|
||||||
|
"""Avoid replaying the same sync on every degraded-state recheck."""
|
||||||
|
signature = self._sync_attempt_signature(source_profile)
|
||||||
|
now = time.time()
|
||||||
|
with self._lock:
|
||||||
|
cached = self._sync_attempt_deadlines.get((source_ifname, target_ifname))
|
||||||
|
if cached is None:
|
||||||
|
return True
|
||||||
|
cached_signature, deadline = cached
|
||||||
|
return cached_signature != signature or deadline <= now
|
||||||
|
|
||||||
|
def _record_sync_attempt(
|
||||||
|
self,
|
||||||
|
source_ifname: str,
|
||||||
|
target_ifname: str,
|
||||||
|
source_profile: EthernetProfile,
|
||||||
|
) -> None:
|
||||||
|
"""Rate-limit repeated sync attempts for the same desired link profile."""
|
||||||
|
signature = self._sync_attempt_signature(source_profile)
|
||||||
|
cooldown = max(
|
||||||
|
settings.bridge_link_state_degraded_recheck_seconds * 4,
|
||||||
|
self.recovery_holdoff_seconds,
|
||||||
|
)
|
||||||
|
with self._lock:
|
||||||
|
self._sync_attempt_deadlines[(source_ifname, target_ifname)] = (signature, time.time() + cooldown)
|
||||||
|
|
||||||
def _suppress_other_members(self, states: dict[str, MemberLinkState], failing_members: list[str]) -> None:
|
def _suppress_other_members(self, states: dict[str, MemberLinkState], failing_members: list[str]) -> None:
|
||||||
desired_suppressed = set(states) - set(failing_members)
|
desired_suppressed = set(states) - set(failing_members)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user