"""Background watcher that propagates bridge member link failures to sibling ports.""" from __future__ import annotations import logging import threading import time from dataclasses import dataclass from typing import Any, Dict, Optional from pyroute2 import IPRoute from src.config import settings from src.utilities.interface_bridge_helpers import ( check_interface_exists, get_bridge_ports_once, read_interface_admin_up, read_interface_carrier, read_interface_operstate, ) logger = logging.getLogger("bridge_link_state_manager") @dataclass class MemberLinkState: """Current sysfs-derived state for one bridge member.""" ifname: str admin_up: Optional[bool] carrier_up: Optional[bool] operstate: Optional[str] @property def link_ready(self) -> bool: """Return whether the member currently looks healthy enough to forward.""" if self.admin_up is not True: return False if self.carrier_up is False: return False if self.operstate in {"down", "lowerlayerdown", "notpresent"}: return False return True def to_dict(self, suppressed: bool = False) -> Dict[str, Any]: """Serialize member state for API responses.""" return { "ifname": self.ifname, "admin_up": self.admin_up, "carrier_up": self.carrier_up, "operstate": self.operstate, "link_ready": self.link_ready, "suppressed": suppressed, } class BridgeLinkStateWatcher: """Watch one bridge and mirror member failures to the other bridge members.""" def __init__(self, bridge_name: str, poll_interval_seconds: float) -> None: self.bridge_name = bridge_name self.poll_interval_seconds = poll_interval_seconds self._stop_event = threading.Event() self._lock = threading.Lock() self._thread = threading.Thread( target=self._run, daemon=True, name=f"bridge-link-state-{bridge_name}", ) self._running = False self._suppressed_members: dict[str, bool] = {} self._settle_deadlines: dict[str, float] = {} self._all_clear_since: Optional[float] = None self._last_poll_ts: Optional[float] = None self._last_error: Optional[str] = None self._last_action: Optional[str] = None self._member_states: dict[str, MemberLinkState] = {} def start(self) -> None: """Start the watcher thread.""" with self._lock: if self._running: return self._running = True self._thread.start() def stop(self) -> None: """Stop the watcher and restore interfaces that were suppressed by it.""" self._stop_event.set() if self._thread.is_alive(): self._thread.join(timeout=settings.bridge_link_state_thread_join_timeout_seconds) try: self._restore_suppressed_members(reason="watcher_stopped") except Exception: logger.exception("Failed to restore suppressed members for bridge=%s", self.bridge_name) with self._lock: self._running = False def status(self) -> dict[str, Any]: """Return a JSON-serializable snapshot of the watcher state.""" with self._lock: members = { ifname: state.to_dict(suppressed=ifname in self._suppressed_members) for ifname, state in sorted(self._member_states.items()) } return { "bridge": self.bridge_name, "active": self._running and self._thread.is_alive() and not self._stop_event.is_set(), "poll_interval_seconds": self.poll_interval_seconds, "last_poll_ts": self._last_poll_ts, "last_error": self._last_error, "last_action": self._last_action, "suppressed_members": sorted(self._suppressed_members), "recovery_holdoff_seconds": settings.bridge_link_state_recovery_holdoff_seconds, "members": members, } def _run(self) -> None: while not self._stop_event.is_set(): try: self._poll_once() with self._lock: self._last_error = None except Exception as exc: logger.exception("Bridge link-state poll failed for bridge=%s", self.bridge_name) with self._lock: self._last_error = str(exc) self._stop_event.wait(self.poll_interval_seconds) def _poll_once(self) -> None: members = [iface for iface in get_bridge_ports_once(self.bridge_name) if check_interface_exists(iface)] states = {iface: self._read_member_state(iface) for iface in members} now = time.time() with self._lock: self._last_poll_ts = now self._member_states = states self._suppressed_members = { iface: restore_up for iface, restore_up in self._suppressed_members.items() if iface in states } self._settle_deadlines = { iface: deadline for iface, deadline in self._settle_deadlines.items() if iface in states and deadline > now } if len(states) < 2: self._all_clear_since = None self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members") return with self._lock: suppressed_snapshot = set(self._suppressed_members) settling_snapshot = { iface for iface, deadline in self._settle_deadlines.items() if deadline > now } failing_members = sorted( ifname for ifname, state in states.items() if ifname not in suppressed_snapshot and ifname not in settling_snapshot and not state.link_ready ) if failing_members: with self._lock: self._all_clear_since = None self._suppress_other_members(states, failing_members) return if suppressed_snapshot: should_restore = False with self._lock: if self._all_clear_since is None: self._all_clear_since = now should_restore = now - self._all_clear_since >= settings.bridge_link_state_recovery_holdoff_seconds if not should_restore: remaining = max( 0.0, settings.bridge_link_state_recovery_holdoff_seconds - (now - self._all_clear_since), ) self._last_action = f"waiting {remaining:.2f}s before restoring suppressed members" if should_restore: self._restore_suppressed_members(reason="all_members_recovered") return with self._lock: self._all_clear_since = None self._restore_suppressed_members(reason="all_members_recovered") def _read_member_state(self, ifname: str) -> MemberLinkState: return MemberLinkState( ifname=ifname, admin_up=read_interface_admin_up(ifname), carrier_up=read_interface_carrier(ifname), operstate=read_interface_operstate(ifname), ) def _suppress_other_members(self, states: dict[str, MemberLinkState], failing_members: list[str]) -> None: desired_suppressed = set(states) - set(failing_members) with self._lock: current_suppressed = dict(self._suppressed_members) next_suppressed: dict[str, bool] = {} changed_members: list[str] = [] for ifname in sorted(desired_suppressed): restore_up = current_suppressed.get(ifname, states[ifname].admin_up is True) if ifname not in current_suppressed and states[ifname].admin_up is True: self._set_interface_admin_state(ifname, target_up=False) changed_members.append(ifname) next_suppressed[ifname] = restore_up with self._lock: self._suppressed_members = next_suppressed action = ( f"suppressed {changed_members} because failing members={failing_members}" if changed_members else f"holding suppressed members because failing members={failing_members}" ) self._last_action = action def _restore_suppressed_members(self, reason: str) -> None: with self._lock: suppressed = dict(self._suppressed_members) if not suppressed: with self._lock: self._last_action = f"no_restore_needed ({reason})" return restored_members: list[str] = [] for ifname, restore_up in sorted(suppressed.items()): if not check_interface_exists(ifname): continue if restore_up: self._set_interface_admin_state(ifname, target_up=True) restored_members.append(ifname) with self._lock: self._suppressed_members = {} self._all_clear_since = None self._last_action = f"restored {restored_members} ({reason})" def _set_interface_admin_state(self, ifname: str, target_up: bool) -> None: if not check_interface_exists(ifname): raise RuntimeError(f"Interface {ifname} disappeared while propagating link state") state_name = "up" if target_up else "down" with IPRoute() as ipr: indices = ipr.link_lookup(ifname=ifname) if not indices: raise RuntimeError(f"Interface {ifname} not found while propagating link state") ipr.link("set", index=indices[0], state=state_name) if target_up: with self._lock: self._settle_deadlines[ifname] = time.time() + settings.bridge_link_state_recovery_holdoff_seconds class BridgeLinkStateManager: """Track bridge link-state watchers keyed by bridge name.""" def __init__(self) -> None: self._watchers: dict[str, BridgeLinkStateWatcher] = {} self._lock = threading.Lock() def enable(self, bridge_name: str, poll_interval_seconds: Optional[float] = None) -> dict[str, Any]: """Start or replace the watcher for the given bridge.""" normalized_interval = poll_interval_seconds or settings.bridge_link_state_poll_interval_seconds with self._lock: old_watcher = self._watchers.pop(bridge_name, None) if old_watcher is not None: old_watcher.stop() watcher = BridgeLinkStateWatcher(bridge_name, normalized_interval) watcher.start() with self._lock: self._watchers[bridge_name] = watcher return watcher.status() def disable(self, bridge_name: str) -> dict[str, Any]: """Stop the watcher for one bridge.""" with self._lock: watcher = self._watchers.pop(bridge_name, None) if watcher is None: return { "bridge": bridge_name, "active": False, "message": "watcher not enabled", } watcher.stop() status = watcher.status() status["active"] = False return status def get_status(self, bridge_name: str) -> Optional[dict[str, Any]]: """Return the current watcher status for one bridge, if present.""" with self._lock: watcher = self._watchers.get(bridge_name) return watcher.status() if watcher is not None else None def list_statuses(self) -> list[dict[str, Any]]: """Return the current status for all bridge watchers.""" with self._lock: watchers = list(self._watchers.values()) return [watcher.status() for watcher in sorted(watchers, key=lambda item: item.bridge_name)] def stop(self) -> None: """Stop all running watchers.""" with self._lock: watchers = list(self._watchers.values()) self._watchers.clear() for watcher in watchers: watcher.stop() bridge_link_state_manager = BridgeLinkStateManager()