test nic state link api

backend/src/utilities/bridge_link_state_manager.py

@@ -0,0 +1,289 @@
+"""Background watcher that propagates bridge member link failures to sibling ports."""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from pyroute2 import IPRoute
+
+from src.config import settings
+from src.utilities.interface_bridge_helpers import (
+    check_interface_exists,
+    get_bridge_ports_once,
+    read_interface_admin_up,
+    read_interface_carrier,
+    read_interface_operstate,
+)
+
+logger = logging.getLogger("bridge_link_state_manager")
+
+
+@dataclass
+class MemberLinkState:
+    """Current sysfs-derived state for one bridge member."""
+
+    ifname: str
+    admin_up: Optional[bool]
+    carrier_up: Optional[bool]
+    operstate: Optional[str]
+
+    @property
+    def link_ready(self) -> bool:
+        """Return whether the member currently looks healthy enough to forward."""
+        if self.admin_up is not True:
+            return False
+        if self.carrier_up is False:
+            return False
+        if self.operstate in {"down", "lowerlayerdown", "notpresent"}:
+            return False
+        return True
+
+    def to_dict(self, suppressed: bool = False) -> Dict[str, Any]:
+        """Serialize member state for API responses."""
+        return {
+            "ifname": self.ifname,
+            "admin_up": self.admin_up,
+            "carrier_up": self.carrier_up,
+            "operstate": self.operstate,
+            "link_ready": self.link_ready,
+            "suppressed": suppressed,
+        }
+
+
+class BridgeLinkStateWatcher:
+    """Watch one bridge and mirror member failures to the other bridge members."""
+
+    def __init__(self, bridge_name: str, poll_interval_seconds: float) -> None:
+        self.bridge_name = bridge_name
+        self.poll_interval_seconds = poll_interval_seconds
+        self._stop_event = threading.Event()
+        self._lock = threading.Lock()
+        self._thread = threading.Thread(
+            target=self._run,
+            daemon=True,
+            name=f"bridge-link-state-{bridge_name}",
+        )
+        self._running = False
+        self._suppressed_members: dict[str, bool] = {}
+        self._last_poll_ts: Optional[float] = None
+        self._last_error: Optional[str] = None
+        self._last_action: Optional[str] = None
+        self._member_states: dict[str, MemberLinkState] = {}
+
+    def start(self) -> None:
+        """Start the watcher thread."""
+        with self._lock:
+            if self._running:
+                return
+            self._running = True
+        self._thread.start()
+
+    def stop(self) -> None:
+        """Stop the watcher and restore interfaces that were suppressed by it."""
+        self._stop_event.set()
+        if self._thread.is_alive():
+            self._thread.join(timeout=settings.bridge_link_state_thread_join_timeout_seconds)
+
+        try:
+            self._restore_suppressed_members(reason="watcher_stopped")
+        except Exception:
+            logger.exception("Failed to restore suppressed members for bridge=%s", self.bridge_name)
+
+        with self._lock:
+            self._running = False
+
+    def status(self) -> dict[str, Any]:
+        """Return a JSON-serializable snapshot of the watcher state."""
+        with self._lock:
+            members = {
+                ifname: state.to_dict(suppressed=ifname in self._suppressed_members)
+                for ifname, state in sorted(self._member_states.items())
+            }
+            return {
+                "bridge": self.bridge_name,
+                "active": self._running and self._thread.is_alive() and not self._stop_event.is_set(),
+                "poll_interval_seconds": self.poll_interval_seconds,
+                "last_poll_ts": self._last_poll_ts,
+                "last_error": self._last_error,
+                "last_action": self._last_action,
+                "suppressed_members": sorted(self._suppressed_members),
+                "members": members,
+            }
+
+    def _run(self) -> None:
+        while not self._stop_event.is_set():
+            try:
+                self._poll_once()
+                with self._lock:
+                    self._last_error = None
+            except Exception as exc:
+                logger.exception("Bridge link-state poll failed for bridge=%s", self.bridge_name)
+                with self._lock:
+                    self._last_error = str(exc)
+            self._stop_event.wait(self.poll_interval_seconds)
+
+    def _poll_once(self) -> None:
+        members = [iface for iface in get_bridge_ports_once(self.bridge_name) if check_interface_exists(iface)]
+        states = {iface: self._read_member_state(iface) for iface in members}
+        now = time.time()
+
+        with self._lock:
+            self._last_poll_ts = now
+            self._member_states = states
+            self._suppressed_members = {
+                iface: restore_up
+                for iface, restore_up in self._suppressed_members.items()
+                if iface in states
+            }
+
+        if len(states) < 2:
+            self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members")
+            return
+
+        suppressed_snapshot = set(self._suppressed_members)
+        failing_members = sorted(
+            ifname
+            for ifname, state in states.items()
+            if ifname not in suppressed_snapshot and not state.link_ready
+        )
+
+        if failing_members:
+            self._suppress_other_members(states, failing_members)
+            return
+
+        self._restore_suppressed_members(reason="all_members_recovered")
+
+    def _read_member_state(self, ifname: str) -> MemberLinkState:
+        return MemberLinkState(
+            ifname=ifname,
+            admin_up=read_interface_admin_up(ifname),
+            carrier_up=read_interface_carrier(ifname),
+            operstate=read_interface_operstate(ifname),
+        )
+
+    def _suppress_other_members(self, states: dict[str, MemberLinkState], failing_members: list[str]) -> None:
+        desired_suppressed = set(states) - set(failing_members)
+
+        with self._lock:
+            current_suppressed = dict(self._suppressed_members)
+
+        next_suppressed: dict[str, bool] = {}
+        changed_members: list[str] = []
+
+        for ifname in sorted(desired_suppressed):
+            restore_up = current_suppressed.get(ifname, states[ifname].admin_up is True)
+            if ifname not in current_suppressed and states[ifname].admin_up is True:
+                self._set_interface_admin_state(ifname, target_up=False)
+                changed_members.append(ifname)
+            next_suppressed[ifname] = restore_up
+
+        with self._lock:
+            self._suppressed_members = next_suppressed
+            action = (
+                f"suppressed {changed_members} because failing members={failing_members}"
+                if changed_members
+                else f"holding suppressed members because failing members={failing_members}"
+            )
+            self._last_action = action
+
+    def _restore_suppressed_members(self, reason: str) -> None:
+        with self._lock:
+            suppressed = dict(self._suppressed_members)
+
+        if not suppressed:
+            with self._lock:
+                self._last_action = f"no_restore_needed ({reason})"
+            return
+
+        restored_members: list[str] = []
+        for ifname, restore_up in sorted(suppressed.items()):
+            if not check_interface_exists(ifname):
+                continue
+            if restore_up:
+                self._set_interface_admin_state(ifname, target_up=True)
+                restored_members.append(ifname)
+
+        with self._lock:
+            self._suppressed_members = {}
+            self._last_action = f"restored {restored_members} ({reason})"
+
+    def _set_interface_admin_state(self, ifname: str, target_up: bool) -> None:
+        if not check_interface_exists(ifname):
+            raise RuntimeError(f"Interface {ifname} disappeared while propagating link state")
+
+        state_name = "up" if target_up else "down"
+        with IPRoute() as ipr:
+            indices = ipr.link_lookup(ifname=ifname)
+            if not indices:
+                raise RuntimeError(f"Interface {ifname} not found while propagating link state")
+            ipr.link("set", index=indices[0], state=state_name)
+
+
+class BridgeLinkStateManager:
+    """Track bridge link-state watchers keyed by bridge name."""
+
+    def __init__(self) -> None:
+        self._watchers: dict[str, BridgeLinkStateWatcher] = {}
+        self._lock = threading.Lock()
+
+    def enable(self, bridge_name: str, poll_interval_seconds: Optional[float] = None) -> dict[str, Any]:
+        """Start or replace the watcher for the given bridge."""
+        normalized_interval = poll_interval_seconds or settings.bridge_link_state_poll_interval_seconds
+        with self._lock:
+            old_watcher = self._watchers.pop(bridge_name, None)
+
+        if old_watcher is not None:
+            old_watcher.stop()
+
+        watcher = BridgeLinkStateWatcher(bridge_name, normalized_interval)
+        watcher.start()
+
+        with self._lock:
+            self._watchers[bridge_name] = watcher
+
+        return watcher.status()
+
+    def disable(self, bridge_name: str) -> dict[str, Any]:
+        """Stop the watcher for one bridge."""
+        with self._lock:
+            watcher = self._watchers.pop(bridge_name, None)
+
+        if watcher is None:
+            return {
+                "bridge": bridge_name,
+                "active": False,
+                "message": "watcher not enabled",
+            }
+
+        watcher.stop()
+        status = watcher.status()
+        status["active"] = False
+        return status
+
+    def get_status(self, bridge_name: str) -> Optional[dict[str, Any]]:
+        """Return the current watcher status for one bridge, if present."""
+        with self._lock:
+            watcher = self._watchers.get(bridge_name)
+        return watcher.status() if watcher is not None else None
+
+    def list_statuses(self) -> list[dict[str, Any]]:
+        """Return the current status for all bridge watchers."""
+        with self._lock:
+            watchers = list(self._watchers.values())
+        return [watcher.status() for watcher in sorted(watchers, key=lambda item: item.bridge_name)]
+
+    def stop(self) -> None:
+        """Stop all running watchers."""
+        with self._lock:
+            watchers = list(self._watchers.values())
+            self._watchers.clear()
+
+        for watcher in watchers:
+            watcher.stop()
+
+
+bridge_link_state_manager = BridgeLinkStateManager()