link watcher switch to event based instead of polling
All checks were successful
Build and Deploy MITM Webserver / traffic_target (push) Successful in 1s
Build and Deploy MITM Webserver / build (push) Successful in 9s

This commit is contained in:
2026-03-09 15:49:13 +01:00
parent c57d6ecc67
commit 665c0b4475
5 changed files with 118 additions and 54 deletions

View File

@@ -86,10 +86,10 @@ class BridgeRemoveRequest(BaseModel):
class BridgeLinkStateEnableRequest(BaseModel): class BridgeLinkStateEnableRequest(BaseModel):
"""Payload for enabling bridge member link-state propagation.""" """Payload for enabling bridge member link-state propagation."""
poll_interval_seconds: float = Field( recovery_holdoff_seconds: float = Field(
settings.bridge_link_state_poll_interval_seconds, settings.bridge_link_state_recovery_holdoff_seconds,
gt=0, gt=0,
description="Polling interval for member link-state checks.", description="Holdoff before sibling interfaces are restored after recovery.",
) )
@@ -109,11 +109,12 @@ class BridgeLinkStateWatcherStatus(BaseModel):
bridge: str = Field(..., description="Bridge interface name.") bridge: str = Field(..., description="Bridge interface name.")
active: bool = Field(..., description="Whether the watcher thread is currently active.") active: bool = Field(..., description="Whether the watcher thread is currently active.")
poll_interval_seconds: Optional[float] = Field(None, description="Watcher polling interval.") event_driven: Optional[bool] = Field(None, description="Whether the watcher is driven by netlink link events.")
last_poll_ts: Optional[float] = Field(None, description="Unix timestamp of the last poll.") last_event_ts: Optional[float] = Field(None, description="Unix timestamp of the last processed event/state evaluation.")
last_error: Optional[str] = Field(None, description="Most recent watcher error, if any.") last_error: Optional[str] = Field(None, description="Most recent watcher error, if any.")
last_action: Optional[str] = Field(None, description="Most recent propagation action.") last_action: Optional[str] = Field(None, description="Most recent propagation action.")
suppressed_members: List[str] = Field(default_factory=list, description="Members currently forced down by the watcher.") suppressed_members: List[str] = Field(default_factory=list, description="Members currently forced down by the watcher.")
recovery_holdoff_seconds: Optional[float] = Field(None, description="Configured recovery holdoff before restoring siblings.")
members: Dict[str, BridgeMemberLinkStateInfo] = Field(default_factory=dict, description="Per-member link-state snapshot.") members: Dict[str, BridgeMemberLinkStateInfo] = Field(default_factory=dict, description="Per-member link-state snapshot.")
message: Optional[str] = Field(None, description="Optional informational message.") message: Optional[str] = Field(None, description="Optional informational message.")
@@ -437,7 +438,7 @@ def enable_bridge_link_state_watcher(
status = bridge_link_state_manager.enable( status = bridge_link_state_manager.enable(
bridge_name=bridge_name, bridge_name=bridge_name,
poll_interval_seconds=req.poll_interval_seconds, recovery_holdoff_seconds=req.recovery_holdoff_seconds,
) )
return _watcher_status_response(status) return _watcher_status_response(status)

View File

@@ -52,7 +52,6 @@ class BackendSettings:
sniffer_buffer_drain_interval_seconds: float sniffer_buffer_drain_interval_seconds: float
sniffer_thread_join_timeout_seconds: float sniffer_thread_join_timeout_seconds: float
bridge_bpf_build_dir: str bridge_bpf_build_dir: str
bridge_link_state_poll_interval_seconds: float
bridge_link_state_thread_join_timeout_seconds: float bridge_link_state_thread_join_timeout_seconds: float
bridge_link_state_recovery_holdoff_seconds: float bridge_link_state_recovery_holdoff_seconds: float
telemetry_process_stop_timeout_seconds: float telemetry_process_stop_timeout_seconds: float
@@ -89,10 +88,6 @@ def load_settings() -> BackendSettings:
sniffer_buffer_drain_interval_seconds=_env_float("BACKEND_SNIFFER_BUFFER_DRAIN_INTERVAL_SECONDS", 5.0), sniffer_buffer_drain_interval_seconds=_env_float("BACKEND_SNIFFER_BUFFER_DRAIN_INTERVAL_SECONDS", 5.0),
sniffer_thread_join_timeout_seconds=_env_float("BACKEND_SNIFFER_THREAD_JOIN_TIMEOUT_SECONDS", 2.0), sniffer_thread_join_timeout_seconds=_env_float("BACKEND_SNIFFER_THREAD_JOIN_TIMEOUT_SECONDS", 2.0),
bridge_bpf_build_dir=_env_str("BACKEND_BRIDGE_BPF_BUILD_DIR", "/tmp/mitm-bpf"), bridge_bpf_build_dir=_env_str("BACKEND_BRIDGE_BPF_BUILD_DIR", "/tmp/mitm-bpf"),
bridge_link_state_poll_interval_seconds=_env_float(
"BACKEND_BRIDGE_LINK_STATE_POLL_INTERVAL_SECONDS",
0.25,
),
bridge_link_state_thread_join_timeout_seconds=_env_float( bridge_link_state_thread_join_timeout_seconds=_env_float(
"BACKEND_BRIDGE_LINK_STATE_THREAD_JOIN_TIMEOUT_SECONDS", "BACKEND_BRIDGE_LINK_STATE_THREAD_JOIN_TIMEOUT_SECONDS",
2.0, 2.0,

View File

@@ -1,8 +1,10 @@
"""Background watcher that propagates bridge member link failures to sibling ports.""" """Event-driven watcher that propagates bridge member link failures to sibling ports."""
from __future__ import annotations from __future__ import annotations
import logging import logging
import os
import select
import threading import threading
import time import time
from dataclasses import dataclass from dataclasses import dataclass
@@ -57,11 +59,14 @@ class MemberLinkState:
class BridgeLinkStateWatcher: class BridgeLinkStateWatcher:
"""Watch one bridge and mirror member failures to the other bridge members.""" """Watch one bridge and mirror member failures to the other bridge members."""
def __init__(self, bridge_name: str, poll_interval_seconds: float) -> None: def __init__(self, bridge_name: str, recovery_holdoff_seconds: float) -> None:
self.bridge_name = bridge_name self.bridge_name = bridge_name
self.poll_interval_seconds = poll_interval_seconds self.recovery_holdoff_seconds = recovery_holdoff_seconds
self._stop_event = threading.Event() self._stop_event = threading.Event()
self._lock = threading.Lock() self._lock = threading.Lock()
self._wake_r, self._wake_w = os.pipe()
os.set_blocking(self._wake_r, False)
os.set_blocking(self._wake_w, False)
self._thread = threading.Thread( self._thread = threading.Thread(
target=self._run, target=self._run,
daemon=True, daemon=True,
@@ -71,7 +76,7 @@ class BridgeLinkStateWatcher:
self._suppressed_members: dict[str, bool] = {} self._suppressed_members: dict[str, bool] = {}
self._settle_deadlines: dict[str, float] = {} self._settle_deadlines: dict[str, float] = {}
self._all_clear_since: Optional[float] = None self._all_clear_since: Optional[float] = None
self._last_poll_ts: Optional[float] = None self._last_event_ts: Optional[float] = None
self._last_error: Optional[str] = None self._last_error: Optional[str] = None
self._last_action: Optional[str] = None self._last_action: Optional[str] = None
self._member_states: dict[str, MemberLinkState] = {} self._member_states: dict[str, MemberLinkState] = {}
@@ -87,6 +92,7 @@ class BridgeLinkStateWatcher:
def stop(self) -> None: def stop(self) -> None:
"""Stop the watcher and restore interfaces that were suppressed by it.""" """Stop the watcher and restore interfaces that were suppressed by it."""
self._stop_event.set() self._stop_event.set()
self._wake_thread()
if self._thread.is_alive(): if self._thread.is_alive():
self._thread.join(timeout=settings.bridge_link_state_thread_join_timeout_seconds) self._thread.join(timeout=settings.bridge_link_state_thread_join_timeout_seconds)
@@ -98,6 +104,12 @@ class BridgeLinkStateWatcher:
with self._lock: with self._lock:
self._running = False self._running = False
for fd in (self._wake_r, self._wake_w):
try:
os.close(fd)
except OSError:
pass
def status(self) -> dict[str, Any]: def status(self) -> dict[str, Any]:
"""Return a JSON-serializable snapshot of the watcher state.""" """Return a JSON-serializable snapshot of the watcher state."""
with self._lock: with self._lock:
@@ -108,34 +120,60 @@ class BridgeLinkStateWatcher:
return { return {
"bridge": self.bridge_name, "bridge": self.bridge_name,
"active": self._running and self._thread.is_alive() and not self._stop_event.is_set(), "active": self._running and self._thread.is_alive() and not self._stop_event.is_set(),
"poll_interval_seconds": self.poll_interval_seconds, "event_driven": True,
"last_poll_ts": self._last_poll_ts, "last_event_ts": self._last_event_ts,
"last_error": self._last_error, "last_error": self._last_error,
"last_action": self._last_action, "last_action": self._last_action,
"suppressed_members": sorted(self._suppressed_members), "suppressed_members": sorted(self._suppressed_members),
"recovery_holdoff_seconds": settings.bridge_link_state_recovery_holdoff_seconds, "recovery_holdoff_seconds": self.recovery_holdoff_seconds,
"members": members, "members": members,
} }
def _run(self) -> None: def _run(self) -> None:
with IPRoute() as ipr:
ipr.bind()
self._evaluate_bridge_state(reason="watcher_started")
while not self._stop_event.is_set(): while not self._stop_event.is_set():
try: try:
self._poll_once() timeout = self._next_wait_timeout()
with self._lock: ready, _, _ = select.select([ipr, self._wake_r], [], [], timeout)
self._last_error = None
except Exception as exc: except Exception as exc:
logger.exception("Bridge link-state poll failed for bridge=%s", self.bridge_name) logger.exception("Bridge link-state select failed for bridge=%s", self.bridge_name)
with self._lock: with self._lock:
self._last_error = str(exc) self._last_error = str(exc)
self._stop_event.wait(self.poll_interval_seconds) continue
def _poll_once(self) -> None: if self._stop_event.is_set():
break
if self._wake_r in ready:
self._drain_wake_pipe()
continue
if ipr in ready:
try:
messages = ipr.get()
except Exception as exc:
logger.exception("Bridge link-state netlink read failed for bridge=%s", self.bridge_name)
with self._lock:
self._last_error = str(exc)
continue
if any(msg.get("event") in {"RTM_NEWLINK", "RTM_DELLINK"} for msg in messages):
self._evaluate_bridge_state(reason="netlink_event")
continue
self._evaluate_bridge_state(reason="recovery_deadline")
def _evaluate_bridge_state(self, reason: str) -> None:
members = [iface for iface in get_bridge_ports_once(self.bridge_name) if check_interface_exists(iface)] members = [iface for iface in get_bridge_ports_once(self.bridge_name) if check_interface_exists(iface)]
states = {iface: self._read_member_state(iface) for iface in members} states = {iface: self._read_member_state(iface) for iface in members}
now = time.time() now = time.time()
with self._lock: with self._lock:
self._last_poll_ts = now self._last_event_ts = now
self._last_error = None
self._member_states = states self._member_states = states
self._suppressed_members = { self._suppressed_members = {
iface: restore_up iface: restore_up
@@ -149,6 +187,7 @@ class BridgeLinkStateWatcher:
} }
if len(states) < 2: if len(states) < 2:
with self._lock:
self._all_clear_since = None self._all_clear_since = None
self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members") self._restore_suppressed_members(reason="bridge_has_fewer_than_two_members")
return return
@@ -176,22 +215,44 @@ class BridgeLinkStateWatcher:
with self._lock: with self._lock:
if self._all_clear_since is None: if self._all_clear_since is None:
self._all_clear_since = now self._all_clear_since = now
should_restore = now - self._all_clear_since >= settings.bridge_link_state_recovery_holdoff_seconds should_restore = now - self._all_clear_since >= self.recovery_holdoff_seconds
if not should_restore: if not should_restore:
remaining = max( remaining = max(0.0, self.recovery_holdoff_seconds - (now - self._all_clear_since))
0.0,
settings.bridge_link_state_recovery_holdoff_seconds - (now - self._all_clear_since),
)
self._last_action = f"waiting {remaining:.2f}s before restoring suppressed members" self._last_action = f"waiting {remaining:.2f}s before restoring suppressed members"
if should_restore: if should_restore:
self._restore_suppressed_members(reason="all_members_recovered") self._restore_suppressed_members(reason=reason)
return return
with self._lock: with self._lock:
self._all_clear_since = None self._all_clear_since = None
self._restore_suppressed_members(reason="all_members_recovered") self._restore_suppressed_members(reason=reason)
def _next_wait_timeout(self) -> Optional[float]:
"""Return how long the watcher may sleep before the next restore deadline."""
with self._lock:
if self._suppressed_members and self._all_clear_since is not None:
deadline = self._all_clear_since + self.recovery_holdoff_seconds
return max(0.0, deadline - time.time())
return None
def _wake_thread(self) -> None:
"""Wake the event loop from another thread."""
try:
os.write(self._wake_w, b"\x00")
except OSError:
pass
def _drain_wake_pipe(self) -> None:
"""Drain pending wake-up bytes from the control pipe."""
try:
while os.read(self._wake_r, 4096):
pass
except BlockingIOError:
return
except OSError:
return
def _read_member_state(self, ifname: str) -> MemberLinkState: def _read_member_state(self, ifname: str) -> MemberLinkState:
return MemberLinkState( return MemberLinkState(
@@ -261,7 +322,7 @@ class BridgeLinkStateWatcher:
if target_up: if target_up:
with self._lock: with self._lock:
self._settle_deadlines[ifname] = time.time() + settings.bridge_link_state_recovery_holdoff_seconds self._settle_deadlines[ifname] = time.time() + self.recovery_holdoff_seconds
class BridgeLinkStateManager: class BridgeLinkStateManager:
@@ -271,16 +332,16 @@ class BridgeLinkStateManager:
self._watchers: dict[str, BridgeLinkStateWatcher] = {} self._watchers: dict[str, BridgeLinkStateWatcher] = {}
self._lock = threading.Lock() self._lock = threading.Lock()
def enable(self, bridge_name: str, poll_interval_seconds: Optional[float] = None) -> dict[str, Any]: def enable(self, bridge_name: str, recovery_holdoff_seconds: Optional[float] = None) -> dict[str, Any]:
"""Start or replace the watcher for the given bridge.""" """Start or replace the watcher for the given bridge."""
normalized_interval = poll_interval_seconds or settings.bridge_link_state_poll_interval_seconds normalized_holdoff = recovery_holdoff_seconds or settings.bridge_link_state_recovery_holdoff_seconds
with self._lock: with self._lock:
old_watcher = self._watchers.pop(bridge_name, None) old_watcher = self._watchers.pop(bridge_name, None)
if old_watcher is not None: if old_watcher is not None:
old_watcher.stop() old_watcher.stop()
watcher = BridgeLinkStateWatcher(bridge_name, normalized_interval) watcher = BridgeLinkStateWatcher(bridge_name, normalized_holdoff)
watcher.start() watcher.start()
with self._lock: with self._lock:

View File

@@ -284,11 +284,11 @@ export default function Network() {
function openEnableWatcherModal(bridgeName: string) { function openEnableWatcherModal(bridgeName: string) {
setWatcherBridgeName(bridgeName); setWatcherBridgeName(bridgeName);
watcherForm.setFieldsValue({ poll_interval_seconds: 0.25 }); watcherForm.setFieldsValue({ recovery_holdoff_seconds: 1.0 });
setWatcherModalVisible(true); setWatcherModalVisible(true);
} }
function handleEnableWatcher(values: { poll_interval_seconds: number }) { function handleEnableWatcher(values: { recovery_holdoff_seconds: number }) {
if (!watcherBridgeName) { if (!watcherBridgeName) {
return; return;
} }
@@ -434,8 +434,9 @@ export default function Network() {
> >
<Descriptions size="small" column={4} style={{ marginBottom: members.length ? 12 : 0 }}> <Descriptions size="small" column={4} style={{ marginBottom: members.length ? 12 : 0 }}>
<Descriptions.Item label="Members">{bridge.members.map((member) => member.ifname).join(', ') || '—'}</Descriptions.Item> <Descriptions.Item label="Members">{bridge.members.map((member) => member.ifname).join(', ') || '—'}</Descriptions.Item>
<Descriptions.Item label="Poll interval"> <Descriptions.Item label="Mode">{watcher?.event_driven ? 'netlink events' : '—'}</Descriptions.Item>
{watcher?.poll_interval_seconds ? `${watcher.poll_interval_seconds}s` : '—'} <Descriptions.Item label="Recovery holdoff">
{watcher?.recovery_holdoff_seconds ? `${watcher.recovery_holdoff_seconds}s` : '—'}
</Descriptions.Item> </Descriptions.Item>
<Descriptions.Item label="Suppressed"> <Descriptions.Item label="Suppressed">
{watcher?.suppressed_members.length ? watcher.suppressed_members.join(', ') : 'none'} {watcher?.suppressed_members.length ? watcher.suppressed_members.join(', ') : 'none'}
@@ -501,16 +502,21 @@ export default function Network() {
}} }}
onOk={() => watcherForm.submit()} onOk={() => watcherForm.submit()}
> >
<Form form={watcherForm} layout="vertical" onFinish={handleEnableWatcher} initialValues={{ poll_interval_seconds: 0.25 }}> <Form
form={watcherForm}
layout="vertical"
onFinish={handleEnableWatcher}
initialValues={{ recovery_holdoff_seconds: 1.0 }}
>
<Form.Item <Form.Item
name="poll_interval_seconds" name="recovery_holdoff_seconds"
label="Poll interval (seconds)" label="Recovery holdoff (seconds)"
rules={[{ required: true, message: 'Please provide a poll interval' }]} rules={[{ required: true, message: 'Please provide a recovery holdoff' }]}
> >
<InputNumber min={0.05} max={10} step={0.05} precision={2} style={{ width: '100%' }} /> <InputNumber min={0.05} max={10} step={0.05} precision={2} style={{ width: '100%' }} />
</Form.Item> </Form.Item>
<Paragraph type="secondary" style={{ marginBottom: 0 }}> <Paragraph type="secondary" style={{ marginBottom: 0 }}>
When enabled, the backend suppresses sibling bridge ports if one member loses link and restores them after recovery. The backend reacts to kernel link events and waits for this holdoff before restoring sibling ports after recovery.
</Paragraph> </Paragraph>
</Form> </Form>
</Modal> </Modal>

View File

@@ -52,7 +52,7 @@ export interface BridgeRemoveRequest {
} }
export interface BridgeLinkStateEnableRequest { export interface BridgeLinkStateEnableRequest {
poll_interval_seconds: number; recovery_holdoff_seconds: number;
} }
export interface BridgeMemberLinkStateInfo { export interface BridgeMemberLinkStateInfo {
@@ -67,11 +67,12 @@ export interface BridgeMemberLinkStateInfo {
export interface BridgeLinkStateWatcherStatus { export interface BridgeLinkStateWatcherStatus {
bridge: string; bridge: string;
active: boolean; active: boolean;
poll_interval_seconds?: number | null; event_driven?: boolean | null;
last_poll_ts?: number | null; last_event_ts?: number | null;
last_error?: string | null; last_error?: string | null;
last_action?: string | null; last_action?: string | null;
suppressed_members: string[]; suppressed_members: string[];
recovery_holdoff_seconds?: number | null;
members: Record<string, BridgeMemberLinkStateInfo>; members: Record<string, BridgeMemberLinkStateInfo>;
message?: string | null; message?: string | null;
} }