marcus/mitm-webserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

try overload fix 2, backend upsert and test script fix
All checks were successful
Build and Deploy MITM Webserver / traffic_target (push) Successful in 0s
Details
Build and Deploy MITM Webserver / build (push) Successful in 12s
Details

Browse Source
This commit is contained in:
malmert
2026-05-03 19:13:24 +02:00
parent 3eb39a71ad
commit b4e4e27c4b
4 changed files with 281 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
backend/src/config.py
View File

@@ -43,6 +43,7 @@ class BackendSettings:
packet_tracker_retention_seconds: float packet_tracker_retention_seconds: float
packet_tracker_min_flush_interval_seconds: float packet_tracker_min_flush_interval_seconds: float
packet_tracker_persist_timeout_seconds: float packet_tracker_persist_timeout_seconds: float
packet_tracker_batch_persist_timeout_seconds: float
packet_tracker_persist_retry_backoff_seconds: float packet_tracker_persist_retry_backoff_seconds: float
packet_tracker_persist_retry_backoff_max_seconds: float packet_tracker_persist_retry_backoff_max_seconds: float
packet_tracker_error_log_interval_seconds: float packet_tracker_error_log_interval_seconds: float
@@ -89,6 +90,10 @@ def load_settings() -> BackendSettings:
packet_tracker_retention_seconds=_env_float("BACKEND_PACKET_TRACKER_RETENTION_SECONDS", 10.0), packet_tracker_retention_seconds=_env_float("BACKEND_PACKET_TRACKER_RETENTION_SECONDS", 10.0),
packet_tracker_min_flush_interval_seconds=_env_float("BACKEND_PACKET_TRACKER_MIN_FLUSH_INTERVAL_SECONDS", 0.05), packet_tracker_min_flush_interval_seconds=_env_float("BACKEND_PACKET_TRACKER_MIN_FLUSH_INTERVAL_SECONDS", 0.05),
packet_tracker_persist_timeout_seconds=_env_float("BACKEND_PACKET_TRACKER_PERSIST_TIMEOUT_SECONDS", 2.0), packet_tracker_persist_timeout_seconds=_env_float("BACKEND_PACKET_TRACKER_PERSIST_TIMEOUT_SECONDS", 2.0),
packet_tracker_batch_persist_timeout_seconds=_env_float(
"BACKEND_PACKET_TRACKER_BATCH_PERSIST_TIMEOUT_SECONDS",
10.0,
),
packet_tracker_persist_retry_backoff_seconds=_env_float( packet_tracker_persist_retry_backoff_seconds=_env_float(
"BACKEND_PACKET_TRACKER_PERSIST_RETRY_BACKOFF_SECONDS", "BACKEND_PACKET_TRACKER_PERSIST_RETRY_BACKOFF_SECONDS",
0.25, 0.25,

323
backend/src/utilities/database.py
View File

@@ -263,6 +263,42 @@ class DatabasePool:
if self._pool is None: if self._pool is None:
await self.init_pool() await self.init_pool()
if self._pool is None:
return
try:
async with self._pool.acquire() as conn:
row = await self._upsert_packet_with_conn(conn, pkt_info)
except Exception:
logger.exception("DB upsert failed")
return
self._publish_packet_row(pkt_info, row)
async def upsert_packets(self, packets: List[Dict[str, Any]]) -> None:
"""Insert or update packet records using one database connection for the batch."""
if not packets:
return
if self._pool is None:
await self.init_pool()
if self._pool is None:
return
published_rows: List[tuple[Dict[str, Any], Optional[Dict[str, Any]]]] = []
try:
async with self._pool.acquire() as conn:
for pkt_info in packets:
row = await self._upsert_packet_with_conn(conn, pkt_info)
published_rows.append((pkt_info, row))
except Exception:
logger.exception("DB packet batch upsert failed")
raise
for pkt_info, row in published_rows:
self._publish_packet_row(pkt_info, row)
async def _upsert_packet_with_conn(self, conn: asyncpg.Connection, pkt_info: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""Insert or update one packet record using an already-acquired connection."""
_normalize_json_fields(pkt_info) _normalize_json_fields(pkt_info)
_attach_derived_fields(pkt_info) _attach_derived_fields(pkt_info)
@@ -270,154 +306,151 @@ class DatabasePool:
telemetry_metadata = pkt_info.get("telemetry_metadata") telemetry_metadata = pkt_info.get("telemetry_metadata")
capture_observations = pkt_info.get("capture_observations") capture_observations = pkt_info.get("capture_observations")
try: row = await conn.fetchrow(
async with self._pool.acquire() as conn: """
row = await conn.fetchrow( INSERT INTO packets (
""" timestamp,
INSERT INTO packets ( correlation_key,
timestamp, packet_id,
correlation_key, packet_uid,
packet_id, flow_id,
packet_uid, capture_session_id,
flow_id, correlation_source,
capture_session_id, skb_mark,
correlation_source, capture_iface,
skb_mark, ingress_if,
capture_iface, egress_if,
ingress_if, verdict,
egress_if, verdict_reason,
verdict, verdict_confidence,
verdict_reason, ingress_seen_at,
verdict_confidence, egress_seen_at,
ingress_seen_at, verdict_seen_at,
egress_seen_at, src_mac,
verdict_seen_at, dst_mac,
src_mac, eth_type_raw,
dst_mac, vlan_id,
eth_type_raw, src_ip,
vlan_id, dst_ip,
src_ip, ip_proto_raw,
dst_ip, src_port,
ip_proto_raw, dst_port,
src_port, length,
dst_port, capture_sources,
length, app_protocol,
capture_sources, app_category,
app_protocol, app_confidence,
app_category, app_hostname,
app_confidence, app_is_encrypted,
app_hostname, app_risk_score,
app_is_encrypted, dpi_metadata,
app_risk_score, capture_metadata,
dpi_metadata, telemetry_metadata,
capture_metadata, capture_observations,
telemetry_metadata, raw
capture_observations, ) VALUES(
raw $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,
) VALUES( $21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35::jsonb,$36::jsonb,$37::jsonb,$38::jsonb,$39
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20, )
$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35::jsonb,$36::jsonb,$37::jsonb,$38::jsonb,$39 ON CONFLICT (correlation_key) DO UPDATE SET
updated_at = NOW(),
timestamp = CASE
WHEN packets.timestamp IS NULL THEN EXCLUDED.timestamp
WHEN EXCLUDED.timestamp IS NULL THEN packets.timestamp
ELSE LEAST(packets.timestamp, EXCLUDED.timestamp)
END,
packet_id = COALESCE(EXCLUDED.packet_id, packets.packet_id),
packet_uid = COALESCE(EXCLUDED.packet_uid, packets.packet_uid),
flow_id = COALESCE(EXCLUDED.flow_id, packets.flow_id),
capture_session_id = COALESCE(EXCLUDED.capture_session_id, packets.capture_session_id),
correlation_source = COALESCE(EXCLUDED.correlation_source, packets.correlation_source),
skb_mark = COALESCE(EXCLUDED.skb_mark, packets.skb_mark),
capture_iface = COALESCE(EXCLUDED.capture_iface, packets.capture_iface),
ingress_if = COALESCE(EXCLUDED.ingress_if, packets.ingress_if),
egress_if = COALESCE(EXCLUDED.egress_if, packets.egress_if),
verdict = COALESCE(EXCLUDED.verdict, packets.verdict),
verdict_reason = COALESCE(EXCLUDED.verdict_reason, packets.verdict_reason),
verdict_confidence = COALESCE(EXCLUDED.verdict_confidence, packets.verdict_confidence),
ingress_seen_at = COALESCE(EXCLUDED.ingress_seen_at, packets.ingress_seen_at),
egress_seen_at = COALESCE(EXCLUDED.egress_seen_at, packets.egress_seen_at),
verdict_seen_at = COALESCE(EXCLUDED.verdict_seen_at, packets.verdict_seen_at),
src_mac = COALESCE(EXCLUDED.src_mac, packets.src_mac),
dst_mac = COALESCE(EXCLUDED.dst_mac, packets.dst_mac),
eth_type_raw = COALESCE(EXCLUDED.eth_type_raw, packets.eth_type_raw),
vlan_id = COALESCE(EXCLUDED.vlan_id, packets.vlan_id),
src_ip = COALESCE(EXCLUDED.src_ip, packets.src_ip),
dst_ip = COALESCE(EXCLUDED.dst_ip, packets.dst_ip),
ip_proto_raw = COALESCE(EXCLUDED.ip_proto_raw, packets.ip_proto_raw),
src_port = COALESCE(EXCLUDED.src_port, packets.src_port),
dst_port = COALESCE(EXCLUDED.dst_port, packets.dst_port),
length = COALESCE(EXCLUDED.length, packets.length),
capture_sources = (
SELECT ARRAY(
SELECT DISTINCT source
FROM unnest(
COALESCE(packets.capture_sources, ARRAY[]::text[]) ||
COALESCE(EXCLUDED.capture_sources, ARRAY[]::text[])
) AS source
) )
ON CONFLICT (correlation_key) DO UPDATE SET ),
updated_at = NOW(), app_protocol = COALESCE(EXCLUDED.app_protocol, packets.app_protocol),
timestamp = CASE app_category = COALESCE(EXCLUDED.app_category, packets.app_category),
WHEN packets.timestamp IS NULL THEN EXCLUDED.timestamp app_confidence = COALESCE(EXCLUDED.app_confidence, packets.app_confidence),
WHEN EXCLUDED.timestamp IS NULL THEN packets.timestamp app_hostname = COALESCE(EXCLUDED.app_hostname, packets.app_hostname),
ELSE LEAST(packets.timestamp, EXCLUDED.timestamp) app_is_encrypted = COALESCE(EXCLUDED.app_is_encrypted, packets.app_is_encrypted),
END, app_risk_score = COALESCE(EXCLUDED.app_risk_score, packets.app_risk_score),
packet_id = COALESCE(EXCLUDED.packet_id, packets.packet_id), dpi_metadata = COALESCE(EXCLUDED.dpi_metadata, packets.dpi_metadata),
packet_uid = COALESCE(EXCLUDED.packet_uid, packets.packet_uid), capture_metadata = COALESCE(EXCLUDED.capture_metadata, packets.capture_metadata),
flow_id = COALESCE(EXCLUDED.flow_id, packets.flow_id), telemetry_metadata = COALESCE(EXCLUDED.telemetry_metadata, packets.telemetry_metadata),
capture_session_id = COALESCE(EXCLUDED.capture_session_id, packets.capture_session_id), capture_observations = COALESCE(EXCLUDED.capture_observations, packets.capture_observations),
correlation_source = COALESCE(EXCLUDED.correlation_source, packets.correlation_source), raw = COALESCE(EXCLUDED.raw, packets.raw)
skb_mark = COALESCE(EXCLUDED.skb_mark, packets.skb_mark), RETURNING *
capture_iface = COALESCE(EXCLUDED.capture_iface, packets.capture_iface), """,
ingress_if = COALESCE(EXCLUDED.ingress_if, packets.ingress_if), pkt_info.get("timestamp"),
egress_if = COALESCE(EXCLUDED.egress_if, packets.egress_if), pkt_info["correlation_key"],
verdict = COALESCE(EXCLUDED.verdict, packets.verdict), pkt_info.get("packet_id"),
verdict_reason = COALESCE(EXCLUDED.verdict_reason, packets.verdict_reason), pkt_info.get("packet_uid"),
verdict_confidence = COALESCE(EXCLUDED.verdict_confidence, packets.verdict_confidence), pkt_info.get("flow_id"),
ingress_seen_at = COALESCE(EXCLUDED.ingress_seen_at, packets.ingress_seen_at), pkt_info.get("capture_session_id"),
egress_seen_at = COALESCE(EXCLUDED.egress_seen_at, packets.egress_seen_at), pkt_info.get("correlation_source"),
verdict_seen_at = COALESCE(EXCLUDED.verdict_seen_at, packets.verdict_seen_at), pkt_info.get("skb_mark"),
src_mac = COALESCE(EXCLUDED.src_mac, packets.src_mac), pkt_info.get("capture_iface"),
dst_mac = COALESCE(EXCLUDED.dst_mac, packets.dst_mac), pkt_info.get("ingress_if"),
eth_type_raw = COALESCE(EXCLUDED.eth_type_raw, packets.eth_type_raw), pkt_info.get("egress_if"),
vlan_id = COALESCE(EXCLUDED.vlan_id, packets.vlan_id), pkt_info.get("verdict"),
src_ip = COALESCE(EXCLUDED.src_ip, packets.src_ip), pkt_info.get("verdict_reason"),
dst_ip = COALESCE(EXCLUDED.dst_ip, packets.dst_ip), pkt_info.get("verdict_confidence"),
ip_proto_raw = COALESCE(EXCLUDED.ip_proto_raw, packets.ip_proto_raw), pkt_info.get("ingress_seen_at"),
src_port = COALESCE(EXCLUDED.src_port, packets.src_port), pkt_info.get("egress_seen_at"),
dst_port = COALESCE(EXCLUDED.dst_port, packets.dst_port), pkt_info.get("verdict_seen_at"),
length = COALESCE(EXCLUDED.length, packets.length), pkt_info.get("src_mac"),
capture_sources = ( pkt_info.get("dst_mac"),
SELECT ARRAY( pkt_info.get("eth_type_raw"),
SELECT DISTINCT source pkt_info.get("vlan_id"),
FROM unnest( pkt_info.get("src_ip"),
COALESCE(packets.capture_sources, ARRAY[]::text[]) || pkt_info.get("dst_ip"),
COALESCE(EXCLUDED.capture_sources, ARRAY[]::text[]) pkt_info.get("protocol_raw"),
) AS source pkt_info.get("src_port"),
) pkt_info.get("dst_port"),
), pkt_info.get("length"),
app_protocol = COALESCE(EXCLUDED.app_protocol, packets.app_protocol), pkt_info.get("capture_sources"),
app_category = COALESCE(EXCLUDED.app_category, packets.app_category), pkt_info.get("app_protocol"),
app_confidence = COALESCE(EXCLUDED.app_confidence, packets.app_confidence), pkt_info.get("app_category"),
app_hostname = COALESCE(EXCLUDED.app_hostname, packets.app_hostname), pkt_info.get("app_confidence"),
app_is_encrypted = COALESCE(EXCLUDED.app_is_encrypted, packets.app_is_encrypted), pkt_info.get("app_hostname"),
app_risk_score = COALESCE(EXCLUDED.app_risk_score, packets.app_risk_score), pkt_info.get("app_is_encrypted"),
dpi_metadata = COALESCE(EXCLUDED.dpi_metadata, packets.dpi_metadata), pkt_info.get("app_risk_score"),
capture_metadata = COALESCE(EXCLUDED.capture_metadata, packets.capture_metadata), json.dumps(dpi_metadata) if dpi_metadata is not None else None,
telemetry_metadata = COALESCE(EXCLUDED.telemetry_metadata, packets.telemetry_metadata), json.dumps(pkt_info.get("capture_metadata")) if pkt_info.get("capture_metadata") is not None else None,
capture_observations = COALESCE(EXCLUDED.capture_observations, packets.capture_observations), json.dumps(telemetry_metadata) if telemetry_metadata is not None else None,
raw = COALESCE(EXCLUDED.raw, packets.raw) json.dumps(capture_observations) if capture_observations is not None else None,
RETURNING * pkt_info.get("raw"),
""", )
pkt_info.get("timestamp"), return dict(row) if row else None
pkt_info["correlation_key"],
pkt_info.get("packet_id"),
pkt_info.get("packet_uid"),
pkt_info.get("flow_id"),
pkt_info.get("capture_session_id"),
pkt_info.get("correlation_source"),
pkt_info.get("skb_mark"),
pkt_info.get("capture_iface"),
pkt_info.get("ingress_if"),
pkt_info.get("egress_if"),
pkt_info.get("verdict"),
pkt_info.get("verdict_reason"),
pkt_info.get("verdict_confidence"),
pkt_info.get("ingress_seen_at"),
pkt_info.get("egress_seen_at"),
pkt_info.get("verdict_seen_at"),
pkt_info.get("src_mac"),
pkt_info.get("dst_mac"),
pkt_info.get("eth_type_raw"),
pkt_info.get("vlan_id"),
pkt_info.get("src_ip"),
pkt_info.get("dst_ip"),
pkt_info.get("protocol_raw"),
pkt_info.get("src_port"),
pkt_info.get("dst_port"),
pkt_info.get("length"),
pkt_info.get("capture_sources"),
pkt_info.get("app_protocol"),
pkt_info.get("app_category"),
pkt_info.get("app_confidence"),
pkt_info.get("app_hostname"),
pkt_info.get("app_is_encrypted"),
pkt_info.get("app_risk_score"),
json.dumps(dpi_metadata) if dpi_metadata is not None else None,
json.dumps(pkt_info.get("capture_metadata")) if pkt_info.get("capture_metadata") is not None else None,
json.dumps(telemetry_metadata) if telemetry_metadata is not None else None,
json.dumps(capture_observations) if capture_observations is not None else None,
pkt_info.get("raw"),
)
except Exception:
logger.exception("DB upsert failed")
return
def _publish_packet_row(self, pkt_info: Dict[str, Any], row: Optional[Dict[str, Any]]) -> None:
if row: if row:
persisted = _serialize_row_for_broadcast(dict(row)) persisted = _serialize_row_for_broadcast(row)
pkt_info.update(persisted) pkt_info.update(persisted)
if self.broadcaster: if self.broadcaster:

60
backend/src/utilities/packet_tracker.py
View File

@@ -178,6 +178,8 @@ class PacketTracker:
"persist_failed_total": 0, "persist_failed_total": 0,
"persist_timeout_total": 0, "persist_timeout_total": 0,
"persist_failure_log_suppressed": 0, "persist_failure_log_suppressed": 0,
"persist_batch_total": 0,
"persist_batch_failed_total": 0,
"evicted_persisted_total": 0, "evicted_persisted_total": 0,
"evicted_unpersisted_total": 0, "evicted_unpersisted_total": 0,
} }
@@ -689,8 +691,7 @@ class PacketTracker:
for correlation_key in expired_keys: for correlation_key in expired_keys:
self._entries.pop(correlation_key, None) self._entries.pop(correlation_key, None)
for entry in due_entries: self._persist_batch(due_entries)
self._persist(entry)
def _persist_backoff_elapsed(self, entry: Dict[str, Any], now_ts: float) -> bool: def _persist_backoff_elapsed(self, entry: Dict[str, Any], now_ts: float) -> bool:
failures = int(entry.get("persist_failures") or 0) failures = int(entry.get("persist_failures") or 0)
@@ -707,24 +708,37 @@ class PacketTracker:
) )
return now_ts - float(entry.get("last_persist_attempt_at") or 0.0) >= backoff return now_ts - float(entry.get("last_persist_attempt_at") or 0.0) >= backoff
def _persist(self, entry: Dict[str, Any]) -> None: def _persist_batch(self, entries: List[Dict[str, Any]]) -> None:
payload = dict(entry["payload"]) if not entries:
return
payloads = [dict(entry["payload"]) for entry in entries]
web_loop = getattr(shared_objects, "web_loop", None) web_loop = getattr(shared_objects, "web_loop", None)
web_db = getattr(shared_objects, "db", None) web_db = getattr(shared_objects, "db", None)
if web_loop is None or web_db is None: if web_loop is None or web_db is None:
return return
fut: concurrent.futures.Future[Any]
try: try:
fut = asyncio.run_coroutine_threadsafe(web_db.upsert_packet(payload), web_loop) if hasattr(web_db, "upsert_packets"):
fut.result(timeout=settings.packet_tracker_persist_timeout_seconds) fut = asyncio.run_coroutine_threadsafe(web_db.upsert_packets(payloads), web_loop)
else:
fut = asyncio.run_coroutine_threadsafe(self._persist_payloads_one_by_one(web_db, payloads), web_loop)
timeout = max(
settings.packet_tracker_persist_timeout_seconds,
settings.packet_tracker_batch_persist_timeout_seconds,
)
fut.result(timeout=timeout)
with self._lock: with self._lock:
current = self._entries.get(entry["correlation_key"]) self._stats["persist_batch_total"] += 1
if current is not None: persisted_at = time.time()
current["persisted"] = True for entry in entries:
current["dirty"] = False current = self._entries.get(entry["correlation_key"])
current["last_persisted_at"] = time.time() if current is not None:
current["persist_failures"] = 0 current["persisted"] = True
current["dirty"] = False
current["last_persisted_at"] = persisted_at
current["persist_failures"] = 0
except Exception as exc: except Exception as exc:
try: try:
fut.cancel() fut.cancel()
@@ -733,26 +747,32 @@ class PacketTracker:
is_timeout = isinstance(exc, (TimeoutError, concurrent.futures.TimeoutError, asyncio.TimeoutError)) is_timeout = isinstance(exc, (TimeoutError, concurrent.futures.TimeoutError, asyncio.TimeoutError))
with self._lock: with self._lock:
self._stats["persist_failed_total"] += 1 self._stats["persist_batch_failed_total"] += 1
self._stats["persist_failed_total"] += len(entries)
if is_timeout: if is_timeout:
self._stats["persist_timeout_total"] += 1 self._stats["persist_timeout_total"] += len(entries)
current = self._entries.get(entry["correlation_key"]) for entry in entries:
if current is not None: current = self._entries.get(entry["correlation_key"])
current["dirty"] = True if current is not None:
current["persist_failures"] = int(current.get("persist_failures") or 0) + 1 current["dirty"] = True
current["persist_failures"] = int(current.get("persist_failures") or 0) + 1
now_ts = time.time() now_ts = time.time()
if now_ts - self._last_persist_error_log_at >= settings.packet_tracker_error_log_interval_seconds: if now_ts - self._last_persist_error_log_at >= settings.packet_tracker_error_log_interval_seconds:
self._last_persist_error_log_at = now_ts self._last_persist_error_log_at = now_ts
logger.warning( logger.warning(
"Packet persistence is overloaded; failed to persist %s (%s). Further errors are rate-limited.", "Packet persistence is overloaded; failed to persist batch of %s packets (%s). Further errors are rate-limited.",
entry["correlation_key"], len(entries),
type(exc).__name__, type(exc).__name__,
) )
else: else:
with self._lock: with self._lock:
self._stats["persist_failure_log_suppressed"] += 1 self._stats["persist_failure_log_suppressed"] += 1
async def _persist_payloads_one_by_one(self, web_db: Any, payloads: List[Dict[str, Any]]) -> None:
for payload in payloads:
await web_db.upsert_packet(payload)
def _record_stats(self, payload: Dict[str, Any]) -> None: def _record_stats(self, payload: Dict[str, Any]) -> None:
capture_sources = set(payload.get("capture_sources") or []) capture_sources = set(payload.get("capture_sources") or [])
self._stats["persisted_total"] += 1 self._stats["persisted_total"] += 1

67
tools/network_timing_benchmark.py
View File

@@ -24,6 +24,7 @@ import os
import platform import platform
import re import re
import shutil import shutil
import signal
import socket import socket
import statistics import statistics
import subprocess import subprocess
@@ -264,33 +265,42 @@ def run_command(
start_monotonic = time.perf_counter() start_monotonic = time.perf_counter()
start_wall = now_iso() start_wall = now_iso()
sampler.start() sampler.start()
process: subprocess.Popen[str] | None = None
try: try:
completed = subprocess.run( process = subprocess.Popen(
command, command,
cwd=str(cwd), cwd=str(cwd),
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, text=True,
timeout=timeout, start_new_session=True,
check=False,
) )
stdout, stderr = process.communicate(timeout=timeout)
result = CommandRun( result = CommandRun(
args=command, args=command,
returncode=completed.returncode, returncode=process.returncode,
stdout=completed.stdout or "", stdout=stdout or "",
stderr=completed.stderr or "", stderr=stderr or "",
) )
except subprocess.TimeoutExpired as exc: except subprocess.TimeoutExpired as exc:
stdout = exc.stdout.decode("utf-8", errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "") if process is not None:
stderr = exc.stderr.decode("utf-8", errors="replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "") terminate_process_group(process)
stdout, stderr = process.communicate()
else:
stdout = exc.stdout.decode("utf-8", errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
stderr = exc.stderr.decode("utf-8", errors="replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
result = CommandRun( result = CommandRun(
args=command, args=command,
returncode=None, returncode=process.returncode if process is not None else None,
stdout=stdout, stdout=stdout,
stderr=stderr, stderr=stderr,
timed_out=True, timed_out=True,
error=f"command timed out after {timeout} seconds", error=f"command timed out after {timeout} seconds",
) )
except KeyboardInterrupt:
if process is not None:
terminate_process_group(process)
raise
except OSError as exc: except OSError as exc:
result = CommandRun( result = CommandRun(
args=command, args=command,
@@ -337,6 +347,22 @@ def run_command(
return result, meta return result, meta
def terminate_process_group(process: subprocess.Popen[str], grace_seconds: float = 2.0) -> None:
"""Terminate a command and any children it spawned."""
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
process.wait(timeout=grace_seconds)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
except ProcessLookupError:
pass
def parse_ping_output(text: str, sent_fallback: int) -> dict[str, Any]: def parse_ping_output(text: str, sent_fallback: int) -> dict[str, Any]:
rtts: list[float] = [] rtts: list[float] = []
for line in text.splitlines(): for line in text.splitlines():
@@ -1031,6 +1057,12 @@ def parse_csv_strings(value: str) -> list[str]:
return items return items
def cooldown(seconds: float) -> None:
if seconds <= 0:
return
time.sleep(seconds)
def build_parser() -> argparse.ArgumentParser: def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Measure latency, jitter, throughput, loss, CPU, and interface counters.", description="Measure latency, jitter, throughput, loss, CPU, and interface counters.",
@@ -1066,6 +1098,12 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--mtr-count", type=int, default=100) parser.add_argument("--mtr-count", type=int, default=100)
parser.add_argument("--no-archive", action="store_true") parser.add_argument("--no-archive", action="store_true")
parser.add_argument("--command-timeout-extra", type=float, default=15.0) parser.add_argument("--command-timeout-extra", type=float, default=15.0)
parser.add_argument(
"--cooldown-seconds",
type=float,
default=0.0,
help="Sleep after each test so local queues, CPU, and the backend can drain before the next test.",
)
return parser return parser
@@ -1083,6 +1121,8 @@ def main() -> int:
parser.error("--flent-duration must be >= 1") parser.error("--flent-duration must be >= 1")
if args.sockperf_duration < 1: if args.sockperf_duration < 1:
parser.error("--sockperf-duration must be >= 1") parser.error("--sockperf-duration must be >= 1")
if args.cooldown_seconds < 0:
parser.error("--cooldown-seconds must be >= 0")
for protocol in args.sockperf_protocols: for protocol in args.sockperf_protocols:
if protocol not in {"tcp", "udp"}: if protocol not in {"tcp", "udp"}:
parser.error("--sockperf-protocols may only contain tcp and/or udp") parser.error("--sockperf-protocols may only contain tcp and/or udp")
@@ -1095,14 +1135,23 @@ def main() -> int:
tests: list[dict[str, Any]] = [] tests: list[dict[str, Any]] = []
tests.extend(run_ping_tests(args, out_dir)) tests.extend(run_ping_tests(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_arping(args, out_dir)) tests.append(run_arping(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_iperf_tcp(args, out_dir)) tests.append(run_iperf_tcp(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_iperf_tcp_reverse(args, out_dir)) tests.append(run_iperf_tcp_reverse(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_iperf_udp(args, out_dir)) tests.append(run_iperf_udp(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_iperf_udp_reverse(args, out_dir)) tests.append(run_iperf_udp_reverse(args, out_dir))
cooldown(args.cooldown_seconds)
tests.extend(run_sockperf(args, out_dir)) tests.extend(run_sockperf(args, out_dir))
cooldown(args.cooldown_seconds)
tests.extend(run_flent(args, out_dir)) tests.extend(run_flent(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_traceroute(args, out_dir)) tests.append(run_traceroute(args, out_dir))
cooldown(args.cooldown_seconds)
tests.append(run_mtr(args, out_dir)) tests.append(run_mtr(args, out_dir))
summary = { summary = {