try overload fix 2, backend upsert and test script fix

tools/network_timing_benchmark.py

@@ -24,6 +24,7 @@ import os
 import platform
 import re
 import shutil
+import signal
 import socket
 import statistics
 import subprocess
@@ -264,33 +265,42 @@ def run_command(
     start_monotonic = time.perf_counter()
     start_wall = now_iso()
     sampler.start()
+    process: subprocess.Popen[str] | None = None
     try:
-        completed = subprocess.run(
+        process = subprocess.Popen(
             command,
             cwd=str(cwd),
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
-            timeout=timeout,
-            check=False,
+            start_new_session=True,
         )
+        stdout, stderr = process.communicate(timeout=timeout)
         result = CommandRun(
             args=command,
-            returncode=completed.returncode,
-            stdout=completed.stdout or "",
-            stderr=completed.stderr or "",
+            returncode=process.returncode,
+            stdout=stdout or "",
+            stderr=stderr or "",
         )
     except subprocess.TimeoutExpired as exc:
-        stdout = exc.stdout.decode("utf-8", errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
-        stderr = exc.stderr.decode("utf-8", errors="replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
+        if process is not None:
+            terminate_process_group(process)
+            stdout, stderr = process.communicate()
+        else:
+            stdout = exc.stdout.decode("utf-8", errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
+            stderr = exc.stderr.decode("utf-8", errors="replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
         result = CommandRun(
             args=command,
-            returncode=None,
+            returncode=process.returncode if process is not None else None,
             stdout=stdout,
             stderr=stderr,
             timed_out=True,
             error=f"command timed out after {timeout} seconds",
         )
+    except KeyboardInterrupt:
+        if process is not None:
+            terminate_process_group(process)
+        raise
     except OSError as exc:
         result = CommandRun(
             args=command,
@@ -337,6 +347,22 @@ def run_command(
     return result, meta
 
 
+def terminate_process_group(process: subprocess.Popen[str], grace_seconds: float = 2.0) -> None:
+    """Terminate a command and any children it spawned."""
+    if process.poll() is not None:
+        return
+    try:
+        os.killpg(process.pid, signal.SIGTERM)
+        process.wait(timeout=grace_seconds)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(process.pid, signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+    except ProcessLookupError:
+        pass
+
+
 def parse_ping_output(text: str, sent_fallback: int) -> dict[str, Any]:
     rtts: list[float] = []
     for line in text.splitlines():
@@ -1031,6 +1057,12 @@ def parse_csv_strings(value: str) -> list[str]:
     return items
 
 
+def cooldown(seconds: float) -> None:
+    if seconds <= 0:
+        return
+    time.sleep(seconds)
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Measure latency, jitter, throughput, loss, CPU, and interface counters.",
@@ -1066,6 +1098,12 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--mtr-count", type=int, default=100)
     parser.add_argument("--no-archive", action="store_true")
     parser.add_argument("--command-timeout-extra", type=float, default=15.0)
+    parser.add_argument(
+        "--cooldown-seconds",
+        type=float,
+        default=0.0,
+        help="Sleep after each test so local queues, CPU, and the backend can drain before the next test.",
+    )
     return parser
 
 
@@ -1083,6 +1121,8 @@ def main() -> int:
         parser.error("--flent-duration must be >= 1")
     if args.sockperf_duration < 1:
         parser.error("--sockperf-duration must be >= 1")
+    if args.cooldown_seconds < 0:
+        parser.error("--cooldown-seconds must be >= 0")
     for protocol in args.sockperf_protocols:
         if protocol not in {"tcp", "udp"}:
             parser.error("--sockperf-protocols may only contain tcp and/or udp")
@@ -1095,14 +1135,23 @@ def main() -> int:
 
     tests: list[dict[str, Any]] = []
     tests.extend(run_ping_tests(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_arping(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_iperf_tcp(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_iperf_tcp_reverse(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_iperf_udp(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_iperf_udp_reverse(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.extend(run_sockperf(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.extend(run_flent(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_traceroute(args, out_dir))
+    cooldown(args.cooldown_seconds)
     tests.append(run_mtr(args, out_dir))
 
     summary = {