diff --git a/tools/network_timing_benchmark.py b/tools/network_timing_benchmark.py index 6a2904d..38e0b74 100755 --- a/tools/network_timing_benchmark.py +++ b/tools/network_timing_benchmark.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Measure latency, jitter, throughput, packet loss, CPU, and interface counters. -The script is intended for repeated thesis experiments such as: +The script is intended for thesis experiments such as: direct link bridge only @@ -90,18 +90,30 @@ def numeric_stats(values: list[float]) -> dict[str, Any]: "mean": None, "median": None, "stdev": None, + "mad": None, + "iqr": None, + "cv_percent": None, "p90": None, "p95": None, "p99": None, } + median = statistics.median(values) + q1 = percentile(values, 0.25) + q3 = percentile(values, 0.75) + mean = statistics.fmean(values) + stdev = statistics.stdev(values) if len(values) >= 2 else 0.0 + return { "count": len(values), "min": min(values), "max": max(values), - "mean": statistics.fmean(values), - "median": statistics.median(values), - "stdev": statistics.stdev(values) if len(values) >= 2 else 0.0, + "mean": mean, + "median": median, + "stdev": stdev, + "mad": statistics.median([abs(value - median) for value in values]), + "iqr": (q3 - q1) if q1 is not None and q3 is not None else None, + "cv_percent": (100.0 * stdev / mean) if mean else None, "p90": percentile(values, 0.90), "p95": percentile(values, 0.95), "p99": percentile(values, 0.99), @@ -1040,6 +1052,52 @@ def write_summary_md(summary: dict[str, Any], out_dir: Path) -> None: (out_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8") +def run_test_suite(args: argparse.Namespace, out_dir: Path) -> list[dict[str, Any]]: + tests: list[dict[str, Any]] = [] + tests.extend(run_ping_tests(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_arping(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_iperf_tcp(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_iperf_tcp_reverse(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_iperf_udp(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_iperf_udp_reverse(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.extend(run_sockperf(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.extend(run_flent(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_traceroute(args, out_dir)) + cooldown(args.cooldown_seconds) + tests.append(run_mtr(args, out_dir)) + return tests + + +def write_run_outputs( + args: argparse.Namespace, + environment: dict[str, Any], + tests: list[dict[str, Any]], + out_dir: Path, +) -> dict[str, Any]: + summary = { + "label": args.label, + "target": args.target, + "interface": args.interface, + "started_at": now_iso(), + "arguments": vars(args), + "environment": environment, + "tests": tests, + } + + (out_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + write_ping_csv([test for test in tests if test.get("type") == "ping"], out_dir) + write_summary_md(summary, out_dir) + return summary + + def parse_payload_sizes(value: str) -> list[int]: try: sizes = [int(item.strip()) for item in value.split(",") if item.strip()] @@ -1071,37 +1129,53 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--interface", "-i", help="Interface used for source binding/counters.") parser.add_argument("--label", default="experiment", help="Experiment label, e.g. bridge-ebpf.") parser.add_argument("--out-dir", default="measurements", help="Base output directory.") + parser.add_argument( + "--packet-count", + type=int, + help=( + "Convenience packet/probe count for packet-count based tests. " + "Used for ping, arping, and mtr unless their specific count option is set." + ), + ) + parser.add_argument( + "--duration", + type=int, + help=( + "Convenience duration in seconds for duration-based tests. " + "Used for iperf3, sockperf, and flent unless their specific duration option is set." + ), + ) parser.add_argument("--payload-sizes", type=parse_payload_sizes, default=[56, 512, 1472]) - parser.add_argument("--ping-count", type=int, default=1000) - parser.add_argument("--ping-interval", type=float, default=0.02) + parser.add_argument("--ping-count", type=int, default=5000) + parser.add_argument("--ping-interval", type=float, default=0.005) parser.add_argument("--ping-timeout-extra", type=float, default=20.0) parser.add_argument("--skip-ping", action="store_true") parser.add_argument("--skip-arping", action="store_true") - parser.add_argument("--arping-count", type=int, default=100) + parser.add_argument("--arping-count", type=int) parser.add_argument("--skip-iperf", action="store_true") parser.add_argument("--skip-udp-iperf", action="store_true") parser.add_argument("--skip-iperf-reverse", action="store_true") parser.add_argument("--iperf-tcp-port", type=int, default=5201) parser.add_argument("--iperf-udp-port", type=int, default=5202) - parser.add_argument("--iperf-duration", type=int, default=20) + parser.add_argument("--iperf-duration", type=int) parser.add_argument("--iperf-parallel", type=int, default=1) - parser.add_argument("--udp-bandwidth", default="100M") + parser.add_argument("--udp-bandwidth", default="900M") parser.add_argument("--skip-flent", action="store_true") parser.add_argument("--flent-tests", type=parse_csv_strings, default=["rrul", "tcp_upload", "tcp_download"]) - parser.add_argument("--flent-duration", type=int, default=60) + parser.add_argument("--flent-duration", type=int) parser.add_argument("--skip-sockperf", action="store_true") parser.add_argument("--sockperf-protocols", type=parse_csv_strings, default=["udp", "tcp"]) parser.add_argument("--sockperf-tcp-port", type=int, default=11111) parser.add_argument("--sockperf-udp-port", type=int, default=11112) - parser.add_argument("--sockperf-duration", type=int, default=30) + parser.add_argument("--sockperf-duration", type=int) parser.add_argument("--skip-path-tools", action="store_true") - parser.add_argument("--mtr-count", type=int, default=100) + parser.add_argument("--mtr-count", type=int) parser.add_argument("--no-archive", action="store_true") parser.add_argument("--command-timeout-extra", type=float, default=15.0) parser.add_argument( "--cooldown-seconds", type=float, - default=0.0, + default=10.0, help="Sleep after each test so local queues, CPU, and the backend can drain before the next test.", ) return parser @@ -1111,6 +1185,24 @@ def main() -> int: parser = build_parser() args = parser.parse_args() + if args.packet_count is not None and args.packet_count < 1: + parser.error("--packet-count must be >= 1") + if args.duration is not None and args.duration < 1: + parser.error("--duration must be >= 1") + + if args.ping_count is None: + args.ping_count = args.packet_count if args.packet_count is not None else 1000 + if args.arping_count is None: + args.arping_count = args.packet_count if args.packet_count is not None else 100 + if args.mtr_count is None: + args.mtr_count = args.packet_count if args.packet_count is not None else 100 + if args.iperf_duration is None: + args.iperf_duration = args.duration if args.duration is not None else 20 + if args.flent_duration is None: + args.flent_duration = args.duration if args.duration is not None else 60 + if args.sockperf_duration is None: + args.sockperf_duration = args.duration if args.duration is not None else 30 + if args.ping_count < 1: parser.error("--ping-count must be >= 1") if args.ping_interval <= 0: @@ -1133,40 +1225,8 @@ def main() -> int: environment = collect_environment(args, out_dir) - tests: list[dict[str, Any]] = [] - tests.extend(run_ping_tests(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_arping(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_iperf_tcp(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_iperf_tcp_reverse(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_iperf_udp(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_iperf_udp_reverse(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.extend(run_sockperf(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.extend(run_flent(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_traceroute(args, out_dir)) - cooldown(args.cooldown_seconds) - tests.append(run_mtr(args, out_dir)) - - summary = { - "label": args.label, - "target": args.target, - "interface": args.interface, - "started_at": now_iso(), - "arguments": vars(args), - "environment": environment, - "tests": tests, - } - - (out_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - write_ping_csv([test for test in tests if test.get("type") == "ping"], out_dir) - write_summary_md(summary, out_dir) + tests = run_test_suite(args, out_dir) + write_run_outputs(args, environment, tests, out_dir) archive_path = None if not args.no_archive: