marcus/mitm-webserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tc test
All checks were successful
Build and Deploy MITM Webserver / build (push) Successful in 10s
Details

Browse Source
This commit is contained in:
malmert
2026-03-07 16:36:24 +01:00
parent f55e899fc5
commit 0c69daf229
13 changed files with 459 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files

494
backend/src/utilities/ebpf_bridge_events.py
View File

@@ -1,19 +1,24 @@
#!/usr/bin/env python3
"""Emit bridge ingress, egress, and drop telemetry events via eBPF."""
"""Emit bridge ingress raw packets plus egress/drop telemetry via tc/eBPF."""
from __future__ import annotations
import argparse
import base64
import ctypes as ct
import hashlib
import ipaddress
import json
import os
import signal
import shutil
import socket
import subprocess
import sys
from pathlib import Path
from typing import Iterable
from pyroute2 import IPRoute
try:
from bcc import BPF # type: ignore
except Exception as exc: # pragma: no cover - depends on host runtime
@@ -23,6 +28,15 @@ except Exception as exc: # pragma: no cover - depends on host runtime
from src.utilities.packet_mark import packet_id_from_mark, verdict_from_mark
EVENT_INGRESS = 1
EVENT_EGRESS = 2
EVENT_DROP = 3
INGRESS_PARENT = "ffff:fff2"
EGRESS_PARENT = "ffff:fff3"
MARK_FILTER_PREF = 10
INGRESS_FILTER_HANDLE = ":20"
EGRESS_FILTER_HANDLE = ":30"
IDENTITY_FIELDS = (
"src_mac",
"dst_mac",
@@ -38,9 +52,10 @@ IDENTITY_FIELDS = (
BPF_SOURCE = r"""
#include <uapi/linux/ptrace.h>
#include <uapi/linux/pkt_cls.h>
#include <linux/bpf.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
@@ -48,7 +63,6 @@ BPF_SOURCE = r"""
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_arp.h>
#include <linux/version.h>
#define EVENT_INGRESS 1
#define EVENT_EGRESS 2
@@ -71,6 +85,7 @@ struct event_t {
__u32 skb_mark;
__u32 length;
__u32 reason;
__u32 ifindex;
__u16 eth_type_raw;
__u16 vlan_id;
__u16 src_port;
@@ -78,31 +93,119 @@ struct event_t {
__u32 protocol_raw;
__u8 event_type;
__u8 ip_version;
char ifname[IFNAMSIZ];
__u8 reserved[2];
unsigned char src_mac[6];
unsigned char dst_mac[6];
unsigned char src_ip[16];
unsigned char dst_ip[16];
};
BPF_PERF_OUTPUT(events);
BPF_PERF_OUTPUT(ingress_events);
BPF_PERF_OUTPUT(meta_events);
static __always_inline int fill_ifname(struct sk_buff *skb, struct event_t *event) {
if (!skb) {
static __always_inline int parse_l3_l4_direct(struct event_t *event, void *data, void *data_end) {
struct ethhdr *eth = data;
__be16 eth_proto;
void *l3;
if ((void *)(eth + 1) > data_end) {
return 0;
}
struct net_device *dev = NULL;
bpf_probe_read_kernel(&dev, sizeof(dev), &skb->dev);
if (!dev) {
return 0;
__builtin_memcpy(event->src_mac, eth->h_source, ETH_ALEN);
__builtin_memcpy(event->dst_mac, eth->h_dest, ETH_ALEN);
eth_proto = eth->h_proto;
l3 = eth + 1;
if (eth_proto == htons(ETH_P_8021Q) || eth_proto == htons(ETH_P_8021AD)) {
struct vlan_hdr_t *vlan = l3;
if ((void *)(vlan + 1) > data_end) {
return 0;
}
event->vlan_id = ntohs(vlan->h_vlan_TCI) & 0x0fff;
eth_proto = vlan->h_vlan_encapsulated_proto;
l3 = vlan + 1;
}
event->eth_type_raw = ntohs(eth_proto);
if (eth_proto == htons(ETH_P_ARP)) {
struct arphdr *arph = l3;
struct arp_eth_ipv4_t *body = (void *)(arph + 1);
if ((void *)(body + 1) > data_end) {
return 1;
}
if (arph->ar_hrd == htons(ARPHRD_ETHER) && arph->ar_pro == htons(ETH_P_IP) &&
arph->ar_hln == ETH_ALEN && arph->ar_pln == 4) {
__builtin_memcpy(event->src_ip, body->spa, 4);
__builtin_memcpy(event->dst_ip, body->tpa, 4);
event->ip_version = 4;
}
return 1;
}
if (eth_proto == htons(ETH_P_IP)) {
struct iphdr *iph = l3;
if ((void *)(iph + 1) > data_end) {
return 0;
}
event->ip_version = 4;
event->protocol_raw = iph->protocol;
__builtin_memcpy(event->src_ip, &iph->saddr, 4);
__builtin_memcpy(event->dst_ip, &iph->daddr, 4);
if (iph->protocol == IPPROTO_TCP) {
struct tcphdr *tcph = (void *)iph + (iph->ihl * 4);
if ((void *)(tcph + 1) > data_end) {
return 1;
}
event->src_port = ntohs(tcph->source);
event->dst_port = ntohs(tcph->dest);
} else if (iph->protocol == IPPROTO_UDP) {
struct udphdr *udph = (void *)iph + (iph->ihl * 4);
if ((void *)(udph + 1) > data_end) {
return 1;
}
event->src_port = ntohs(udph->source);
event->dst_port = ntohs(udph->dest);
}
return 1;
}
if (eth_proto == htons(ETH_P_IPV6)) {
struct ipv6hdr *ip6h = l3;
if ((void *)(ip6h + 1) > data_end) {
return 0;
}
event->ip_version = 6;
event->protocol_raw = ip6h->nexthdr;
__builtin_memcpy(event->src_ip, &ip6h->saddr, 16);
__builtin_memcpy(event->dst_ip, &ip6h->daddr, 16);
if (ip6h->nexthdr == IPPROTO_TCP) {
struct tcphdr *tcph = ip6h + 1;
if ((void *)(tcph + 1) > data_end) {
return 1;
}
event->src_port = ntohs(tcph->source);
event->dst_port = ntohs(tcph->dest);
} else if (ip6h->nexthdr == IPPROTO_UDP) {
struct udphdr *udph = ip6h + 1;
if ((void *)(udph + 1) > data_end) {
return 1;
}
event->src_port = ntohs(udph->source);
event->dst_port = ntohs(udph->dest);
}
return 1;
}
bpf_probe_read_kernel(event->ifname, sizeof(event->ifname), dev->name);
return 1;
}
static __always_inline int parse_skb(struct sk_buff *skb, struct event_t *event) {
static __always_inline int parse_skb_linear(struct event_t *event, struct sk_buff *skb) {
unsigned char *head = NULL;
__u16 mac_header = 0;
__u16 network_header = 0;
@@ -126,8 +229,8 @@ static __always_inline int parse_skb(struct sk_buff *skb, struct event_t *event)
struct ethhdr eth = {};
unsigned char *eth_ptr = head + mac_header;
bpf_probe_read_kernel(&eth, sizeof(eth), eth_ptr);
__builtin_memcpy(event->src_mac, eth.h_source, 6);
__builtin_memcpy(event->dst_mac, eth.h_dest, 6);
__builtin_memcpy(event->src_mac, eth.h_source, ETH_ALEN);
__builtin_memcpy(event->dst_mac, eth.h_dest, ETH_ALEN);
__be16 eth_proto = eth.h_proto;
unsigned char *l3_ptr = head + network_header;
@@ -136,10 +239,8 @@ static __always_inline int parse_skb(struct sk_buff *skb, struct event_t *event)
bpf_probe_read_kernel(&vlan, sizeof(vlan), eth_ptr + sizeof(struct ethhdr));
event->vlan_id = ntohs(vlan.h_vlan_TCI) & 0x0fff;
eth_proto = vlan.h_vlan_encapsulated_proto;
event->eth_type_raw = ntohs(eth_proto);
} else {
event->eth_type_raw = ntohs(eth_proto);
}
event->eth_type_raw = ntohs(eth_proto);
if (eth_proto == htons(ETH_P_ARP)) {
struct arphdr arph = {};
@@ -202,64 +303,76 @@ static __always_inline int parse_skb(struct sk_buff *skb, struct event_t *event)
return 1;
}
static __always_inline int emit_event(struct pt_regs *ctx, struct sk_buff *skb, __u8 event_type, __u32 reason) {
int handle_ingress(struct __sk_buff *skb) {
struct event_t event = {};
void *data = (void *)(long)skb->data;
void *data_end = (void *)(long)skb->data_end;
event.ts_ns = bpf_ktime_get_ns();
event.event_type = event_type;
event.reason = reason;
event.event_type = EVENT_INGRESS;
event.ifindex = skb->ifindex;
event.length = skb->len;
event.skb_mark = skb->mark;
if (!fill_ifname(skb, &event)) {
return 0;
if (!event.skb_mark) {
return TC_ACT_OK;
}
if (!parse_skb(skb, &event)) {
return 0;
if (!parse_l3_l4_direct(&event, data, data_end)) {
return TC_ACT_OK;
}
events.perf_submit(ctx, &event, sizeof(event));
return 0;
ingress_events.perf_submit_skb(skb, skb->len, &event, sizeof(event));
return TC_ACT_OK;
}
int trace_ingress(struct pt_regs *ctx, struct sk_buff *skb) {
return emit_event(ctx, skb, EVENT_INGRESS, 0);
}
int handle_egress(struct __sk_buff *skb) {
struct event_t event = {};
void *data = (void *)(long)skb->data;
void *data_end = (void *)(long)skb->data_end;
int trace_egress(struct pt_regs *ctx, struct sk_buff *skb) {
return emit_event(ctx, skb, EVENT_EGRESS, 0);
event.ts_ns = bpf_ktime_get_ns();
event.event_type = EVENT_EGRESS;
event.ifindex = skb->ifindex;
event.length = skb->len;
event.skb_mark = skb->mark;
if (!event.skb_mark) {
return TC_ACT_OK;
}
if (!parse_l3_l4_direct(&event, data, data_end)) {
return TC_ACT_OK;
}
meta_events.perf_submit(skb, &event, sizeof(event));
return TC_ACT_OK;
}
TRACEPOINT_PROBE(skb, kfree_skb) {
struct sk_buff *skb = (struct sk_buff *)args->skbaddr;
struct event_t event = {};
struct net_device *dev = NULL;
event.ts_ns = bpf_ktime_get_ns();
event.event_type = EVENT_DROP;
event.reason = args->reason;
if (!fill_ifname(skb, &event)) {
return 0;
}
if (!parse_skb(skb, &event)) {
if (!skb) {
return 0;
}
events.perf_submit(args, &event, sizeof(event));
return 0;
}
TRACEPOINT_PROBE(net, net_dev_queue) {
struct sk_buff *skb = (struct sk_buff *)args->skbaddr;
struct event_t event = {};
event.ts_ns = bpf_ktime_get_ns();
event.event_type = EVENT_EGRESS;
event.reason = 0;
if (!fill_ifname(skb, &event)) {
bpf_probe_read_kernel(&dev, sizeof(dev), &skb->dev);
if (!dev) {
return 0;
}
if (!parse_skb(skb, &event)) {
bpf_probe_read_kernel(&event.ifindex, sizeof(event.ifindex), &dev->ifindex);
if (!parse_skb_linear(&event, skb)) {
return 0;
}
if (!event.skb_mark) {
return 0;
}
events.perf_submit(args, &event, sizeof(event));
meta_events.perf_submit(args, &event, sizeof(event));
return 0;
}
"""
@@ -271,6 +384,7 @@ class Event(ct.Structure):
("skb_mark", ct.c_uint),
("length", ct.c_uint),
("reason", ct.c_uint),
("ifindex", ct.c_uint),
("eth_type_raw", ct.c_ushort),
("vlan_id", ct.c_ushort),
("src_port", ct.c_ushort),
@@ -278,7 +392,7 @@ class Event(ct.Structure):
("protocol_raw", ct.c_uint),
("event_type", ct.c_ubyte),
("ip_version", ct.c_ubyte),
("ifname", ct.c_char * 16),
("reserved", ct.c_ubyte * 2),
("src_mac", ct.c_ubyte * 6),
("dst_mac", ct.c_ubyte * 6),
("src_ip", ct.c_ubyte * 16),
@@ -286,6 +400,51 @@ class Event(ct.Structure):
]
TARGET_INTERFACES: set[str] = set()
IPR: IPRoute | None = None
def _run_checked(cmd: list[str]) -> None:
subprocess.run(cmd, check=True, capture_output=True, text=True)
def _multiarch_include_flag() -> list[str]:
if shutil.which("gcc") is None:
return []
try:
triple = subprocess.run(["gcc", "-dumpmachine"], check=True, capture_output=True, text=True).stdout.strip()
except Exception:
return []
include_dir = Path("/usr/include") / triple
return [f"-I{include_dir}"] if include_dir.is_dir() else []
def _repo_root() -> Path:
return Path(__file__).resolve().parents[3]
def _ensure_mark_object(build_dir: str) -> Path:
build_path = Path(build_dir)
build_path.mkdir(parents=True, exist_ok=True)
src = _repo_root() / "tools" / "ebpf" / "mark_packet_id.c"
obj = build_path / "mark_packet_id.o"
if obj.exists() and obj.stat().st_mtime >= src.stat().st_mtime:
return obj
cmd = ["clang", "-O2", "-g", "-target", "bpf", *_multiarch_include_flag(), "-c", str(src), "-o", str(obj)]
_run_checked(cmd)
return obj
def _ifname_from_index(ifindex: int) -> str | None:
if ifindex <= 0:
return None
try:
return socket.if_indextoname(ifindex)
except OSError:
return None
def _mac_to_str(value: Iterable[int]) -> str:
return ":".join(f"{byte:02x}" for byte in value)
@@ -314,22 +473,20 @@ def _build_packet_uid(payload: dict[str, object]) -> str:
def _event_name(value: int) -> str:
return {1: "ingress", 2: "egress", 3: "drop"}.get(value, "unknown")
return {EVENT_INGRESS: "ingress", EVENT_EGRESS: "egress", EVENT_DROP: "drop"}.get(value, "unknown")
def _reason_name(reason: int) -> str:
return f"skb_drop_reason_{reason}"
def _emit_event(cpu: int, data: int, size: int) -> None:
del cpu, size
event = ct.cast(data, ct.POINTER(Event)).contents
iface = bytes(event.ifname).split(b"\x00", 1)[0].decode("utf-8", "replace")
def _build_payload(event: Event) -> dict[str, object] | None:
iface = _ifname_from_index(int(event.ifindex))
if iface not in TARGET_INTERFACES:
return
return None
payload: dict[str, object] = {
"event_type": _event_name(event.event_type),
"event_type": _event_name(int(event.event_type)),
"iface": iface,
"skb_mark": int(event.skb_mark) or None,
"length": int(event.length),
@@ -342,9 +499,10 @@ def _emit_event(cpu: int, data: int, size: int) -> None:
"protocol_raw": int(event.protocol_raw) or None,
"src_port": int(event.src_port) or None,
"dst_port": int(event.dst_port) or None,
"reason": _reason_name(int(event.reason)) if event.event_type == 3 else None,
"reason_code": int(event.reason) if event.event_type == 3 else None,
"reason": _reason_name(int(event.reason)) if int(event.event_type) == EVENT_DROP else None,
"reason_code": int(event.reason) if int(event.event_type) == EVENT_DROP else None,
}
packet_id = packet_id_from_mark(payload.get("skb_mark"))
if packet_id:
payload["packet_id"] = packet_id
@@ -357,67 +515,37 @@ def _emit_event(cpu: int, data: int, size: int) -> None:
payload["packet_uid"] = _build_packet_uid(payload)
payload["correlation_key"] = f"uid:{payload['packet_uid']}"
payload["correlation_source"] = "legacy_hash"
return payload
def _emit_ingress_event(cpu: int, data: int, size: int) -> None:
del cpu
event = ct.cast(data, ct.POINTER(Event)).contents
payload = _build_payload(event)
if payload is None:
return
raw_size = size - ct.sizeof(Event)
if raw_size > 0:
raw = ct.string_at(data + ct.sizeof(Event), raw_size)
payload["raw_b64"] = base64.b64encode(raw).decode("ascii")
print(json.dumps(payload, separators=(",", ":")), flush=True)
def _attach_kprobe_first(bpf: BPF, symbols: list[str], fn_name: str) -> str:
supported = _supported_kprobe_symbols(symbols)
if not supported:
raise RuntimeError(f"No supported kprobe symbols found for {fn_name}: {symbols}")
for symbol in supported:
try:
bpf.attach_kprobe(event=symbol, fn_name=fn_name)
return symbol
except Exception:
continue
raise RuntimeError(f"Failed to attach {fn_name} to any of {supported}")
def _supported_kprobe_symbols(symbols: list[str]) -> list[str]:
try:
available = set()
for symbol in symbols:
for candidate in BPF.get_kprobe_functions(symbol.encode()):
decoded = candidate.decode("utf-8", "replace")
if decoded == symbol:
available.add(symbol)
if available:
return [symbol for symbol in symbols if symbol in available]
except Exception:
pass
if os.path.exists("/proc/kallsyms"):
try:
with open("/proc/kallsyms", "r", encoding="utf-8", errors="replace") as handle:
names = {line.rsplit(" ", 1)[-1].strip() for line in handle}
return [symbol for symbol in symbols if symbol in names]
except Exception:
pass
return symbols
def _attach_egress_probe(bpf: BPF) -> str:
symbols = ["__dev_queue_xmit", "dev_queue_xmit"]
try:
return _attach_kprobe_first(bpf, symbols, "trace_egress")
except Exception:
pass
try:
bpf.attach_tracepoint(tp="net:net_dev_queue", fn_name="tracepoint__net__net_dev_queue")
return "tracepoint:net:net_dev_queue"
except Exception as exc:
raise RuntimeError(
"Failed to attach egress telemetry to any of "
f"{symbols} or tracepoint net:net_dev_queue"
) from exc
def _emit_meta_event(cpu: int, data: int, size: int) -> None:
del cpu, size
event = ct.cast(data, ct.POINTER(Event)).contents
payload = _build_payload(event)
if payload is None:
return
print(json.dumps(payload, separators=(",", ":")), flush=True)
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="eBPF bridge telemetry collector")
parser.add_argument("--ifaces", required=True, help="Comma-separated list of interfaces to keep")
parser = argparse.ArgumentParser(description="tc/eBPF bridge telemetry collector")
parser.add_argument("--ifaces", required=True, help="Comma-separated list of interfaces to instrument")
parser.add_argument("--build-dir", required=True, help="Directory for compiled tc BPF objects")
return parser.parse_args()
@@ -425,47 +553,131 @@ def _sigterm(_signum: int, _frame: object) -> None:
raise KeyboardInterrupt
def _ensure_clean_clsact(iface: str) -> None:
subprocess.run(["tc", "qdisc", "del", "dev", iface, "clsact"], check=False, capture_output=True, text=True)
_run_checked(["tc", "qdisc", "add", "dev", iface, "clsact"])
def _attach_mark_filter(iface: str, obj_path: Path) -> None:
_run_checked(
[
"tc",
"filter",
"replace",
"dev",
iface,
"ingress",
"pref",
str(MARK_FILTER_PREF),
"protocol",
"all",
"bpf",
"direct-action",
"obj",
str(obj_path),
"sec",
"classifier",
]
)
def _attach_tc_programs(bpf: BPF, ifaces: list[str], build_dir: str) -> tuple[str, str]:
global IPR
obj_path = _ensure_mark_object(build_dir)
ingress_fn = bpf.load_func("handle_ingress", BPF.SCHED_CLS)
egress_fn = bpf.load_func("handle_egress", BPF.SCHED_CLS)
ipr = IPRoute()
try:
for iface in ifaces:
matches = ipr.link_lookup(ifname=iface)
if not matches:
raise RuntimeError(f"Interface not found: {iface}")
ifindex = matches[0]
_ensure_clean_clsact(iface)
_attach_mark_filter(iface, obj_path)
ipr.tc(
"add-filter",
"bpf",
ifindex,
INGRESS_FILTER_HANDLE,
fd=ingress_fn.fd,
name=ingress_fn.name,
parent=INGRESS_PARENT,
classid=1,
direct_action=True,
)
ipr.tc(
"add-filter",
"bpf",
ifindex,
EGRESS_FILTER_HANDLE,
fd=egress_fn.fd,
name=egress_fn.name,
parent=EGRESS_PARENT,
classid=1,
direct_action=True,
)
except Exception:
for iface in ifaces:
subprocess.run(["tc", "qdisc", "del", "dev", iface, "clsact"], check=False, capture_output=True, text=True)
if ipr is not None:
ipr.close()
IPR = None
raise
return ingress_fn.name, egress_fn.name
def _cleanup_tc(ifaces: Iterable[str]) -> None:
for iface in ifaces:
subprocess.run(["tc", "qdisc", "del", "dev", iface, "clsact"], check=False, capture_output=True, text=True)
global IPR
if IPR is not None:
try:
IPR.close()
finally:
IPR = None
def main() -> int:
args = _parse_args()
global TARGET_INTERFACES
TARGET_INTERFACES = {iface.strip() for iface in args.ifaces.split(",") if iface.strip()}
if not TARGET_INTERFACES:
print("No interfaces provided", file=sys.stderr)
print("No interfaces provided", file=sys.stderr, flush=True)
return 1
signal.signal(signal.SIGTERM, _sigterm)
signal.signal(signal.SIGINT, _sigterm)
bpf = BPF(text=BPF_SOURCE)
ingress_symbol = _attach_kprobe_first(
bpf,
["__netif_receive_skb_core", "netif_receive_skb", "__netif_receive_skb_one_core"],
"trace_ingress",
)
egress_symbol = _attach_egress_probe(bpf)
print(
json.dumps(
{
"status": "collector_started",
"ifaces": sorted(TARGET_INTERFACES),
"ingress_symbol": ingress_symbol,
"egress_symbol": egress_symbol,
},
separators=(",", ":"),
),
flush=True,
)
bpf["events"].open_perf_buffer(_emit_event, page_cnt=128)
ingress_prog_name = ""
egress_prog_name = ""
try:
ingress_prog_name, egress_prog_name = _attach_tc_programs(bpf, sorted(TARGET_INTERFACES), args.build_dir)
print(
json.dumps(
{
"status": "collector_started",
"ifaces": sorted(TARGET_INTERFACES),
"ingress_program": ingress_prog_name,
"egress_program": egress_prog_name,
"build_dir": args.build_dir,
},
separators=(",", ":"),
),
flush=True,
)
bpf["ingress_events"].open_perf_buffer(_emit_ingress_event, page_cnt=256)
bpf["meta_events"].open_perf_buffer(_emit_meta_event, page_cnt=128)
while True:
bpf.perf_buffer_poll()
except KeyboardInterrupt:
return 0
TARGET_INTERFACES: set[str] = set()
finally:
_cleanup_tc(sorted(TARGET_INTERFACES))
if __name__ == "__main__":