tc full packet test
All checks were successful
Build and Deploy MITM Webserver / build (push) Successful in 10s

This commit is contained in:
2026-03-07 15:13:08 +01:00
parent 2bdbe230d1
commit a282b89d45
17 changed files with 533 additions and 52 deletions

View File

@@ -15,6 +15,8 @@ BACKEND_SNIFFER_SELECTOR_TIMEOUT_SECONDS=1.0
BACKEND_SNIFFER_RECV_BYTES=65536
BACKEND_SNIFFER_BUFFER_DRAIN_INTERVAL_SECONDS=5.0
BACKEND_SNIFFER_THREAD_JOIN_TIMEOUT_SECONDS=2.0
BACKEND_CAPTURE_INTERFACE=
BACKEND_CAPTURE_HEADER_ENABLED=true
BACKEND_TELEMETRY_PROCESS_STOP_TIMEOUT_SECONDS=3.0
BACKEND_TELEMETRY_READER_JOIN_TIMEOUT_SECONDS=2.0
BACKEND_NDPI_MAX_FLOWS=200000

View File

@@ -11,9 +11,15 @@ class PacketDBModel(BaseModel):
id: Union[int, str]
timestamp: datetime = Field(..., description="Packet timestamp in ISO format.")
packet_uid: str = Field(..., description="Stable packet identity used for upserts/correlation.")
updated_at: Optional[datetime] = Field(None, description="Last DB update time for this row.")
correlation_key: str = Field(..., description="Primary upsert key for this packet row.")
correlation_source: Optional[str] = Field(None, description="How the correlation key was derived.")
packet_id: Optional[str] = Field(None, description="Kernel-side packet identifier derived from skb mark.")
packet_uid: Optional[str] = Field(None, description="Legacy hash-based packet identity fallback.")
skb_mark: Optional[int] = Field(None, description="Raw skb mark observed in telemetry or capture header.")
ingress_if: Optional[str] = None
egress_if: Optional[str] = None
capture_iface: Optional[str] = None
src_mac: Optional[str] = None
dst_mac: Optional[str] = None
eth_type_raw: Optional[int] = Field(None, description="Numeric Ethernet type from the frame header.")
@@ -37,6 +43,7 @@ class PacketDBModel(BaseModel):
app_is_encrypted: Optional[bool] = Field(None, description="Whether detected protocol appears encrypted.")
app_risk_score: Optional[int] = Field(None, description="Count/score of detected nDPI risks.")
dpi_metadata: Optional[dict] = Field(None, description="Raw DPI metadata from nDPI.")
capture_metadata: Optional[dict] = Field(None, description="Capture-side metadata from the mirrored packet shim.")
telemetry_metadata: Optional[dict] = Field(None, description="Kernel telemetry details from eBPF collector.")
verdict: Optional[str] = None
verdict_reason: Optional[str] = None
@@ -51,9 +58,15 @@ class PacketDBModel(BaseModel):
"example": {
"id": 123,
"timestamp": "2026-03-05T12:34:56.789Z",
"updated_at": "2026-03-05T12:34:56.900Z",
"correlation_key": "pid:123456",
"correlation_source": "kernel_mark",
"packet_id": "123456",
"packet_uid": "9f6d3af0d3c81cb20ee8e7d32df7c56414460542",
"skb_mark": 123456,
"ingress_if": "eth0",
"egress_if": "eth1",
"capture_iface": "mitmcap0",
"src_mac": "aa:bb:cc:dd:ee:ff",
"dst_mac": "11:22:33:44:55:66",
"eth_type_raw": 2048,
@@ -77,6 +90,7 @@ class PacketDBModel(BaseModel):
"app_is_encrypted": False,
"app_risk_score": 0,
"dpi_metadata": {"method": "GET"},
"capture_metadata": {"header_magic": "MTCP", "header_version": 1},
"telemetry_metadata": {"event_type": "egress", "iface": "eth1"},
"verdict": "accept",
"verdict_reason": "egress-observed",

View File

@@ -25,6 +25,13 @@ def _env_float(name: str, default: float) -> float:
return float(value)
def _env_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
if value in (None, ""):
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
@dataclass(frozen=True)
class BackendSettings:
db_dsn: str
@@ -44,6 +51,8 @@ class BackendSettings:
sniffer_recv_bytes: int
sniffer_buffer_drain_interval_seconds: float
sniffer_thread_join_timeout_seconds: float
capture_interface: str
capture_header_enabled: bool
telemetry_process_stop_timeout_seconds: float
telemetry_reader_join_timeout_seconds: float
ndpi_max_flows: int
@@ -73,6 +82,8 @@ def load_settings() -> BackendSettings:
sniffer_recv_bytes=_env_int("BACKEND_SNIFFER_RECV_BYTES", 65_536),
sniffer_buffer_drain_interval_seconds=_env_float("BACKEND_SNIFFER_BUFFER_DRAIN_INTERVAL_SECONDS", 5.0),
sniffer_thread_join_timeout_seconds=_env_float("BACKEND_SNIFFER_THREAD_JOIN_TIMEOUT_SECONDS", 2.0),
capture_interface=_env_str("BACKEND_CAPTURE_INTERFACE", ""),
capture_header_enabled=_env_bool("BACKEND_CAPTURE_HEADER_ENABLED", True),
telemetry_process_stop_timeout_seconds=_env_float("BACKEND_TELEMETRY_PROCESS_STOP_TIMEOUT_SECONDS", 3.0),
telemetry_reader_join_timeout_seconds=_env_float("BACKEND_TELEMETRY_READER_JOIN_TIMEOUT_SECONDS", 2.0),
ndpi_max_flows=_env_int("BACKEND_NDPI_MAX_FLOWS", 200_000),

View File

@@ -33,6 +33,7 @@ from src.utilities.interface_bridge_helpers import (
)
from src.config import settings
from src.utilities.bridge_telemetry import bridge_telemetry_manager
from src.utilities.capture_header import parse_capture_header
from src.utilities.ndpi_classifier import ndpi_classifier
from src.utilities.packet_identity import build_packet_uid
from src.utilities.packet_tracker import packet_tracker
@@ -61,8 +62,13 @@ sessions: Dict[str, Dict[str, Any]] = {}
class PacketInfo(TypedDict, total=False):
"""TypedDict for parsed packet data used by persistence and telemetry."""
correlation_key: str
correlation_source: str
packet_id: Optional[str]
packet_uid: str
skb_mark: Optional[int]
iface: str
capture_iface: Optional[str]
length: int
raw: bytes
src_mac: Optional[str]
@@ -85,6 +91,7 @@ class PacketInfo(TypedDict, total=False):
app_is_encrypted: Optional[bool]
app_risk_score: Optional[int]
dpi_metadata: Optional[Dict[str, Any]]
capture_metadata: Optional[Dict[str, Any]]
ip_id: Optional[int]
icmp_type: Optional[int]
icmp_code: Optional[int]
@@ -155,7 +162,7 @@ def _safe_get_attr(layer, attr: str):
return None
def parse_packet(pkt, bridge_label: str) -> None:
def parse_packet(pkt, bridge_label: str, capture_metadata: Optional[Dict[str, Any]] = None) -> None:
"""
Parse a scapy Packet object into a normalized PacketInfo and schedule DB insert.
bridge_label indicates whether the packet was captured as part of a bridge-snapshot or single-interface.
@@ -168,8 +175,11 @@ def parse_packet(pkt, bridge_label: str) -> None:
pkt_info: PacketInfo = {
"iface": pkt_iface,
"capture_iface": pkt_iface if capture_metadata else None,
"length": len(pkt),
"raw": bytes(pkt),
"packet_id": capture_metadata.get("packet_id") if capture_metadata else None,
"skb_mark": capture_metadata.get("skb_mark") if capture_metadata else None,
"src_mac": None,
"dst_mac": None,
"eth_type_raw": None,
@@ -190,6 +200,7 @@ def parse_packet(pkt, bridge_label: str) -> None:
"app_is_encrypted": None,
"app_risk_score": None,
"dpi_metadata": None,
"capture_metadata": capture_metadata,
"ip_id": None,
"icmp_type": None,
"icmp_code": None,
@@ -369,7 +380,13 @@ def parse_packet(pkt, bridge_label: str) -> None:
pkt_info["icmp_embedded_src_port"] = _safe_get_attr(inner[UDP], "sport")
pkt_info["icmp_embedded_dst_port"] = _safe_get_attr(inner[UDP], "dport")
pkt_info["packet_uid"] = build_packet_uid(pkt_info)
if pkt_info.get("packet_id"):
pkt_info["correlation_key"] = f"pid:{pkt_info['packet_id']}"
pkt_info["correlation_source"] = "kernel_mark"
else:
pkt_info["packet_uid"] = build_packet_uid(pkt_info)
pkt_info["correlation_key"] = f"uid:{pkt_info['packet_uid']}"
pkt_info["correlation_source"] = "legacy_hash"
try:
web_loop = getattr(shared_objects, "web_loop", None)
@@ -548,9 +565,13 @@ def _session_reader_loop(session_id: str) -> None:
# parse with scapy
try:
pkt = Ether(raw)
capture_meta = None
packet_bytes = raw
if settings.capture_header_enabled and settings.capture_interface and iface == settings.capture_interface:
capture_meta, packet_bytes = parse_capture_header(raw)
pkt = Ether(packet_bytes)
pkt.sniffed_on = iface
parse_packet(pkt, label)
parse_packet(pkt, label, capture_metadata=capture_meta)
logger.debug("Captured packet on %s in session %s (len=%d)", iface, session_id, len(raw))
except Exception:
logger.exception("Failed to parse/process packet from %s in session %s", iface, session_id)
@@ -595,19 +616,23 @@ def start_afpacket_sniffer(target: str, target_is_interface: bool = False) -> st
"label": target,
"is_bridge": not target_is_interface,
"ports": [],
"capture_ifaces": [],
}
sessions[session_id] = session
# determine ports for this session
if target_is_interface:
ports = [target]
capture_ifaces = [target]
else:
ports = get_bridge_ports_once(target)
capture_ifaces = [settings.capture_interface] if settings.capture_interface else list(ports)
session["ports"] = ports
session["capture_ifaces"] = capture_ifaces
# create sockets for this session only
for iface in ports:
for iface in capture_ifaces:
if not check_interface_exists(iface):
logger.warning("Snapshot port %s missing for session %s, skipping", iface, session_id)
continue
@@ -620,7 +645,13 @@ def start_afpacket_sniffer(target: str, target_is_interface: bool = False) -> st
session["thread"] = t
t.start()
_sync_bridge_telemetry()
logger.info("Started sniffer session %s label=%s ports=%s", session_id, target, ports)
logger.info(
"Started sniffer session %s label=%s ports=%s capture_ifaces=%s",
session_id,
target,
ports,
capture_ifaces,
)
return session_id
@@ -718,6 +749,7 @@ def get_internal_debug_state() -> dict:
"label": s.get("label"),
"is_bridge": s.get("is_bridge"),
"ports": list(s.get("ports", [])),
"capture_ifaces": list(s.get("capture_ifaces", [])),
"sockets": list(s.get("sockets", {}).keys()),
"thread_alive": bool(s.get("thread") and s.get("thread").is_alive()),
}

View File

@@ -0,0 +1,37 @@
"""Parse the capture-side shim that carries the skb mark to AF_PACKET userspace."""
from __future__ import annotations
import struct
from typing import Dict, Optional, Tuple
from src.utilities.packet_mark import packet_id_from_mark
CAPTURE_HEADER_MAGIC = b"MTCP"
CAPTURE_HEADER_VERSION = 1
CAPTURE_HEADER_STRUCT = struct.Struct("!4sBBHII")
CAPTURE_HEADER_SIZE = CAPTURE_HEADER_STRUCT.size
def parse_capture_header(frame: bytes) -> Tuple[Optional[Dict[str, object]], bytes]:
"""Return parsed capture metadata and the stripped Ethernet frame."""
if len(frame) < CAPTURE_HEADER_SIZE:
return None, frame
magic, version, flags, header_len, skb_mark, aux_value = CAPTURE_HEADER_STRUCT.unpack_from(frame)
if magic != CAPTURE_HEADER_MAGIC or version != CAPTURE_HEADER_VERSION:
return None, frame
if header_len < CAPTURE_HEADER_SIZE or len(frame) < header_len:
return None, frame
metadata: Dict[str, object] = {
"header_magic": magic.decode("ascii", "replace"),
"header_version": int(version),
"header_flags": int(flags),
"header_len": int(header_len),
"skb_mark": int(skb_mark),
"aux_value": int(aux_value),
"packet_id": packet_id_from_mark(skb_mark),
}
return metadata, frame[header_len:]

View File

@@ -42,7 +42,7 @@ def _serialize_row_for_broadcast(row: Dict[str, Any]) -> Dict[str, Any]:
def _normalize_json_fields(payload: Dict[str, Any]) -> None:
for key in ("dpi_metadata", "telemetry_metadata"):
for key in ("dpi_metadata", "capture_metadata", "telemetry_metadata"):
value = payload.get(key)
if isinstance(value, str):
try:
@@ -118,7 +118,12 @@ class DatabasePool:
row = await conn.fetchrow(
"""
INSERT INTO packets (
correlation_key,
packet_id,
packet_uid,
correlation_source,
skb_mark,
capture_iface,
ingress_if,
egress_if,
verdict,
@@ -149,14 +154,21 @@ class DatabasePool:
app_is_encrypted,
app_risk_score,
dpi_metadata,
capture_metadata,
telemetry_metadata,
raw
) VALUES(
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,
$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,
$26,$27,$28,$29,$30,$31::jsonb,$32::jsonb,$33
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,
$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33,$34,$35,$36::jsonb,
$37::jsonb,$38::jsonb,$39
)
ON CONFLICT (packet_uid) DO UPDATE SET
ON CONFLICT (correlation_key) DO UPDATE SET
updated_at = NOW(),
packet_id = COALESCE(EXCLUDED.packet_id, packets.packet_id),
packet_uid = COALESCE(EXCLUDED.packet_uid, packets.packet_uid),
correlation_source = COALESCE(EXCLUDED.correlation_source, packets.correlation_source),
skb_mark = COALESCE(EXCLUDED.skb_mark, packets.skb_mark),
capture_iface = COALESCE(EXCLUDED.capture_iface, packets.capture_iface),
ingress_if = COALESCE(EXCLUDED.ingress_if, packets.ingress_if),
egress_if = COALESCE(EXCLUDED.egress_if, packets.egress_if),
verdict = COALESCE(EXCLUDED.verdict, packets.verdict),
@@ -195,11 +207,17 @@ class DatabasePool:
app_is_encrypted = COALESCE(EXCLUDED.app_is_encrypted, packets.app_is_encrypted),
app_risk_score = COALESCE(EXCLUDED.app_risk_score, packets.app_risk_score),
dpi_metadata = COALESCE(EXCLUDED.dpi_metadata, packets.dpi_metadata),
capture_metadata = COALESCE(EXCLUDED.capture_metadata, packets.capture_metadata),
telemetry_metadata = COALESCE(EXCLUDED.telemetry_metadata, packets.telemetry_metadata),
raw = COALESCE(EXCLUDED.raw, packets.raw)
RETURNING *
""",
pkt_info["packet_uid"],
pkt_info["correlation_key"],
pkt_info.get("packet_id"),
pkt_info.get("packet_uid"),
pkt_info.get("correlation_source"),
pkt_info.get("skb_mark"),
pkt_info.get("capture_iface"),
pkt_info.get("ingress_if"),
pkt_info.get("egress_if"),
pkt_info.get("verdict"),
@@ -230,6 +248,7 @@ class DatabasePool:
pkt_info.get("app_is_encrypted"),
pkt_info.get("app_risk_score"),
json.dumps(dpi_metadata) if dpi_metadata is not None else None,
json.dumps(pkt_info.get("capture_metadata")) if pkt_info.get("capture_metadata") is not None else None,
json.dumps(telemetry_metadata) if telemetry_metadata is not None else None,
pkt_info.get("raw"),
)
@@ -257,7 +276,7 @@ class DatabasePool:
"""
SELECT *
FROM packets
ORDER BY id DESC
ORDER BY updated_at DESC, id DESC
LIMIT $1
""",
limit,

View File

@@ -20,6 +20,8 @@ except Exception as exc: # pragma: no cover - depends on host runtime
print(f"Failed to import python3-bpfcc: {exc}", file=sys.stderr, flush=True)
raise
from src.utilities.packet_mark import packet_id_from_mark, verdict_from_mark
IDENTITY_FIELDS = (
"src_mac",
@@ -75,6 +77,7 @@ struct arp_eth_ipv4_t {
struct event_t {
__u64 ts_ns;
__u32 skb_mark;
__u32 length;
__u32 reason;
__u16 eth_type_raw;
@@ -130,6 +133,7 @@ static __always_inline int parse_skb(struct sk_buff *skb, struct event_t *event)
bpf_probe_read_kernel(&network_header, sizeof(network_header), &skb->network_header);
bpf_probe_read_kernel(&transport_header, sizeof(transport_header), &skb->transport_header);
bpf_probe_read_kernel(&event->length, sizeof(event->length), &skb->len);
bpf_probe_read_kernel(&event->skb_mark, sizeof(event->skb_mark), &skb->mark);
if (!head) {
return 0;
@@ -302,6 +306,7 @@ TRACEPOINT_PROBE(net, net_dev_queue) {
class Event(ct.Structure):
_fields_ = [
("ts_ns", ct.c_ulonglong),
("skb_mark", ct.c_uint),
("length", ct.c_uint),
("reason", ct.c_uint),
("eth_type_raw", ct.c_ushort),
@@ -371,6 +376,7 @@ def _emit_event(cpu: int, data: int, size: int) -> None:
payload: dict[str, object] = {
"event_type": _event_name(event.event_type),
"iface": iface,
"skb_mark": int(event.skb_mark) or None,
"length": int(event.length),
"src_mac": _mac_to_str(event.src_mac),
"dst_mac": _mac_to_str(event.dst_mac),
@@ -391,7 +397,18 @@ def _emit_event(cpu: int, data: int, size: int) -> None:
"reason": _reason_name(int(event.reason)) if event.event_type == 3 else None,
"reason_code": int(event.reason) if event.event_type == 3 else None,
}
payload["packet_uid"] = _build_packet_uid(payload)
packet_id = packet_id_from_mark(payload.get("skb_mark"))
if packet_id:
payload["packet_id"] = packet_id
payload["correlation_key"] = f"pid:{packet_id}"
payload["correlation_source"] = "kernel_mark"
verdict_hint = verdict_from_mark(payload.get("skb_mark"))
if verdict_hint:
payload["verdict_hint"] = verdict_hint
else:
payload["packet_uid"] = _build_packet_uid(payload)
payload["correlation_key"] = f"uid:{payload['packet_uid']}"
payload["correlation_source"] = "legacy_hash"
print(json.dumps(payload, separators=(",", ":")), flush=True)

View File

@@ -0,0 +1,46 @@
"""Helpers for the shared skb mark layout used for packet correlation and verdict hints."""
from __future__ import annotations
from typing import Optional
PACKET_ID_MASK = 0x0FFFFFFF
VERDICT_MASK = 0xF0000000
VERDICT_FLAG_DROP = 0x10000000
VERDICT_FLAG_REJECT = 0x20000000
def normalize_skb_mark(value: object) -> Optional[int]:
"""Return a positive integer skb mark or `None` when the value is empty."""
if value in (None, "", 0, "0"):
return None
try:
mark = int(value)
except (TypeError, ValueError):
return None
if mark < 0:
mark &= 0xFFFFFFFF
return mark or None
def packet_id_from_mark(value: object) -> Optional[str]:
"""Extract the packet correlation identifier from the shared skb mark layout."""
mark = normalize_skb_mark(value)
if mark is None:
return None
packet_id = mark & PACKET_ID_MASK
return str(packet_id) if packet_id else None
def verdict_from_mark(value: object) -> Optional[str]:
"""Return a verdict hint encoded into the upper mark bits, if any."""
mark = normalize_skb_mark(value)
if mark is None:
return None
verdict_bits = mark & VERDICT_MASK
if verdict_bits & VERDICT_FLAG_REJECT:
return "reject"
if verdict_bits & VERDICT_FLAG_DROP:
return "drop"
return None

View File

@@ -14,6 +14,7 @@ from src.Models.etherType import EtherTypeEnum, ethertype_from_int
from src.Models.ip_protocol import protocol_from_number
from src.config import settings
from src.utilities.packet_identity import build_packet_uid
from src.utilities.packet_mark import packet_id_from_mark, verdict_from_mark
logger = logging.getLogger("packet_tracker")
@@ -42,6 +43,8 @@ class PacketTracker:
"persisted_merged": 0,
"persisted_with_raw": 0,
"persisted_without_raw": 0,
"persisted_kernel_mark": 0,
"persisted_legacy_hash": 0,
}
self._lock = threading.Lock()
self._stop_event = threading.Event()
@@ -55,46 +58,46 @@ class PacketTracker:
def observe_packet(self, pkt_info: Dict[str, Any]) -> str:
"""Merge parsed packet information into a pending packet entry."""
now_ts = time.time()
packet_uid = pkt_info.get("packet_uid") or build_packet_uid(pkt_info)
pkt_info["packet_uid"] = packet_uid
correlation_key = self._ensure_correlation(pkt_info)
pkt_info["raw_present"] = pkt_info.get("raw") is not None
pkt_info["capture_sources"] = ["af_packet"]
with self._lock:
entry = self._entries.get(packet_uid)
entry = self._entries.get(correlation_key)
if entry is None:
entry = self._new_entry(packet_uid, now_ts)
self._entries[packet_uid] = entry
entry = self._new_entry(correlation_key, now_ts)
self._entries[correlation_key] = entry
self._merge_packet_info(entry, pkt_info, now_ts)
self._maybe_promote_reject_from_reply(pkt_info, now_ts)
self._maybe_mark_complete(entry)
return packet_uid
return correlation_key
def observe_telemetry(self, event: Dict[str, Any]) -> Optional[str]:
"""Merge ingress/egress/verdict telemetry into a pending packet entry."""
packet_uid = event.get("packet_uid")
if not packet_uid:
try:
packet_uid = build_packet_uid(event)
except Exception:
logger.debug("Telemetry event missing packet identity: %s", event)
return None
correlation_key = self._ensure_correlation(event)
if not correlation_key:
logger.debug("Telemetry event missing packet identity: %s", event)
return None
now_ts = time.time()
with self._lock:
entry = self._entries.get(packet_uid)
entry = self._entries.get(correlation_key)
if entry is None:
entry = self._new_entry(packet_uid, now_ts)
self._entries[packet_uid] = entry
entry = self._new_entry(correlation_key, now_ts)
self._entries[correlation_key] = entry
payload = entry["payload"]
payload["packet_uid"] = packet_uid
payload["correlation_key"] = correlation_key
payload["packet_id"] = event.get("packet_id") or payload.get("packet_id")
payload["packet_uid"] = event.get("packet_uid") or payload.get("packet_uid")
payload["correlation_source"] = event.get("correlation_source") or payload.get("correlation_source")
payload["skb_mark"] = event.get("skb_mark") or payload.get("skb_mark")
payload["telemetry_metadata"] = event
payload["last_observed_at"] = now_ts
self._add_capture_source(payload, "telemetry")
for key, value in event.items():
if value is None or key in {"event_type", "reason", "reason_code", "iface", "packet_uid"}:
if value is None or key in {"event_type", "reason", "reason_code", "iface", "packet_uid", "correlation_key"}:
continue
if payload.get(key) is None:
payload[key] = value
@@ -112,6 +115,7 @@ class PacketTracker:
payload["protocol"] = int(payload["protocol_raw"])
event_type = event.get("event_type")
iface = event.get("iface")
verdict_hint = event.get("verdict_hint")
if event_type == "ingress":
payload["ingress_if"] = iface
@@ -124,11 +128,11 @@ class PacketTracker:
payload["verdict_confidence"] = "high"
payload["verdict_seen_at"] = _utcnow()
elif event_type == "drop":
payload["verdict"] = "drop"
payload["verdict_reason"] = event.get("reason") or "kfree_skb"
payload["verdict"] = verdict_hint or "drop"
payload["verdict_reason"] = event.get("reason") or ("mark-verdict" if verdict_hint else "kfree_skb")
payload["verdict_confidence"] = "high"
payload["verdict_seen_at"] = _utcnow()
elif event_type == "reject":
elif event_type == "reject" or verdict_hint == "reject":
payload["verdict"] = "reject"
payload["verdict_reason"] = event.get("reason") or "netfilter-reject"
payload["verdict_confidence"] = event.get("verdict_confidence") or "medium"
@@ -137,18 +141,23 @@ class PacketTracker:
entry["last_observed_at"] = now_ts
entry["dirty"] = True
self._maybe_mark_complete(entry)
return packet_uid
return correlation_key
def _new_entry(self, packet_uid: str, now_ts: float) -> Dict[str, Any]:
def _new_entry(self, correlation_key: str, now_ts: float) -> Dict[str, Any]:
return {
"packet_uid": packet_uid,
"correlation_key": correlation_key,
"payload": {
"packet_uid": packet_uid,
"correlation_key": correlation_key,
"correlation_source": None,
"packet_id": None,
"packet_uid": None,
"skb_mark": None,
"verdict": "pending",
"verdict_reason": None,
"verdict_confidence": None,
"raw_present": False,
"capture_sources": [],
"capture_metadata": None,
"telemetry_metadata": None,
},
"persisted": False,
@@ -160,6 +169,34 @@ class PacketTracker:
"last_persisted_at": 0.0,
}
def _ensure_correlation(self, payload: Dict[str, Any]) -> Optional[str]:
skb_mark = payload.get("skb_mark")
if payload.get("packet_id") is None and skb_mark is not None:
payload["packet_id"] = packet_id_from_mark(skb_mark)
if payload.get("verdict_hint") is None and skb_mark is not None:
payload["verdict_hint"] = verdict_from_mark(skb_mark)
packet_id = payload.get("packet_id")
if packet_id not in (None, ""):
packet_id = str(packet_id)
payload["packet_id"] = packet_id
payload["correlation_key"] = f"pid:{packet_id}"
if not payload.get("correlation_source"):
payload["correlation_source"] = "kernel_mark"
return payload["correlation_key"]
packet_uid = payload.get("packet_uid")
if packet_uid in (None, ""):
try:
packet_uid = build_packet_uid(payload)
except Exception:
return None
payload["packet_uid"] = packet_uid
payload["correlation_key"] = f"uid:{packet_uid}"
if not payload.get("correlation_source"):
payload["correlation_source"] = "legacy_hash"
return payload["correlation_key"]
def _add_capture_source(self, payload: Dict[str, Any], source: str) -> None:
capture_sources = payload.setdefault("capture_sources", [])
if source not in capture_sources:
@@ -182,7 +219,14 @@ class PacketTracker:
changed = True
iface = pkt_info.get("iface")
if iface and not payload.get("ingress_if"):
if iface and pkt_info.get("capture_iface") and not payload.get("capture_iface"):
payload["capture_iface"] = pkt_info.get("capture_iface")
changed = True
elif iface and pkt_info.get("capture_metadata") and not payload.get("capture_iface"):
payload["capture_iface"] = iface
changed = True
if iface and not pkt_info.get("capture_metadata") and not payload.get("ingress_if"):
payload["ingress_if"] = iface
payload["ingress_seen_at"] = _utcnow()
changed = True
@@ -267,11 +311,11 @@ class PacketTracker:
while not self._stop_event.is_set():
time.sleep(0.05)
due_entries: List[Dict[str, Any]] = []
expired_uids: List[str] = []
expired_keys: List[str] = []
now_ts = time.time()
with self._lock:
for packet_uid, entry in list(self._entries.items()):
for correlation_key, entry in list(self._entries.items()):
age = now_ts - entry["last_observed_at"]
if not entry["finalized"] and age >= self._finalize_delay_seconds:
entry["finalized"] = True
@@ -290,7 +334,7 @@ class PacketTracker:
if should_flush:
due_entries.append(
{
"packet_uid": entry["packet_uid"],
"correlation_key": entry["correlation_key"],
"payload": dict(entry["payload"]),
}
)
@@ -298,10 +342,10 @@ class PacketTracker:
if entry["finalized"] and not entry["stats_recorded"]:
self._record_stats(entry["payload"])
entry["stats_recorded"] = True
expired_uids.append(packet_uid)
expired_keys.append(correlation_key)
for packet_uid in expired_uids:
self._entries.pop(packet_uid, None)
for correlation_key in expired_keys:
self._entries.pop(correlation_key, None)
for entry in due_entries:
self._persist(entry)
@@ -318,13 +362,13 @@ class PacketTracker:
fut = asyncio.run_coroutine_threadsafe(web_db.upsert_packet(payload), web_loop)
fut.result(timeout=settings.packet_tracker_persist_timeout_seconds)
with self._lock:
current = self._entries.get(entry["packet_uid"])
current = self._entries.get(entry["correlation_key"])
if current is not None:
current["persisted"] = True
current["dirty"] = False
current["last_persisted_at"] = time.time()
except Exception:
logger.exception("Failed to persist packet %s", entry["packet_uid"])
logger.exception("Failed to persist packet %s", entry["correlation_key"])
def _record_stats(self, payload: Dict[str, Any]) -> None:
capture_sources = set(payload.get("capture_sources") or [])
@@ -333,6 +377,10 @@ class PacketTracker:
self._stats["persisted_with_raw"] += 1
else:
self._stats["persisted_without_raw"] += 1
if payload.get("correlation_source") == "kernel_mark":
self._stats["persisted_kernel_mark"] = self._stats.get("persisted_kernel_mark", 0) + 1
else:
self._stats["persisted_legacy_hash"] = self._stats.get("persisted_legacy_hash", 0) + 1
if capture_sources == {"af_packet"}:
self._stats["persisted_af_packet_only"] += 1
@@ -352,6 +400,8 @@ class PacketTracker:
active_af_packet_only = 0
active_telemetry_only = 0
active_merged = 0
active_kernel_mark = 0
active_legacy_hash = 0
for entry in active_entries:
capture_sources = set(entry["payload"].get("capture_sources") or [])
if capture_sources == {"af_packet"}:
@@ -360,6 +410,10 @@ class PacketTracker:
active_telemetry_only += 1
else:
active_merged += 1
if entry["payload"].get("correlation_source") == "kernel_mark":
active_kernel_mark += 1
else:
active_legacy_hash += 1
return {
"active_total": active_total,
@@ -368,6 +422,8 @@ class PacketTracker:
"active_af_packet_only": active_af_packet_only,
"active_telemetry_only": active_telemetry_only,
"active_merged": active_merged,
"active_kernel_mark": active_kernel_mark,
"active_legacy_hash": active_legacy_hash,
"cumulative": stats,
}