test ebpf
All checks were successful
Build and Deploy MITM Webserver / build (push) Successful in 1m40s

This commit is contained in:
2026-03-07 10:06:24 +01:00
parent a98054cb5b
commit f9bbe9b73c
13 changed files with 1271 additions and 177 deletions

Binary file not shown.

View File

@@ -11,7 +11,11 @@ class PacketDBModel(BaseModel):
id: Union[int, str] id: Union[int, str]
timestamp: datetime = Field(..., description="Packet timestamp in ISO format.") timestamp: datetime = Field(..., description="Packet timestamp in ISO format.")
iface: str packet_uid: str = Field(..., description="Stable packet identity used for upserts/correlation.")
iface: Optional[str] = None
ingress_if: Optional[str] = None
egress_if: Optional[str] = None
observed_ifaces: Optional[list[str]] = None
src_mac: Optional[str] = None src_mac: Optional[str] = None
dst_mac: Optional[str] = None dst_mac: Optional[str] = None
eth_type: Optional[Union[int, str]] = None eth_type: Optional[Union[int, str]] = None
@@ -31,7 +35,14 @@ class PacketDBModel(BaseModel):
app_is_encrypted: Optional[bool] = Field(None, description="Whether detected protocol appears encrypted.") app_is_encrypted: Optional[bool] = Field(None, description="Whether detected protocol appears encrypted.")
app_risk_score: Optional[int] = Field(None, description="Count/score of detected nDPI risks.") app_risk_score: Optional[int] = Field(None, description="Count/score of detected nDPI risks.")
dpi_metadata: Optional[dict] = Field(None, description="Raw DPI metadata from nDPI.") dpi_metadata: Optional[dict] = Field(None, description="Raw DPI metadata from nDPI.")
telemetry_metadata: Optional[dict] = Field(None, description="Kernel telemetry details from eBPF collector.")
direction: Optional[str] = None direction: Optional[str] = None
verdict: Optional[str] = None
verdict_reason: Optional[str] = None
verdict_confidence: Optional[str] = None
ingress_seen_at: Optional[datetime] = None
egress_seen_at: Optional[datetime] = None
verdict_seen_at: Optional[datetime] = None
packets: Optional[int] = None packets: Optional[int] = None
class Config: class Config:
@@ -39,7 +50,11 @@ class PacketDBModel(BaseModel):
"example": { "example": {
"id": 123, "id": 123,
"timestamp": "2026-03-05T12:34:56.789Z", "timestamp": "2026-03-05T12:34:56.789Z",
"packet_uid": "9f6d3af0d3c81cb20ee8e7d32df7c56414460542",
"iface": "eth0", "iface": "eth0",
"ingress_if": "eth0",
"egress_if": "eth1",
"observed_ifaces": ["eth0", "eth1"],
"src_mac": "aa:bb:cc:dd:ee:ff", "src_mac": "aa:bb:cc:dd:ee:ff",
"dst_mac": "11:22:33:44:55:66", "dst_mac": "11:22:33:44:55:66",
"eth_type": "IPv4", "eth_type": "IPv4",
@@ -59,5 +74,10 @@ class PacketDBModel(BaseModel):
"app_is_encrypted": False, "app_is_encrypted": False,
"app_risk_score": 0, "app_risk_score": 0,
"dpi_metadata": {"method": "GET"}, "dpi_metadata": {"method": "GET"},
"telemetry_metadata": {"event_type": "egress", "iface": "eth1"},
"direction": "forwarded",
"verdict": "accept",
"verdict_reason": "egress-observed",
"verdict_confidence": "high",
} }
} }

View File

@@ -77,6 +77,26 @@ async def shutdown_event() -> None:
except Exception: except Exception:
logging.exception("Error shutting down network API") logging.exception("Error shutting down network API")
try:
from src.utilities.bridge_telemetry import bridge_telemetry_manager
bridge_telemetry_manager.stop()
except Exception:
logging.exception("Failed to stop bridge telemetry collector")
try:
from src.utilities.packet_tracker import packet_tracker
packet_tracker.stop()
except Exception:
logging.exception("Failed to stop packet tracker")
try:
if shared_objects.broadcaster is not None:
await shared_objects.broadcaster.close()
except Exception:
logging.exception("Failed to close packet broadcaster during shutdown")
try: try:
web_db = getattr(shared_objects, "db", None) web_db = getattr(shared_objects, "db", None)
if web_db is not None: if web_db is not None:

View File

@@ -31,7 +31,10 @@ from src.utilities.interface_bridge_helpers import (
check_interface_up, check_interface_up,
get_bridge_ports_once, get_bridge_ports_once,
) )
from src.utilities.bridge_telemetry import bridge_telemetry_manager
from src.utilities.ndpi_classifier import ndpi_classifier from src.utilities.ndpi_classifier import ndpi_classifier
from src.utilities.packet_identity import build_packet_uid
from src.utilities.packet_tracker import packet_tracker
from src.Models.etherType import EtherTypeEnum, ethertype_from_int from src.Models.etherType import EtherTypeEnum, ethertype_from_int
from src.Models.ip_protocol import IPProtocolEnum, protocol_from_number from src.Models.ip_protocol import IPProtocolEnum, protocol_from_number
@@ -55,13 +58,12 @@ sessions: Dict[str, Dict[str, Any]] = {}
# PacketInfo typing # PacketInfo typing
# ------------------------- # -------------------------
class PacketInfo(TypedDict, total=False): class PacketInfo(TypedDict, total=False):
""" """TypedDict for parsed packet data used by persistence and telemetry."""
TypedDict for the parsed packet info produced by parse_packet.
Fields marked optional (total=False) for flexibility across contexts. packet_uid: str
"""
iface: str iface: str
length: int length: int
raw: bytes # original raw bytes (kept for buffering; DB helper may convert to base64) raw: bytes
src_mac: Optional[str] src_mac: Optional[str]
dst_mac: Optional[str] dst_mac: Optional[str]
eth_type_raw: Optional[int] eth_type_raw: Optional[int]
@@ -82,6 +84,18 @@ class PacketInfo(TypedDict, total=False):
app_is_encrypted: Optional[bool] app_is_encrypted: Optional[bool]
app_risk_score: Optional[int] app_risk_score: Optional[int]
dpi_metadata: Optional[Dict[str, Any]] dpi_metadata: Optional[Dict[str, Any]]
ip_id: Optional[int]
icmp_type: Optional[int]
icmp_code: Optional[int]
arp_op: Optional[int]
tcp_seq: Optional[int]
tcp_ack: Optional[int]
tcp_flags: Optional[int]
icmp_embedded_src_ip: Optional[str]
icmp_embedded_dst_ip: Optional[str]
icmp_embedded_protocol: Optional[int]
icmp_embedded_src_port: Optional[int]
icmp_embedded_dst_port: Optional[int]
# small bounded buffer for packets produced before shared_objects is ready # small bounded buffer for packets produced before shared_objects is ready
@@ -105,24 +119,19 @@ threading.Thread(target=_start_async_loop, args=(async_loop,), daemon=True).star
# Helpers for buffer draining # Helpers for buffer draining
# ------------------------- # -------------------------
def drain_buffer_to_shared_db() -> None: def drain_buffer_to_shared_db() -> None:
""" """Replay buffered packets once the shared DB loop becomes available."""
Attempt to schedule buffered packets for insertion on shared_objects.web_loop.
Call this from main.py after shared_objects.db and shared_objects.web_loop are initialized.
"""
try: try:
web_loop = getattr(shared_objects, "web_loop", None) web_loop = getattr(shared_objects, "web_loop", None)
web_db = getattr(shared_objects, "db", None) web_db = getattr(shared_objects, "db", None)
if web_db is None or web_loop is None: if web_db is None or web_loop is None:
return return
# schedule draining on the web loop to avoid blocking this thread
def _drain() -> None: def _drain() -> None:
while _PACKET_BUFFER: while _PACKET_BUFFER:
pkt = _PACKET_BUFFER.pop(0) pkt = _PACKET_BUFFER.pop(0)
try: try:
asyncio.run_coroutine_threadsafe(web_db.insert_packet(pkt), web_loop) packet_tracker.observe_packet(pkt)
except Exception: except Exception:
# re-buffer first element and stop to avoid busy loop
_PACKET_BUFFER.insert(0, pkt) _PACKET_BUFFER.insert(0, pkt)
break break
@@ -180,6 +189,18 @@ def parse_packet(pkt, bridge_label: str) -> None:
"app_is_encrypted": None, "app_is_encrypted": None,
"app_risk_score": None, "app_risk_score": None,
"dpi_metadata": None, "dpi_metadata": None,
"ip_id": None,
"icmp_type": None,
"icmp_code": None,
"arp_op": None,
"tcp_seq": None,
"tcp_ack": None,
"tcp_flags": None,
"icmp_embedded_src_ip": None,
"icmp_embedded_dst_ip": None,
"icmp_embedded_protocol": None,
"icmp_embedded_src_port": None,
"icmp_embedded_dst_port": None,
} }
# Ethernet layer # Ethernet layer
@@ -220,6 +241,10 @@ def parse_packet(pkt, bridge_label: str) -> None:
pkt_info["protocol_name"] = "ARP" pkt_info["protocol_name"] = "ARP"
pkt_info["src_ip"] = _safe_get_attr(arp, "psrc") pkt_info["src_ip"] = _safe_get_attr(arp, "psrc")
pkt_info["dst_ip"] = _safe_get_attr(arp, "pdst") pkt_info["dst_ip"] = _safe_get_attr(arp, "pdst")
try:
pkt_info["arp_op"] = int(_safe_get_attr(arp, "op"))
except Exception:
pkt_info["arp_op"] = None
pkt_info["src_port"] = None pkt_info["src_port"] = None
pkt_info["dst_port"] = None pkt_info["dst_port"] = None
@@ -228,6 +253,10 @@ def parse_packet(pkt, bridge_label: str) -> None:
ip = pkt[IP] ip = pkt[IP]
pkt_info["src_ip"] = _safe_get_attr(ip, "src") pkt_info["src_ip"] = _safe_get_attr(ip, "src")
pkt_info["dst_ip"] = _safe_get_attr(ip, "dst") pkt_info["dst_ip"] = _safe_get_attr(ip, "dst")
try:
pkt_info["ip_id"] = int(_safe_get_attr(ip, "id"))
except Exception:
pkt_info["ip_id"] = None
try: try:
proto_num = int(_safe_get_attr(ip, "proto")) proto_num = int(_safe_get_attr(ip, "proto"))
@@ -245,12 +274,23 @@ def parse_packet(pkt, bridge_label: str) -> None:
pkt_info["protocol_name"] = "TCP" pkt_info["protocol_name"] = "TCP"
pkt_info["src_port"] = _safe_get_attr(pkt[TCP], "sport") pkt_info["src_port"] = _safe_get_attr(pkt[TCP], "sport")
pkt_info["dst_port"] = _safe_get_attr(pkt[TCP], "dport") pkt_info["dst_port"] = _safe_get_attr(pkt[TCP], "dport")
try:
pkt_info["tcp_seq"] = int(_safe_get_attr(pkt[TCP], "seq"))
pkt_info["tcp_ack"] = int(_safe_get_attr(pkt[TCP], "ack"))
pkt_info["tcp_flags"] = int(_safe_get_attr(pkt[TCP], "flags"))
except Exception:
pass
elif proto_num == 17 and UDP in pkt: elif proto_num == 17 and UDP in pkt:
pkt_info["protocol_name"] = "UDP" pkt_info["protocol_name"] = "UDP"
pkt_info["src_port"] = _safe_get_attr(pkt[UDP], "sport") pkt_info["src_port"] = _safe_get_attr(pkt[UDP], "sport")
pkt_info["dst_port"] = _safe_get_attr(pkt[UDP], "dport") pkt_info["dst_port"] = _safe_get_attr(pkt[UDP], "dport")
elif proto_num == 1 and ICMP in pkt: elif proto_num == 1 and ICMP in pkt:
pkt_info["protocol_name"] = "ICMP" pkt_info["protocol_name"] = "ICMP"
try:
pkt_info["icmp_type"] = int(_safe_get_attr(pkt[ICMP], "type"))
pkt_info["icmp_code"] = int(_safe_get_attr(pkt[ICMP], "code"))
except Exception:
pass
else: else:
if pkt_info.get("protocol_name") is None: if pkt_info.get("protocol_name") is None:
pkt_info["protocol_name"] = f"IP_PROTO_{proto_num}" if proto_num is not None else None pkt_info["protocol_name"] = f"IP_PROTO_{proto_num}" if proto_num is not None else None
@@ -277,12 +317,23 @@ def parse_packet(pkt, bridge_label: str) -> None:
pkt_info["protocol_name"] = "TCP" pkt_info["protocol_name"] = "TCP"
pkt_info["src_port"] = _safe_get_attr(pkt[TCP], "sport") pkt_info["src_port"] = _safe_get_attr(pkt[TCP], "sport")
pkt_info["dst_port"] = _safe_get_attr(pkt[TCP], "dport") pkt_info["dst_port"] = _safe_get_attr(pkt[TCP], "dport")
try:
pkt_info["tcp_seq"] = int(_safe_get_attr(pkt[TCP], "seq"))
pkt_info["tcp_ack"] = int(_safe_get_attr(pkt[TCP], "ack"))
pkt_info["tcp_flags"] = int(_safe_get_attr(pkt[TCP], "flags"))
except Exception:
pass
elif nh == 17 and UDP in pkt: elif nh == 17 and UDP in pkt:
pkt_info["protocol_name"] = "UDP" pkt_info["protocol_name"] = "UDP"
pkt_info["src_port"] = _safe_get_attr(pkt[UDP], "sport") pkt_info["src_port"] = _safe_get_attr(pkt[UDP], "sport")
pkt_info["dst_port"] = _safe_get_attr(pkt[UDP], "dport") pkt_info["dst_port"] = _safe_get_attr(pkt[UDP], "dport")
elif ICMPv6Unknown in pkt: elif ICMPv6Unknown in pkt:
pkt_info["protocol_name"] = "ICMPv6" pkt_info["protocol_name"] = "ICMPv6"
try:
pkt_info["icmp_type"] = int(_safe_get_attr(pkt[ICMPv6Unknown], "type"))
pkt_info["icmp_code"] = int(_safe_get_attr(pkt[ICMPv6Unknown], "code"))
except Exception:
pass
else: else:
if pkt_info.get("protocol_name") is None: if pkt_info.get("protocol_name") is None:
pkt_info["protocol_name"] = f"IPV6_PROTO_{nh}" if nh is not None else None pkt_info["protocol_name"] = f"IPV6_PROTO_{nh}" if nh is not None else None
@@ -299,21 +350,35 @@ def parse_packet(pkt, bridge_label: str) -> None:
except Exception: except Exception:
logger.exception("nDPI enrichment failed") logger.exception("nDPI enrichment failed")
# Submit DB insert to shared web loop if available, otherwise buffer if ICMP in pkt:
inner = pkt[ICMP].payload
if inner and IP in inner:
inner_ip = inner[IP]
pkt_info["icmp_embedded_src_ip"] = _safe_get_attr(inner_ip, "src")
pkt_info["icmp_embedded_dst_ip"] = _safe_get_attr(inner_ip, "dst")
try:
pkt_info["icmp_embedded_protocol"] = int(_safe_get_attr(inner_ip, "proto"))
except Exception:
pkt_info["icmp_embedded_protocol"] = None
if TCP in inner:
pkt_info["icmp_embedded_src_port"] = _safe_get_attr(inner[TCP], "sport")
pkt_info["icmp_embedded_dst_port"] = _safe_get_attr(inner[TCP], "dport")
elif UDP in inner:
pkt_info["icmp_embedded_src_port"] = _safe_get_attr(inner[UDP], "sport")
pkt_info["icmp_embedded_dst_port"] = _safe_get_attr(inner[UDP], "dport")
pkt_info["packet_uid"] = build_packet_uid(pkt_info)
try: try:
web_loop = getattr(shared_objects, "web_loop", None) web_loop = getattr(shared_objects, "web_loop", None)
web_db = getattr(shared_objects, "db", None) web_db = getattr(shared_objects, "db", None)
if web_db is not None and web_loop is not None: if web_db is not None and web_loop is not None:
try: try:
fut = asyncio.run_coroutine_threadsafe(web_db.insert_packet(pkt_info), web_loop) packet_tracker.observe_packet(pkt_info)
# best-effort non-blocking check for immediate errors logger.debug("Tracked packet on %s (len=%d)", pkt_info.get("iface"), pkt_info.get("length"))
try:
fut.result(timeout=0.005)
except Exception:
pass
logger.debug("Scheduled insert for packet on %s (len=%d)", pkt_info.get("iface"), pkt_info.get("length"))
except Exception as e: except Exception as e:
logger.exception("Failed to schedule insert_packet for %s — buffering: %s", pkt_info.get("iface"), e) logger.exception("Failed to track packet for %s — buffering: %s", pkt_info.get("iface"), e)
_PACKET_BUFFER.append(pkt_info) _PACKET_BUFFER.append(pkt_info)
if len(_PACKET_BUFFER) > _BUFFER_CAPACITY: if len(_PACKET_BUFFER) > _BUFFER_CAPACITY:
_PACKET_BUFFER.pop(0) _PACKET_BUFFER.pop(0)
@@ -399,6 +464,14 @@ def _ensure_socket_for_session(sockets: Dict[str, socket.socket], iface: str, br
logger.warning("Failed to create AF_PACKET socket for %s (label=%s)", iface, bridge_label) logger.warning("Failed to create AF_PACKET socket for %s (label=%s)", iface, bridge_label)
def _sync_bridge_telemetry() -> None:
interfaces = sorted({iface for session in sessions.values() for iface in session.get("ports", [])})
try:
bridge_telemetry_manager.update_interfaces(interfaces)
except Exception:
logger.exception("Failed to update bridge telemetry collector")
# ------------------------- # -------------------------
# Per-session reader loop # Per-session reader loop
# ------------------------- # -------------------------
@@ -545,6 +618,7 @@ def start_afpacket_sniffer(target: str, target_is_interface: bool = False) -> st
t = threading.Thread(target=_session_reader_loop, args=(session_id,), daemon=True) t = threading.Thread(target=_session_reader_loop, args=(session_id,), daemon=True)
session["thread"] = t session["thread"] = t
t.start() t.start()
_sync_bridge_telemetry()
logger.info("Started sniffer session %s label=%s ports=%s", session_id, target, ports) logger.info("Started sniffer session %s label=%s ports=%s", session_id, target, ports)
return session_id return session_id
@@ -566,6 +640,7 @@ def stop_afpacket_sniffer(session_id: Optional[str] = None, target: Optional[str
t = s.get("thread") t = s.get("thread")
if t and isinstance(t, threading.Thread): if t and isinstance(t, threading.Thread):
t.join(timeout=2) t.join(timeout=2)
_sync_bridge_telemetry()
logger.info("Stopped session %s", session_id) logger.info("Stopped session %s", session_id)
return return
@@ -586,6 +661,7 @@ def stop_afpacket_sniffer(session_id: Optional[str] = None, target: Optional[str
except Exception: except Exception:
pass pass
logger.info("Removed target %s from session %s", target, sid) logger.info("Removed target %s from session %s", target, sid)
_sync_bridge_telemetry()
return return
# Global stop: stop all sessions # Global stop: stop all sessions
@@ -602,6 +678,10 @@ def stop_afpacket_sniffer(session_id: Optional[str] = None, target: Optional[str
logger.exception("Failed to schedule DB pool close") logger.exception("Failed to schedule DB pool close")
logger.info("All sniffer sessions stopped") logger.info("All sniffer sessions stopped")
try:
bridge_telemetry_manager.stop()
except Exception:
logger.exception("Failed to stop bridge telemetry collector")
def get_sniffer_status() -> Dict[str, Dict[str, object]]: def get_sniffer_status() -> Dict[str, Dict[str, object]]:
@@ -643,4 +723,5 @@ def get_internal_debug_state() -> dict:
for sid, s in sessions.items() for sid, s in sessions.items()
}, },
"buffer_len": len(_PACKET_BUFFER), "buffer_len": len(_PACKET_BUFFER),
"telemetry_ports": sorted({iface for session in sessions.values() for iface in session.get("ports", [])}),
} }

View File

@@ -0,0 +1,128 @@
"""Manage the eBPF telemetry subprocess used for bridge direction and verdict events."""
from __future__ import annotations
import json
import logging
import os
import signal
import subprocess
import sys
import threading
from pathlib import Path
from typing import Iterable, Optional
from src.utilities.packet_tracker import packet_tracker
logger = logging.getLogger("bridge_telemetry")
class BridgeTelemetryManager:
"""Run a single eBPF collector process for the active sniffed interfaces."""
def __init__(self) -> None:
self._interfaces: set[str] = set()
self._process: Optional[subprocess.Popen[str]] = None
self._reader_thread: Optional[threading.Thread] = None
self._lock = threading.Lock()
def update_interfaces(self, interfaces: Iterable[str]) -> None:
"""Restart the collector when the active interface set changes."""
normalized = {iface.strip() for iface in interfaces if iface and iface.strip()}
with self._lock:
if normalized == self._interfaces:
return
self._interfaces = normalized
self._restart_locked()
def stop(self) -> None:
"""Stop the collector process and reader thread."""
with self._lock:
self._interfaces = set()
self._stop_process_locked()
def _restart_locked(self) -> None:
self._stop_process_locked()
if not self._interfaces:
return
helper = Path(__file__).with_name("ebpf_bridge_events.py")
python_bin = sys.executable or "python3"
if "venv" in python_bin:
python_bin = "python3"
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
cmd = [python_bin, str(helper), "--ifaces", ",".join(sorted(self._interfaces))]
logger.info("Starting bridge telemetry collector for interfaces=%s", sorted(self._interfaces))
try:
self._process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
env=env,
)
except Exception:
logger.exception("Failed to start bridge telemetry collector")
self._process = None
return
self._reader_thread = threading.Thread(
target=self._read_loop,
args=(self._process,),
daemon=True,
name="bridge-telemetry-reader",
)
self._reader_thread.start()
def _stop_process_locked(self) -> None:
process = self._process
reader = self._reader_thread
self._process = None
self._reader_thread = None
if process is not None and process.poll() is None:
try:
process.send_signal(signal.SIGTERM)
process.wait(timeout=3)
except subprocess.TimeoutExpired:
process.kill()
except Exception:
logger.exception("Failed to stop bridge telemetry collector cleanly")
if reader is not None and reader.is_alive():
reader.join(timeout=2)
def _read_loop(self, process: subprocess.Popen[str]) -> None:
stdout = process.stdout
if stdout is None:
return
for line in stdout:
text = line.strip()
if not text:
continue
try:
event = json.loads(text)
except json.JSONDecodeError:
logger.info("bridge-telemetry: %s", text)
continue
if "event_type" not in event:
logger.info("bridge-telemetry: %s", event)
continue
try:
packet_tracker.observe_telemetry(event)
except Exception:
logger.exception("Failed to process telemetry event: %s", event)
rc = process.poll()
if rc not in (0, None):
logger.warning("Bridge telemetry collector exited with code %s", rc)
bridge_telemetry_manager = BridgeTelemetryManager()

View File

@@ -15,6 +15,16 @@ from src.Models.packets import PacketDBModel
logger = logging.getLogger("af_packet_sniffer") logger = logging.getLogger("af_packet_sniffer")
def _db_text(value: Any) -> Any:
if value is None:
return None
if isinstance(value, (str, int)):
return value
if hasattr(value, "value"):
return getattr(value, "value")
return str(value)
class DatabasePool: class DatabasePool:
"""Asyncpg connection pool wrapper used by the packet APIs.""" """Asyncpg connection pool wrapper used by the packet APIs."""
@@ -65,15 +75,33 @@ class DatabasePool:
async def insert_packet(self, pkt_info: Dict[str, Any]) -> None: async def insert_packet(self, pkt_info: Dict[str, Any]) -> None:
"""Insert one packet record and publish it to subscribers.""" """Insert one packet record and publish it to subscribers."""
await self.upsert_packet(pkt_info)
async def upsert_packet(self, pkt_info: Dict[str, Any]) -> None:
"""Insert or update one packet record and publish it to subscribers."""
if self._pool is None: if self._pool is None:
await self.init_pool() await self.init_pool()
dpi_metadata = pkt_info.get("dpi_metadata")
telemetry_metadata = pkt_info.get("telemetry_metadata")
try: try:
async with self._pool.acquire() as conn: async with self._pool.acquire() as conn:
new_row = await conn.fetchrow( new_row = await conn.fetchrow(
""" """
INSERT INTO packets( INSERT INTO packets (
packet_uid,
iface, iface,
ingress_if,
egress_if,
observed_ifaces,
direction,
verdict,
verdict_reason,
verdict_confidence,
ingress_seen_at,
egress_seen_at,
verdict_seen_at,
src_mac, src_mac,
dst_mac, dst_mac,
eth_type, eth_type,
@@ -84,7 +112,6 @@ class DatabasePool:
src_port, src_port,
dst_port, dst_port,
length, length,
raw,
app_protocol, app_protocol,
app_master_protocol, app_master_protocol,
app_category, app_category,
@@ -92,22 +119,70 @@ class DatabasePool:
app_hostname, app_hostname,
app_is_encrypted, app_is_encrypted,
app_risk_score, app_risk_score,
dpi_metadata dpi_metadata,
) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20::jsonb) telemetry_metadata,
raw
) VALUES(
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,
$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,
$24,$25,$26,$27,$28,$29,$30::jsonb,$31::jsonb,$32
)
ON CONFLICT (packet_uid) DO UPDATE SET
iface = COALESCE(EXCLUDED.iface, packets.iface),
ingress_if = COALESCE(EXCLUDED.ingress_if, packets.ingress_if),
egress_if = COALESCE(EXCLUDED.egress_if, packets.egress_if),
observed_ifaces = COALESCE(EXCLUDED.observed_ifaces, packets.observed_ifaces),
direction = COALESCE(EXCLUDED.direction, packets.direction),
verdict = COALESCE(EXCLUDED.verdict, packets.verdict),
verdict_reason = COALESCE(EXCLUDED.verdict_reason, packets.verdict_reason),
verdict_confidence = COALESCE(EXCLUDED.verdict_confidence, packets.verdict_confidence),
ingress_seen_at = COALESCE(EXCLUDED.ingress_seen_at, packets.ingress_seen_at),
egress_seen_at = COALESCE(EXCLUDED.egress_seen_at, packets.egress_seen_at),
verdict_seen_at = COALESCE(EXCLUDED.verdict_seen_at, packets.verdict_seen_at),
src_mac = COALESCE(EXCLUDED.src_mac, packets.src_mac),
dst_mac = COALESCE(EXCLUDED.dst_mac, packets.dst_mac),
eth_type = COALESCE(EXCLUDED.eth_type, packets.eth_type),
vlan_id = COALESCE(EXCLUDED.vlan_id, packets.vlan_id),
src_ip = COALESCE(EXCLUDED.src_ip, packets.src_ip),
dst_ip = COALESCE(EXCLUDED.dst_ip, packets.dst_ip),
ip_proto = COALESCE(EXCLUDED.ip_proto, packets.ip_proto),
src_port = COALESCE(EXCLUDED.src_port, packets.src_port),
dst_port = COALESCE(EXCLUDED.dst_port, packets.dst_port),
length = COALESCE(EXCLUDED.length, packets.length),
app_protocol = COALESCE(EXCLUDED.app_protocol, packets.app_protocol),
app_master_protocol = COALESCE(EXCLUDED.app_master_protocol, packets.app_master_protocol),
app_category = COALESCE(EXCLUDED.app_category, packets.app_category),
app_confidence = COALESCE(EXCLUDED.app_confidence, packets.app_confidence),
app_hostname = COALESCE(EXCLUDED.app_hostname, packets.app_hostname),
app_is_encrypted = COALESCE(EXCLUDED.app_is_encrypted, packets.app_is_encrypted),
app_risk_score = COALESCE(EXCLUDED.app_risk_score, packets.app_risk_score),
dpi_metadata = COALESCE(EXCLUDED.dpi_metadata, packets.dpi_metadata),
telemetry_metadata = COALESCE(EXCLUDED.telemetry_metadata, packets.telemetry_metadata),
raw = COALESCE(EXCLUDED.raw, packets.raw)
RETURNING id, timestamp RETURNING id, timestamp
""", """,
pkt_info["iface"], pkt_info["packet_uid"],
pkt_info.get("iface"),
pkt_info.get("ingress_if"),
pkt_info.get("egress_if"),
pkt_info.get("observed_ifaces"),
pkt_info.get("direction"),
pkt_info.get("verdict"),
pkt_info.get("verdict_reason"),
pkt_info.get("verdict_confidence"),
pkt_info.get("ingress_seen_at"),
pkt_info.get("egress_seen_at"),
pkt_info.get("verdict_seen_at"),
pkt_info.get("src_mac"), pkt_info.get("src_mac"),
pkt_info.get("dst_mac"), pkt_info.get("dst_mac"),
pkt_info.get("eth_type"), _db_text(pkt_info.get("eth_type")),
pkt_info.get("vlan_id"), pkt_info.get("vlan_id"),
pkt_info.get("src_ip"), pkt_info.get("src_ip"),
pkt_info.get("dst_ip"), pkt_info.get("dst_ip"),
pkt_info.get("protocol"), _db_text(pkt_info.get("protocol_name") or pkt_info.get("protocol")),
pkt_info.get("src_port"), pkt_info.get("src_port"),
pkt_info.get("dst_port"), pkt_info.get("dst_port"),
pkt_info["length"], pkt_info.get("length"),
pkt_info["raw"],
pkt_info.get("app_protocol"), pkt_info.get("app_protocol"),
pkt_info.get("app_master_protocol"), pkt_info.get("app_master_protocol"),
pkt_info.get("app_category"), pkt_info.get("app_category"),
@@ -115,10 +190,12 @@ class DatabasePool:
pkt_info.get("app_hostname"), pkt_info.get("app_hostname"),
pkt_info.get("app_is_encrypted"), pkt_info.get("app_is_encrypted"),
pkt_info.get("app_risk_score"), pkt_info.get("app_risk_score"),
json.dumps(pkt_info.get("dpi_metadata")) if pkt_info.get("dpi_metadata") is not None else None, json.dumps(dpi_metadata) if dpi_metadata is not None else None,
json.dumps(telemetry_metadata) if telemetry_metadata is not None else None,
pkt_info.get("raw"),
) )
except Exception: except Exception:
logger.exception("DB insert failed") logger.exception("DB upsert failed")
return return
if new_row: if new_row:

View File

@@ -0,0 +1,443 @@
#!/usr/bin/env python3
"""Emit bridge ingress, egress, and drop telemetry events via eBPF."""
from __future__ import annotations
import argparse
import ctypes as ct
import hashlib
import ipaddress
import json
import signal
import socket
import sys
from typing import Iterable
try:
from bcc import BPF # type: ignore
except Exception as exc: # pragma: no cover - depends on host runtime
print(f"Failed to import python3-bpfcc: {exc}", file=sys.stderr, flush=True)
raise
IDENTITY_FIELDS = (
"src_mac",
"dst_mac",
"eth_type_raw",
"vlan_id",
"src_ip",
"dst_ip",
"protocol_raw",
"src_port",
"dst_port",
"length",
"ip_id",
"icmp_type",
"icmp_code",
"arp_op",
"tcp_seq",
"tcp_ack",
"tcp_flags",
)
BPF_SOURCE = r"""
#include <uapi/linux/ptrace.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/in.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/icmp.h>
#include <linux/icmpv6.h>
#include <linux/if_arp.h>
#include <linux/version.h>
#define EVENT_INGRESS 1
#define EVENT_EGRESS 2
#define EVENT_DROP 3
struct vlan_hdr_t {
__be16 h_vlan_TCI;
__be16 h_vlan_encapsulated_proto;
};
struct arp_eth_ipv4_t {
__u8 sha[6];
__u8 spa[4];
__u8 tha[6];
__u8 tpa[4];
};
struct event_t {
__u64 ts_ns;
__u32 length;
__u32 reason;
__u16 eth_type_raw;
__u16 vlan_id;
__u16 src_port;
__u16 dst_port;
__u16 ip_id;
__u16 arp_op;
__u32 protocol_raw;
__u32 tcp_seq;
__u32 tcp_ack;
__u8 event_type;
__u8 ip_version;
__u8 icmp_type;
__u8 icmp_code;
__u8 tcp_flags;
char ifname[IFNAMSIZ];
unsigned char src_mac[6];
unsigned char dst_mac[6];
unsigned char src_ip[16];
unsigned char dst_ip[16];
};
BPF_PERF_OUTPUT(events);
static __always_inline int fill_ifname(struct sk_buff *skb, struct event_t *event) {
if (!skb) {
return 0;
}
struct net_device *dev = NULL;
bpf_probe_read_kernel(&dev, sizeof(dev), &skb->dev);
if (!dev) {
return 0;
}
bpf_probe_read_kernel(event->ifname, sizeof(event->ifname), dev->name);
return 1;
}
static __always_inline int parse_skb(struct sk_buff *skb, struct event_t *event) {
unsigned char *head = NULL;
__u16 mac_header = 0;
__u16 network_header = 0;
__u16 transport_header = 0;
if (!skb) {
return 0;
}
bpf_probe_read_kernel(&head, sizeof(head), &skb->head);
bpf_probe_read_kernel(&mac_header, sizeof(mac_header), &skb->mac_header);
bpf_probe_read_kernel(&network_header, sizeof(network_header), &skb->network_header);
bpf_probe_read_kernel(&transport_header, sizeof(transport_header), &skb->transport_header);
bpf_probe_read_kernel(&event->length, sizeof(event->length), &skb->len);
if (!head) {
return 0;
}
struct ethhdr eth = {};
unsigned char *eth_ptr = head + mac_header;
bpf_probe_read_kernel(&eth, sizeof(eth), eth_ptr);
__builtin_memcpy(event->src_mac, eth.h_source, 6);
__builtin_memcpy(event->dst_mac, eth.h_dest, 6);
__be16 eth_proto = eth.h_proto;
unsigned char *l3_ptr = head + network_header;
if (eth_proto == htons(ETH_P_8021Q) || eth_proto == htons(ETH_P_8021AD)) {
struct vlan_hdr_t vlan = {};
bpf_probe_read_kernel(&vlan, sizeof(vlan), eth_ptr + sizeof(struct ethhdr));
event->vlan_id = ntohs(vlan.h_vlan_TCI) & 0x0fff;
eth_proto = vlan.h_vlan_encapsulated_proto;
event->eth_type_raw = ntohs(eth_proto);
} else {
event->eth_type_raw = ntohs(eth_proto);
}
if (eth_proto == htons(ETH_P_ARP)) {
struct arphdr arph = {};
struct arp_eth_ipv4_t arp_body = {};
bpf_probe_read_kernel(&arph, sizeof(arph), l3_ptr);
event->arp_op = ntohs(arph.ar_op);
if (arph.ar_hrd == htons(ARPHRD_ETHER) && arph.ar_pro == htons(ETH_P_IP) &&
arph.ar_hln == ETH_ALEN && arph.ar_pln == 4) {
bpf_probe_read_kernel(&arp_body, sizeof(arp_body), l3_ptr + sizeof(struct arphdr));
__builtin_memcpy(event->src_ip, arp_body.spa, 4);
__builtin_memcpy(event->dst_ip, arp_body.tpa, 4);
event->ip_version = 4;
}
return 1;
}
if (eth_proto == htons(ETH_P_IP)) {
struct iphdr iph = {};
bpf_probe_read_kernel(&iph, sizeof(iph), l3_ptr);
event->ip_version = 4;
event->protocol_raw = iph.protocol;
event->ip_id = ntohs(iph.id);
bpf_probe_read_kernel(event->src_ip, 4, &iph.saddr);
bpf_probe_read_kernel(event->dst_ip, 4, &iph.daddr);
if (iph.protocol == IPPROTO_TCP) {
struct tcphdr tcph = {};
unsigned char flags = 0;
bpf_probe_read_kernel(&tcph, sizeof(tcph), head + transport_header);
event->src_port = ntohs(tcph.source);
event->dst_port = ntohs(tcph.dest);
event->tcp_seq = ntohl(tcph.seq);
event->tcp_ack = ntohl(tcph.ack_seq);
bpf_probe_read_kernel(&flags, sizeof(flags), (void *)(head + transport_header + 13));
event->tcp_flags = flags;
} else if (iph.protocol == IPPROTO_UDP) {
struct udphdr udph = {};
bpf_probe_read_kernel(&udph, sizeof(udph), head + transport_header);
event->src_port = ntohs(udph.source);
event->dst_port = ntohs(udph.dest);
} else if (iph.protocol == IPPROTO_ICMP) {
struct icmphdr icmph = {};
bpf_probe_read_kernel(&icmph, sizeof(icmph), head + transport_header);
event->icmp_type = icmph.type;
event->icmp_code = icmph.code;
}
return 1;
}
if (eth_proto == htons(ETH_P_IPV6)) {
struct ipv6hdr ip6h = {};
bpf_probe_read_kernel(&ip6h, sizeof(ip6h), l3_ptr);
event->ip_version = 6;
event->protocol_raw = ip6h.nexthdr;
__builtin_memcpy(event->src_ip, &ip6h.saddr, 16);
__builtin_memcpy(event->dst_ip, &ip6h.daddr, 16);
if (ip6h.nexthdr == IPPROTO_TCP) {
struct tcphdr tcph = {};
unsigned char flags = 0;
bpf_probe_read_kernel(&tcph, sizeof(tcph), head + transport_header);
event->src_port = ntohs(tcph.source);
event->dst_port = ntohs(tcph.dest);
event->tcp_seq = ntohl(tcph.seq);
event->tcp_ack = ntohl(tcph.ack_seq);
bpf_probe_read_kernel(&flags, sizeof(flags), (void *)(head + transport_header + 13));
event->tcp_flags = flags;
} else if (ip6h.nexthdr == IPPROTO_UDP) {
struct udphdr udph = {};
bpf_probe_read_kernel(&udph, sizeof(udph), head + transport_header);
event->src_port = ntohs(udph.source);
event->dst_port = ntohs(udph.dest);
} else if (ip6h.nexthdr == IPPROTO_ICMPV6) {
struct icmp6hdr icmp6 = {};
bpf_probe_read_kernel(&icmp6, sizeof(icmp6), head + transport_header);
event->icmp_type = icmp6.icmp6_type;
event->icmp_code = icmp6.icmp6_code;
}
return 1;
}
return 1;
}
static __always_inline int emit_event(struct pt_regs *ctx, struct sk_buff *skb, __u8 event_type, __u32 reason) {
struct event_t event = {};
event.ts_ns = bpf_ktime_get_ns();
event.event_type = event_type;
event.reason = reason;
if (!fill_ifname(skb, &event)) {
return 0;
}
if (!parse_skb(skb, &event)) {
return 0;
}
events.perf_submit(ctx, &event, sizeof(event));
return 0;
}
int trace_ingress(struct pt_regs *ctx, struct sk_buff *skb) {
return emit_event(ctx, skb, EVENT_INGRESS, 0);
}
int trace_egress(struct pt_regs *ctx, struct sk_buff *skb) {
return emit_event(ctx, skb, EVENT_EGRESS, 0);
}
TRACEPOINT_PROBE(skb, kfree_skb) {
struct sk_buff *skb = (struct sk_buff *)args->skbaddr;
struct event_t event = {};
event.ts_ns = bpf_ktime_get_ns();
event.event_type = EVENT_DROP;
event.reason = args->reason;
if (!fill_ifname(skb, &event)) {
return 0;
}
if (!parse_skb(skb, &event)) {
return 0;
}
events.perf_submit(args, &event, sizeof(event));
return 0;
}
"""
class Event(ct.Structure):
_fields_ = [
("ts_ns", ct.c_ulonglong),
("length", ct.c_uint),
("reason", ct.c_uint),
("eth_type_raw", ct.c_ushort),
("vlan_id", ct.c_ushort),
("src_port", ct.c_ushort),
("dst_port", ct.c_ushort),
("ip_id", ct.c_ushort),
("arp_op", ct.c_ushort),
("protocol_raw", ct.c_uint),
("tcp_seq", ct.c_uint),
("tcp_ack", ct.c_uint),
("event_type", ct.c_ubyte),
("ip_version", ct.c_ubyte),
("icmp_type", ct.c_ubyte),
("icmp_code", ct.c_ubyte),
("tcp_flags", ct.c_ubyte),
("ifname", ct.c_char * 16),
("src_mac", ct.c_ubyte * 6),
("dst_mac", ct.c_ubyte * 6),
("src_ip", ct.c_ubyte * 16),
("dst_ip", ct.c_ubyte * 16),
]
def _mac_to_str(value: Iterable[int]) -> str:
return ":".join(f"{byte:02x}" for byte in value)
def _ip_to_str(ip_version: int, raw: Iterable[int]) -> str | None:
data = bytes(raw)
if ip_version == 4:
try:
return str(ipaddress.IPv4Address(data[:4]))
except ipaddress.AddressValueError:
return None
if ip_version == 6:
try:
return str(ipaddress.IPv6Address(data[:16]))
except ipaddress.AddressValueError:
return None
return None
def _build_packet_uid(payload: dict[str, object]) -> str:
normalized = []
for field in IDENTITY_FIELDS:
value = payload.get(field)
normalized.append("" if value is None else str(value))
return hashlib.sha1("|".join(normalized).encode("utf-8")).hexdigest()
def _event_name(value: int) -> str:
return {1: "ingress", 2: "egress", 3: "drop"}.get(value, "unknown")
def _reason_name(reason: int) -> str:
return f"skb_drop_reason_{reason}"
def _emit_event(cpu: int, data: int, size: int) -> None:
del cpu, size
event = ct.cast(data, ct.POINTER(Event)).contents
iface = bytes(event.ifname).split(b"\x00", 1)[0].decode("utf-8", "replace")
if iface not in TARGET_INTERFACES:
return
payload: dict[str, object] = {
"event_type": _event_name(event.event_type),
"iface": iface,
"length": int(event.length),
"src_mac": _mac_to_str(event.src_mac),
"dst_mac": _mac_to_str(event.dst_mac),
"eth_type_raw": int(event.eth_type_raw) or None,
"vlan_id": int(event.vlan_id) or None,
"src_ip": _ip_to_str(int(event.ip_version), event.src_ip),
"dst_ip": _ip_to_str(int(event.ip_version), event.dst_ip),
"protocol_raw": int(event.protocol_raw) or None,
"src_port": int(event.src_port) or None,
"dst_port": int(event.dst_port) or None,
"ip_id": int(event.ip_id) or None,
"arp_op": int(event.arp_op) or None,
"icmp_type": int(event.icmp_type) or None,
"icmp_code": int(event.icmp_code) or None,
"tcp_seq": int(event.tcp_seq) or None,
"tcp_ack": int(event.tcp_ack) or None,
"tcp_flags": int(event.tcp_flags) or None,
"reason": _reason_name(int(event.reason)) if event.event_type == 3 else None,
"reason_code": int(event.reason) if event.event_type == 3 else None,
}
payload["packet_uid"] = _build_packet_uid(payload)
print(json.dumps(payload, separators=(",", ":")), flush=True)
def _attach_kprobe_first(bpf: BPF, symbols: list[str], fn_name: str) -> str:
for symbol in symbols:
try:
bpf.attach_kprobe(event=symbol, fn_name=fn_name)
return symbol
except Exception:
continue
raise RuntimeError(f"Failed to attach {fn_name} to any of {symbols}")
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="eBPF bridge telemetry collector")
parser.add_argument("--ifaces", required=True, help="Comma-separated list of interfaces to keep")
return parser.parse_args()
def _sigterm(_signum: int, _frame: object) -> None:
raise KeyboardInterrupt
def main() -> int:
args = _parse_args()
global TARGET_INTERFACES
TARGET_INTERFACES = {iface.strip() for iface in args.ifaces.split(",") if iface.strip()}
if not TARGET_INTERFACES:
print("No interfaces provided", file=sys.stderr)
return 1
signal.signal(signal.SIGTERM, _sigterm)
signal.signal(signal.SIGINT, _sigterm)
bpf = BPF(text=BPF_SOURCE)
ingress_symbol = _attach_kprobe_first(
bpf,
["__netif_receive_skb_core", "netif_receive_skb", "__netif_receive_skb_one_core"],
"trace_ingress",
)
egress_symbol = _attach_kprobe_first(bpf, ["dev_queue_xmit"], "trace_egress")
print(
json.dumps(
{
"status": "collector_started",
"ifaces": sorted(TARGET_INTERFACES),
"ingress_symbol": ingress_symbol,
"egress_symbol": egress_symbol,
},
separators=(",", ":"),
),
flush=True,
)
bpf["events"].open_perf_buffer(_emit_event, page_cnt=128)
try:
while True:
bpf.perf_buffer_poll()
except KeyboardInterrupt:
return 0
TARGET_INTERFACES: set[str] = set()
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,44 @@
"""Helpers for stable packet identity across AF_PACKET and eBPF events."""
from __future__ import annotations
import hashlib
from typing import Any, Dict
IDENTITY_FIELDS = (
"src_mac",
"dst_mac",
"eth_type_raw",
"vlan_id",
"src_ip",
"dst_ip",
"protocol_raw",
"src_port",
"dst_port",
"length",
"ip_id",
"icmp_type",
"icmp_code",
"arp_op",
"tcp_seq",
"tcp_ack",
"tcp_flags",
)
def build_packet_uid(fields: Dict[str, Any]) -> str:
"""Build a deterministic packet identifier from selected L2-L4 fields."""
normalized = []
for field in IDENTITY_FIELDS:
value = fields.get(field)
if isinstance(value, bytes):
value = value.hex()
normalized.append("" if value is None else str(value))
digest = hashlib.sha1("|".join(normalized).encode("utf-8")).hexdigest()
return digest
def minimal_identity_dict(fields: Dict[str, Any]) -> Dict[str, Any]:
"""Return only the identity-relevant subset of packet fields."""
return {field: fields.get(field) for field in IDENTITY_FIELDS}

View File

@@ -0,0 +1,312 @@
"""Aggregate AF_PACKET observations and kernel telemetry into one packet record."""
from __future__ import annotations
import asyncio
import logging
import threading
import time
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
import src.shared_objects as shared_objects
from src.utilities.packet_identity import build_packet_uid
logger = logging.getLogger("packet_tracker")
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
class PacketTracker:
"""Deduplicate packet observations and persist one upserted row per packet."""
def __init__(
self,
finalize_delay_seconds: float = 0.25,
retention_seconds: float = 10.0,
min_flush_interval_seconds: float = 0.05,
):
self._finalize_delay_seconds = finalize_delay_seconds
self._retention_seconds = retention_seconds
self._min_flush_interval_seconds = min_flush_interval_seconds
self._entries: Dict[str, Dict[str, Any]] = {}
self._lock = threading.Lock()
self._stop_event = threading.Event()
self._thread = threading.Thread(target=self._run, daemon=True, name="packet-tracker")
self._thread.start()
def stop(self) -> None:
self._stop_event.set()
self._thread.join(timeout=2)
def observe_packet(self, pkt_info: Dict[str, Any]) -> str:
"""Merge parsed packet information into a pending packet entry."""
now_ts = time.time()
packet_uid = pkt_info.get("packet_uid") or build_packet_uid(pkt_info)
pkt_info["packet_uid"] = packet_uid
with self._lock:
entry = self._entries.get(packet_uid)
if entry is None:
entry = self._new_entry(packet_uid, now_ts)
self._entries[packet_uid] = entry
self._merge_packet_info(entry, pkt_info, now_ts)
self._maybe_promote_reject_from_reply(pkt_info, now_ts)
self._maybe_mark_complete(entry)
return packet_uid
def observe_telemetry(self, event: Dict[str, Any]) -> Optional[str]:
"""Merge ingress/egress/verdict telemetry into a pending packet entry."""
packet_uid = event.get("packet_uid")
if not packet_uid:
try:
packet_uid = build_packet_uid(event)
except Exception:
logger.debug("Telemetry event missing packet identity: %s", event)
return None
now_ts = time.time()
with self._lock:
entry = self._entries.get(packet_uid)
if entry is None:
entry = self._new_entry(packet_uid, now_ts)
self._entries[packet_uid] = entry
payload = entry["payload"]
payload["packet_uid"] = packet_uid
payload["telemetry_metadata"] = event
payload["last_observed_at"] = now_ts
for key, value in event.items():
if value is None or key in {"event_type", "reason", "reason_code", "iface", "packet_uid"}:
continue
if payload.get(key) is None:
payload[key] = value
event_type = event.get("event_type")
iface = event.get("iface")
if iface:
observed_ifaces = payload.setdefault("observed_ifaces", [])
if iface not in observed_ifaces:
observed_ifaces.append(iface)
if event_type == "ingress":
payload["ingress_if"] = iface
payload["iface"] = iface
payload["ingress_seen_at"] = _utcnow()
payload["direction"] = "ingress"
elif event_type == "egress":
payload["egress_if"] = iface
payload["egress_seen_at"] = _utcnow()
payload["direction"] = "forwarded"
payload["verdict"] = "accept"
payload["verdict_reason"] = "egress-observed"
payload["verdict_confidence"] = "high"
payload["verdict_seen_at"] = _utcnow()
elif event_type == "drop":
payload["verdict"] = "drop"
payload["verdict_reason"] = event.get("reason") or "kfree_skb"
payload["verdict_confidence"] = "high"
payload["verdict_seen_at"] = _utcnow()
payload["direction"] = "dropped"
elif event_type == "reject":
payload["verdict"] = "reject"
payload["verdict_reason"] = event.get("reason") or "netfilter-reject"
payload["verdict_confidence"] = event.get("verdict_confidence") or "medium"
payload["verdict_seen_at"] = _utcnow()
payload["direction"] = "rejected"
entry["last_observed_at"] = now_ts
entry["dirty"] = True
self._maybe_mark_complete(entry)
return packet_uid
def _new_entry(self, packet_uid: str, now_ts: float) -> Dict[str, Any]:
return {
"packet_uid": packet_uid,
"payload": {
"packet_uid": packet_uid,
"observed_ifaces": [],
"verdict": "pending",
"verdict_reason": None,
"verdict_confidence": None,
"telemetry_metadata": None,
},
"persisted": False,
"dirty": True,
"finalized": False,
"created_at": now_ts,
"last_observed_at": now_ts,
"last_persisted_at": 0.0,
}
def _merge_packet_info(self, entry: Dict[str, Any], pkt_info: Dict[str, Any], now_ts: float) -> None:
payload = entry["payload"]
changed = False
for key, value in pkt_info.items():
if key == "observed_ifaces":
continue
if value is None:
continue
if key == "raw" and payload.get("raw") is not None:
continue
if payload.get(key) == value:
continue
payload[key] = value
changed = True
iface = pkt_info.get("iface")
if iface:
observed_ifaces = payload.setdefault("observed_ifaces", [])
if iface not in observed_ifaces:
observed_ifaces.append(iface)
changed = True
if not payload.get("ingress_if"):
payload["ingress_if"] = iface
payload["iface"] = iface
payload["ingress_seen_at"] = _utcnow()
changed = True
payload["last_observed_at"] = now_ts
entry["last_observed_at"] = now_ts
entry["dirty"] = entry["dirty"] or changed
def _maybe_promote_reject_from_reply(self, pkt_info: Dict[str, Any], now_ts: float) -> None:
reject_reason = None
tcp_flags = pkt_info.get("tcp_flags")
if pkt_info.get("protocol_raw") == 6 and tcp_flags is not None and int(tcp_flags) & 0x04:
reject_reason = "tcp-rst-observed"
match = self._find_recent_drop(
src_ip=pkt_info.get("dst_ip"),
dst_ip=pkt_info.get("src_ip"),
protocol_raw=6,
src_port=pkt_info.get("dst_port"),
dst_port=pkt_info.get("src_port"),
)
elif pkt_info.get("protocol_raw") == 1 and pkt_info.get("icmp_type") == 3:
reject_reason = "icmp-unreachable-observed"
match = self._find_recent_drop(
src_ip=pkt_info.get("icmp_embedded_src_ip"),
dst_ip=pkt_info.get("icmp_embedded_dst_ip"),
protocol_raw=pkt_info.get("icmp_embedded_protocol"),
src_port=pkt_info.get("icmp_embedded_src_port"),
dst_port=pkt_info.get("icmp_embedded_dst_port"),
)
else:
return
if match is None:
return
payload = match["payload"]
if payload.get("verdict") != "drop":
return
payload["verdict"] = "reject"
payload["verdict_reason"] = reject_reason
payload["verdict_confidence"] = "medium"
payload["verdict_seen_at"] = _utcnow()
payload["direction"] = "rejected"
match["last_observed_at"] = now_ts
match["dirty"] = True
match["finalized"] = True
def _find_recent_drop(
self,
src_ip: Any,
dst_ip: Any,
protocol_raw: Any,
src_port: Any,
dst_port: Any,
) -> Optional[Dict[str, Any]]:
if not src_ip or not dst_ip or protocol_raw is None:
return None
cutoff = time.time() - 1.0
for entry in self._entries.values():
payload = entry["payload"]
if entry["last_observed_at"] < cutoff:
continue
if payload.get("verdict") != "drop":
continue
if payload.get("src_ip") != src_ip or payload.get("dst_ip") != dst_ip:
continue
if payload.get("protocol_raw") != protocol_raw:
continue
if payload.get("src_port") != src_port or payload.get("dst_port") != dst_port:
continue
return entry
return None
def _maybe_mark_complete(self, entry: Dict[str, Any]) -> None:
payload = entry["payload"]
if payload.get("verdict") in {"accept", "drop", "reject"}:
entry["finalized"] = True
def _run(self) -> None:
while not self._stop_event.is_set():
time.sleep(0.05)
due_entries: List[Dict[str, Any]] = []
expired_uids: List[str] = []
now_ts = time.time()
with self._lock:
for packet_uid, entry in list(self._entries.items()):
age = now_ts - entry["last_observed_at"]
if not entry["finalized"] and age >= self._finalize_delay_seconds:
entry["finalized"] = True
if entry["payload"].get("verdict") == "pending":
entry["payload"]["verdict"] = "unknown"
entry["payload"]["verdict_reason"] = "timeout"
entry["payload"]["verdict_confidence"] = "low"
entry["payload"]["verdict_seen_at"] = _utcnow()
entry["payload"]["direction"] = "observed"
entry["dirty"] = True
should_flush = entry["dirty"] and (
not entry["persisted"]
or entry["finalized"]
or (now_ts - entry["last_persisted_at"]) >= self._min_flush_interval_seconds
)
if should_flush:
due_entries.append(
{
"packet_uid": entry["packet_uid"],
"payload": dict(entry["payload"]),
}
)
elif entry["persisted"] and age >= self._retention_seconds:
expired_uids.append(packet_uid)
for packet_uid in expired_uids:
self._entries.pop(packet_uid, None)
for entry in due_entries:
self._persist(entry)
def _persist(self, entry: Dict[str, Any]) -> None:
payload = dict(entry["payload"])
if payload.get("observed_ifaces") == []:
payload["observed_ifaces"] = None
web_loop = getattr(shared_objects, "web_loop", None)
web_db = getattr(shared_objects, "db", None)
if web_loop is None or web_db is None:
return
try:
fut = asyncio.run_coroutine_threadsafe(web_db.upsert_packet(payload), web_loop)
fut.result(timeout=2.0)
with self._lock:
current = self._entries.get(entry["packet_uid"])
if current is not None:
current["persisted"] = True
current["dirty"] = False
current["last_persisted_at"] = time.time()
except Exception:
logger.exception("Failed to persist packet %s", entry["packet_uid"])
packet_tracker = PacketTracker()

View File

@@ -84,8 +84,12 @@ function colorForName(name: string) {
return result; return result;
} }
/** Parse iface field into an array of strings */ function packetKey(packet: PacketRow) {
function parseIfaces(ifaceField?: string | string[]) { return String(packet.packet_uid ?? packet.id ?? `${packet.iface ?? 'if'}:${packet.timestamp ?? ''}`);
}
function parseIfaces(ifaceField?: string | string[] | null, observedIfaces?: string[] | null) {
if (Array.isArray(observedIfaces) && observedIfaces.length > 0) return observedIfaces;
if (!ifaceField) return []; if (!ifaceField) return [];
if (Array.isArray(ifaceField)) return ifaceField; if (Array.isArray(ifaceField)) return ifaceField;
return String(ifaceField) return String(ifaceField)
@@ -131,14 +135,32 @@ export default function PacketViewer(): ReactElement {
const [selectedPacket, setSelectedPacket] = useState<PacketRow | null>(null); const [selectedPacket, setSelectedPacket] = useState<PacketRow | null>(null);
const queuedDuringPause = useRef<PacketRow[]>([]); const queuedDuringPause = useRef<PacketRow[]>([]);
// helper: prepend new packets (newest first). Keep list capped.
const pushNew = useCallback((newPkts: PacketRow[] | PacketRow) => { const pushNew = useCallback((newPkts: PacketRow[] | PacketRow) => {
setPackets((prev) => { setPackets((prev) => {
const arr = Array.isArray(newPkts) ? newPkts : [newPkts]; const arr = Array.isArray(newPkts) ? newPkts : [newPkts];
const normalized = arr.map((p) => p); const next = [...prev];
const merged = [...normalized, ...prev];
if (merged.length > MAX_PACKETS) return merged.slice(0, MAX_PACKETS); for (const packet of arr) {
return merged; const key = packetKey(packet);
const index = next.findIndex((item) => packetKey(item) === key);
if (index >= 0) {
next[index] = { ...next[index], ...packet };
} else {
next.unshift(packet);
}
}
next.sort((left, right) => {
const leftId = Number(left.id ?? 0);
const rightId = Number(right.id ?? 0);
if (!Number.isNaN(leftId) && !Number.isNaN(rightId) && leftId !== rightId) {
return rightId - leftId;
}
return String(right.timestamp ?? '').localeCompare(String(left.timestamp ?? ''));
});
if (next.length > MAX_PACKETS) return next.slice(0, MAX_PACKETS);
return next;
}); });
}, []); }, []);
@@ -284,7 +306,7 @@ export default function PacketViewer(): ReactElement {
key: 'iface', key: 'iface',
width: 100, width: 100,
render: (_: any, rec: PacketRow) => { render: (_: any, rec: PacketRow) => {
const ifaces = parseIfaces(rec.iface); const ifaces = parseIfaces(rec.iface, rec.observed_ifaces);
if (ifaces.length === 0) return <Text type="secondary">-</Text>; if (ifaces.length === 0) return <Text type="secondary">-</Text>;
return ( return (
<Space wrap size={[6, 6]}> <Space wrap size={[6, 6]}>
@@ -301,10 +323,38 @@ export default function PacketViewer(): ReactElement {
); );
}, },
}, },
{
title: 'Path',
key: 'path',
width: 150,
render: (_: any, rec: PacketRow) => (
<Space direction="vertical" size={0}>
<Text>{rec.ingress_if ?? '-'}</Text>
<Text type="secondary">{rec.egress_if ?? '-'}</Text>
</Space>
),
},
{
title: 'Verdict',
key: 'verdict',
width: 120,
render: (_: any, rec: PacketRow) => {
const verdict = rec.verdict ?? '-';
const confidence = rec.verdict_confidence ? ` (${rec.verdict_confidence})` : '';
return (
<Space direction="vertical" size={0}>
<Text>{verdict}</Text>
<Text type="secondary" style={{ fontSize: 12 }}>
{(rec.verdict_reason ?? '') + confidence}
</Text>
</Space>
);
},
},
{ {
title: 'MAC (src → dst)', title: 'MAC (src → dst)',
key: 'macs', key: 'macs',
width: 100, width: 180,
render: (_: any, rec: PacketRow) => ( render: (_: any, rec: PacketRow) => (
<div> <div>
<div> <div>
@@ -445,7 +495,14 @@ export default function PacketViewer(): ReactElement {
<Switch checked={!paused} onChange={(checked) => handlePauseToggle(!checked ? true : false)} /> <Switch checked={!paused} onChange={(checked) => handlePauseToggle(!checked ? true : false)} />
<Button icon={<SyncOutlined />} onClick={handleRefresh} loading={statusLoading} /> <Button icon={<SyncOutlined />} onClick={handleRefresh} loading={statusLoading} />
<Button onClick={() => clearPackets()} danger icon={<DeleteOutlined />} /> <Button
onClick={async () => {
await clearPackets();
setPackets([]);
}}
danger
icon={<DeleteOutlined />}
/>
</Space> </Space>
</Col> </Col>
</Row> </Row>
@@ -461,9 +518,7 @@ export default function PacketViewer(): ReactElement {
</div> </div>
) : ( ) : (
<Table <Table
rowKey={(rec: PacketRow) => rowKey={(rec: PacketRow) => packetKey(rec)}
String(rec.id ?? `${rec.iface ?? 'if'}:${rec.timestamp ?? ''}:${Math.random()}`)
}
dataSource={packets} dataSource={packets}
columns={columns as any} columns={columns as any}
pagination={{ pageSize: 25 }} pagination={{ pageSize: 25 }}

View File

@@ -1,7 +1,11 @@
export interface PacketRow { export interface PacketRow {
id?: number | string; id?: number | string;
timestamp?: string; timestamp?: string;
packet_uid?: string;
iface?: string | string[]; iface?: string | string[];
ingress_if?: string | null;
egress_if?: string | null;
observed_ifaces?: string[] | null;
src_mac?: string | null; src_mac?: string | null;
dst_mac?: string | null; dst_mac?: string | null;
eth_type?: string | number | null; eth_type?: string | number | null;
@@ -21,6 +25,14 @@ export interface PacketRow {
app_is_encrypted?: boolean | null; app_is_encrypted?: boolean | null;
app_risk_score?: number | null; app_risk_score?: number | null;
dpi_metadata?: Record<string, unknown> | null; dpi_metadata?: Record<string, unknown> | null;
telemetry_metadata?: Record<string, unknown> | null;
direction?: string | null;
verdict?: string | null;
verdict_reason?: string | null;
verdict_confidence?: string | null;
ingress_seen_at?: string | null;
egress_seen_at?: string | null;
verdict_seen_at?: string | null;
[key: string]: unknown; [key: string]: unknown;
} }

View File

@@ -25,8 +25,9 @@ PYTHON_VERSION="3" # aktuelle Python 3 Version
echo "==> Update & Upgrade" echo "==> Update & Upgrade"
apt update && apt upgrade -y apt update && apt upgrade -y
apt install -y git curl build-essential nginx python3 python3-pip python3-venv unzip wget python3-dev \ apt install -y git curl build-essential nginx python3 python3-pip python3-venv unzip wget python3-dev \
libnetfilter-queue-dev libnfnetlink-dev libpcap-dev autoconf automake libtool pkg-config libjson-c-dev \ libpcap-dev autoconf automake libtool pkg-config libjson-c-dev gettext flex bison libnuma-dev \
gettext flex bison libnuma-dev libpcre2-dev libmaxminddb-dev librrd-dev libpcre2-dev libmaxminddb-dev librrd-dev python3-bpfcc bpfcc-tools linux-headers-generic
apt install -y linux-headers-$(uname -r) || apt install -y linux-headers-generic
# ----------------------------- # -----------------------------
# Node.js installieren (LTS) # Node.js installieren (LTS)
@@ -64,81 +65,6 @@ python3 -m venv venv
source venv/bin/activate source venv/bin/activate
pip install --upgrade pip pip install --upgrade pip
# -----------------------------
# nDPI installieren (system lib + python bindings)
# -----------------------------
NDPIDIR="/tmp/nDPI"
patch_ndpi_pack_on_if_needed() {
local py_dir="$1"
if [ -f "$py_dir/pyproject.toml" ] && grep -q "PACK_ON" "$py_dir/pyproject.toml"; then
sed -i 's/PACK_ON/"0.0.0"/g' "$py_dir/pyproject.toml"
echo "==> Applied PACK_ON workaround in pyproject.toml"
fi
if [ -f "$py_dir/setup.cfg" ] && grep -q "PACK_ON" "$py_dir/setup.cfg"; then
sed -i 's/PACK_ON/0.0.0/g' "$py_dir/setup.cfg"
echo "==> Applied PACK_ON workaround in setup.cfg"
fi
if [ -f "$py_dir/setup.py" ] && grep -q "PACK_ON" "$py_dir/setup.py"; then
sed -i "s/PACK_ON/'0.0.0'/g" "$py_dir/setup.py"
echo "==> Applied PACK_ON workaround in setup.py"
fi
}
if ! ldconfig -p | grep -q "libndpi"; then
echo "==> Build and install nDPI library"
rm -rf "$NDPIDIR"
git clone --depth 1 --branch dev https://github.com/ntop/nDPI.git "$NDPIDIR"
cd "$NDPIDIR"
./autogen.sh
./configure
make -j"$(nproc)"
make install
ldconfig
else
echo "==> nDPI library already installed"
# Keep python bindings source in sync with official instructions.
if [ ! -d "$NDPIDIR/python" ]; then
rm -rf "$NDPIDIR"
git clone --depth 1 --branch dev https://github.com/ntop/nDPI.git "$NDPIDIR"
fi
fi
if [ -d "$NDPIDIR/python" ]; then
echo "==> Install nDPI Python bindings"
cd "$NDPIDIR/python"
python3 -m pip install --upgrade pip
if ! python3 -m pip install -r dev_requirements.txt; then
echo "==> WARNING: failed to install nDPI python dev_requirements"
fi
if ! python3 -m pip install .; then
echo "==> Initial nDPI pip install failed; retrying with PACK_ON workaround"
patch_ndpi_pack_on_if_needed "$NDPIDIR/python"
if ! python3 -m pip install .; then
echo "==> WARNING: nDPI Python bindings install failed, backend will run without DPI enrichment"
fi
fi
if python - <<'PY'
import importlib
mods = []
for name in ("ndpi", "pyndpi", "nDPI"):
try:
m = importlib.import_module(name)
mods.append((name, getattr(m, "__file__", "unknown")))
except Exception:
pass
if not mods:
raise SystemExit(1)
print("nDPI Python modules:", mods)
PY
then
echo "==> nDPI Python import check passed"
else
echo "==> WARNING: nDPI installed but import failed in backend venv"
fi
cd "$BACKEND_DIR"
fi
cd "$BACKEND_DIR" cd "$BACKEND_DIR"
pip install -r requirements.txt pip install -r requirements.txt
deactivate deactivate
@@ -266,19 +192,6 @@ FRONTEND_DIR="$APP_DIR/frontend"
BACKEND_DIR="$APP_DIR/backend" BACKEND_DIR="$APP_DIR/backend"
BACKEND_SERVICE="mitm-backend" BACKEND_SERVICE="mitm-backend"
patch_ndpi_pack_on_if_needed() {
local py_dir="$1"
if [ -f "$py_dir/pyproject.toml" ] && grep -q "PACK_ON" "$py_dir/pyproject.toml"; then
sed -i 's/PACK_ON/"0.0.0"/g' "$py_dir/pyproject.toml"
fi
if [ -f "$py_dir/setup.cfg" ] && grep -q "PACK_ON" "$py_dir/setup.cfg"; then
sed -i 's/PACK_ON/0.0.0/g' "$py_dir/setup.cfg"
fi
if [ -f "$py_dir/setup.py" ] && grep -q "PACK_ON" "$py_dir/setup.py"; then
sed -i "s/PACK_ON/'0.0.0'/g" "$py_dir/setup.py"
fi
}
cd $APP_DIR cd $APP_DIR
git reset --hard git reset --hard
git pull git pull
@@ -291,35 +204,6 @@ npm run build
# Backend Dependencies # Backend Dependencies
cd $BACKEND_DIR cd $BACKEND_DIR
source venv/bin/activate source venv/bin/activate
NDPIDIR="/tmp/nDPI"
if [ ! -d "$NDPIDIR/python" ]; then
rm -rf "$NDPIDIR"
git clone --depth 1 https://github.com/ntop/nDPI.git "$NDPIDIR"
fi
if [ -d "/tmp/nDPI/python" ]; then
cd "$NDPIDIR/python"
python3 -m pip install --upgrade pip
python3 -m pip install -r dev_requirements.txt || true
if ! python3 -m pip install .; then
echo "WARNING: Initial nDPI install failed during runner build; retry with PACK_ON workaround"
patch_ndpi_pack_on_if_needed "$NDPIDIR/python"
if ! python3 -m pip install .; then
echo "WARNING: nDPI Python bindings install failed during runner build"
fi
fi
python - <<'PY' || echo "WARNING: nDPI import check failed during runner build"
import importlib
for name in ("ndpi", "pyndpi", "nDPI"):
try:
m = importlib.import_module(name)
print("nDPI import OK:", name, getattr(m, "__file__", "unknown"))
raise SystemExit(0)
except Exception:
pass
raise SystemExit(1)
PY
cd $BACKEND_DIR
fi
pip install -r requirements.txt pip install -r requirements.txt
deactivate deactivate

View File

@@ -31,29 +31,33 @@ echo "[4] Restarting PostgreSQL…"
sudo systemctl restart postgresql sudo systemctl restart postgresql
echo "[5] Creating database + user (idempotent)…" echo "[5] Creating database + user (idempotent)…"
sudo -u postgres psql <<EOF if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_roles WHERE rolname = '$DB_USER'" | grep -q 1; then
DO \$\$ sudo -u postgres psql -c "CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';"
BEGIN fi
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '$DB_USER') THEN
CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';
END IF;
IF NOT EXISTS (SELECT FROM pg_database WHERE datname = '$DB_NAME') THEN if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1; then
CREATE DATABASE $DB_NAME OWNER $DB_USER; sudo -u postgres psql -c "CREATE DATABASE $DB_NAME OWNER $DB_USER;"
END IF; fi
END
\$\$;
EOF
echo "[6] Creating packets table (idempotent)…" echo "[6] Creating packets table (idempotent)…"
sudo -u postgres psql -d "$DB_NAME" <<EOF sudo -u postgres psql -d "$DB_NAME" <<EOF
CREATE TABLE IF NOT EXISTS packets ( CREATE TABLE IF NOT EXISTS packets (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
timestamp TIMESTAMPTZ DEFAULT NOW(), timestamp TIMESTAMPTZ DEFAULT NOW(),
packet_uid TEXT UNIQUE,
-- Interface metadata -- Interface metadata
iface VARCHAR(64), iface VARCHAR(64),
ingress_if VARCHAR(64),
egress_if VARCHAR(64),
observed_ifaces TEXT[],
direction VARCHAR(64), direction VARCHAR(64),
verdict VARCHAR(32),
verdict_reason VARCHAR(128),
verdict_confidence VARCHAR(32),
ingress_seen_at TIMESTAMPTZ,
egress_seen_at TIMESTAMPTZ,
verdict_seen_at TIMESTAMPTZ,
-- Ethernet -- Ethernet
src_mac MACADDR, src_mac MACADDR,
@@ -84,11 +88,13 @@ CREATE TABLE IF NOT EXISTS packets (
app_is_encrypted BOOLEAN, app_is_encrypted BOOLEAN,
app_risk_score INTEGER, app_risk_score INTEGER,
dpi_metadata JSONB, dpi_metadata JSONB,
telemetry_metadata JSONB,
-- Full packet dump -- Full packet dump
raw BYTEA raw BYTEA
); );
ALTER TABLE packets ADD COLUMN IF NOT EXISTS packet_uid TEXT;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_protocol VARCHAR(128); ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_protocol VARCHAR(128);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_master_protocol VARCHAR(128); ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_master_protocol VARCHAR(128);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_category VARCHAR(128); ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_category VARCHAR(128);
@@ -97,6 +103,17 @@ ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_hostname VARCHAR(255);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_is_encrypted BOOLEAN; ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_is_encrypted BOOLEAN;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_risk_score INTEGER; ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_risk_score INTEGER;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS dpi_metadata JSONB; ALTER TABLE packets ADD COLUMN IF NOT EXISTS dpi_metadata JSONB;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS ingress_if VARCHAR(64);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS egress_if VARCHAR(64);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS observed_ifaces TEXT[];
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict VARCHAR(32);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_reason VARCHAR(128);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_confidence VARCHAR(32);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS ingress_seen_at TIMESTAMPTZ;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS egress_seen_at TIMESTAMPTZ;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_seen_at TIMESTAMPTZ;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS telemetry_metadata JSONB;
CREATE UNIQUE INDEX IF NOT EXISTS idx_packets_packet_uid ON packets(packet_uid);
EOF EOF
@@ -121,6 +138,7 @@ CREATE INDEX IF NOT EXISTS idx_packets_timestamp ON packets(timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_packets_src_ip ON packets(src_ip); CREATE INDEX IF NOT EXISTS idx_packets_src_ip ON packets(src_ip);
CREATE INDEX IF NOT EXISTS idx_packets_app_protocol ON packets(app_protocol); CREATE INDEX IF NOT EXISTS idx_packets_app_protocol ON packets(app_protocol);
CREATE INDEX IF NOT EXISTS idx_packets_app_hostname ON packets(app_hostname); CREATE INDEX IF NOT EXISTS idx_packets_app_hostname ON packets(app_hostname);
CREATE INDEX IF NOT EXISTS idx_packets_verdict ON packets(verdict);
EOF EOF
echo "Done. PostgreSQL is ready for LAN + localhost connections." echo "Done. PostgreSQL is ready for LAN + localhost connections."