marcus/mitm-webserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix eth type and db
All checks were successful
Build and Deploy MITM Webserver / build (push) Successful in 1m40s
Details

Browse Source
This commit is contained in:
malmert
2026-03-07 10:26:51 +01:00
parent 79b2f1e673
commit 161ed0f145
6 changed files with 79 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/Models/packets.py
View File

@@ -12,13 +12,13 @@ class PacketDBModel(BaseModel):
id: Union[int, str]
timestamp: datetime = Field(..., description="Packet timestamp in ISO format.")
packet_uid: str = Field(..., description="Stable packet identity used for upserts/correlation.")
iface: Optional[str] = None
ingress_if: Optional[str] = None
egress_if: Optional[str] = None
observed_ifaces: Optional[list[str]] = None
src_mac: Optional[str] = None
dst_mac: Optional[str] = None
eth_type_raw: Optional[int] = Field(None, description="Numeric Ethernet type from the frame header.")
eth_type: Optional[Union[int, str]] = None
ip_proto_raw: Optional[int] = Field(None, description="Numeric IP protocol / next-header value.")
ip_proto: Optional[Union[int, str]] = None
src_ip: Optional[IPvAnyAddress] = None
dst_ip: Optional[IPvAnyAddress] = None
@@ -36,7 +36,6 @@ class PacketDBModel(BaseModel):
app_risk_score: Optional[int] = Field(None, description="Count/score of detected nDPI risks.")
dpi_metadata: Optional[dict] = Field(None, description="Raw DPI metadata from nDPI.")
telemetry_metadata: Optional[dict] = Field(None, description="Kernel telemetry details from eBPF collector.")
direction: Optional[str] = None
verdict: Optional[str] = None
verdict_reason: Optional[str] = None
verdict_confidence: Optional[str] = None
@@ -51,13 +50,13 @@ class PacketDBModel(BaseModel):
"id": 123,
"timestamp": "2026-03-05T12:34:56.789Z",
"packet_uid": "9f6d3af0d3c81cb20ee8e7d32df7c56414460542",
"iface": "eth0",
"ingress_if": "eth0",
"egress_if": "eth1",
"observed_ifaces": ["eth0", "eth1"],
"src_mac": "aa:bb:cc:dd:ee:ff",
"dst_mac": "11:22:33:44:55:66",
"eth_type_raw": 2048,
"eth_type": "IPv4",
"ip_proto_raw": 6,
"ip_proto": "TCP",
"src_ip": "192.168.1.10",
"dst_ip": "192.168.1.1",
@@ -75,7 +74,6 @@ class PacketDBModel(BaseModel):
"app_risk_score": 0,
"dpi_metadata": {"method": "GET"},
"telemetry_metadata": {"event_type": "egress", "iface": "eth1"},
"direction": "forwarded",
"verdict": "accept",
"verdict_reason": "egress-observed",
"verdict_confidence": "high",

17
backend/src/utilities/database.py
View File

@@ -91,11 +91,8 @@ class DatabasePool:
"""
INSERT INTO packets (
packet_uid,
iface,
ingress_if,
egress_if,
observed_ifaces,
direction,
verdict,
verdict_reason,
verdict_confidence,
@@ -104,10 +101,12 @@ class DatabasePool:
verdict_seen_at,
src_mac,
dst_mac,
eth_type_raw,
eth_type,
vlan_id,
src_ip,
dst_ip,
ip_proto_raw,
ip_proto,
src_port,
dst_port,
@@ -125,14 +124,11 @@ class DatabasePool:
) VALUES(
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,
$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,
$24,$25,$26,$27,$28,$29,$30::jsonb,$31::jsonb,$32
$24,$25,$26,$27,$28,$29::jsonb,$30::jsonb,$31
)
ON CONFLICT (packet_uid) DO UPDATE SET
iface = COALESCE(EXCLUDED.iface, packets.iface),
ingress_if = COALESCE(EXCLUDED.ingress_if, packets.ingress_if),
egress_if = COALESCE(EXCLUDED.egress_if, packets.egress_if),
observed_ifaces = COALESCE(EXCLUDED.observed_ifaces, packets.observed_ifaces),
direction = COALESCE(EXCLUDED.direction, packets.direction),
verdict = COALESCE(EXCLUDED.verdict, packets.verdict),
verdict_reason = COALESCE(EXCLUDED.verdict_reason, packets.verdict_reason),
verdict_confidence = COALESCE(EXCLUDED.verdict_confidence, packets.verdict_confidence),
@@ -141,10 +137,12 @@ class DatabasePool:
verdict_seen_at = COALESCE(EXCLUDED.verdict_seen_at, packets.verdict_seen_at),
src_mac = COALESCE(EXCLUDED.src_mac, packets.src_mac),
dst_mac = COALESCE(EXCLUDED.dst_mac, packets.dst_mac),
eth_type_raw = COALESCE(EXCLUDED.eth_type_raw, packets.eth_type_raw),
eth_type = COALESCE(EXCLUDED.eth_type, packets.eth_type),
vlan_id = COALESCE(EXCLUDED.vlan_id, packets.vlan_id),
src_ip = COALESCE(EXCLUDED.src_ip, packets.src_ip),
dst_ip = COALESCE(EXCLUDED.dst_ip, packets.dst_ip),
ip_proto_raw = COALESCE(EXCLUDED.ip_proto_raw, packets.ip_proto_raw),
ip_proto = COALESCE(EXCLUDED.ip_proto, packets.ip_proto),
src_port = COALESCE(EXCLUDED.src_port, packets.src_port),
dst_port = COALESCE(EXCLUDED.dst_port, packets.dst_port),
@@ -162,11 +160,8 @@ class DatabasePool:
RETURNING id, timestamp
""",
pkt_info["packet_uid"],
pkt_info.get("iface"),
pkt_info.get("ingress_if"),
pkt_info.get("egress_if"),
pkt_info.get("observed_ifaces"),
pkt_info.get("direction"),
pkt_info.get("verdict"),
pkt_info.get("verdict_reason"),
pkt_info.get("verdict_confidence"),
@@ -175,10 +170,12 @@ class DatabasePool:
pkt_info.get("verdict_seen_at"),
pkt_info.get("src_mac"),
pkt_info.get("dst_mac"),
pkt_info.get("eth_type_raw"),
_db_text(pkt_info.get("eth_type")),
pkt_info.get("vlan_id"),
pkt_info.get("src_ip"),
pkt_info.get("dst_ip"),
pkt_info.get("protocol_raw"),
_db_text(pkt_info.get("protocol_name") or pkt_info.get("protocol")),
pkt_info.get("src_port"),
pkt_info.get("dst_port"),

44
backend/src/utilities/packet_tracker.py
View File

@@ -10,6 +10,8 @@ from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
import src.shared_objects as shared_objects
from src.Models.etherType import EtherTypeEnum, ethertype_from_int
from src.Models.ip_protocol import protocol_from_number
from src.utilities.packet_identity import build_packet_uid
logger = logging.getLogger("packet_tracker")
@@ -84,22 +86,27 @@ class PacketTracker:
continue
if payload.get(key) is None:
payload[key] = value
if payload.get("eth_type") is None and payload.get("eth_type_raw") is not None:
try:
payload["eth_type"] = ethertype_from_int(int(payload["eth_type_raw"]))
except Exception:
payload["eth_type"] = EtherTypeEnum.UNKNOWN
if payload.get("protocol") is None and payload.get("protocol_raw") is not None:
try:
payload["protocol"] = protocol_from_number(int(payload["protocol_raw"]))
except Exception:
payload["protocol"] = int(payload["protocol_raw"])
event_type = event.get("event_type")
iface = event.get("iface")
if iface:
observed_ifaces = payload.setdefault("observed_ifaces", [])
if iface not in observed_ifaces:
observed_ifaces.append(iface)
if event_type == "ingress":
payload["ingress_if"] = iface
payload["iface"] = iface
payload["ingress_seen_at"] = _utcnow()
payload["direction"] = "ingress"
elif event_type == "egress":
payload["egress_if"] = iface
payload["egress_seen_at"] = _utcnow()
payload["direction"] = "forwarded"
payload["verdict"] = "accept"
payload["verdict_reason"] = "egress-observed"
payload["verdict_confidence"] = "high"
@@ -109,13 +116,11 @@ class PacketTracker:
payload["verdict_reason"] = event.get("reason") or "kfree_skb"
payload["verdict_confidence"] = "high"
payload["verdict_seen_at"] = _utcnow()
payload["direction"] = "dropped"
elif event_type == "reject":
payload["verdict"] = "reject"
payload["verdict_reason"] = event.get("reason") or "netfilter-reject"
payload["verdict_confidence"] = event.get("verdict_confidence") or "medium"
payload["verdict_seen_at"] = _utcnow()
payload["direction"] = "rejected"
entry["last_observed_at"] = now_ts
entry["dirty"] = True
@@ -127,7 +132,6 @@ class PacketTracker:
"packet_uid": packet_uid,
"payload": {
"packet_uid": packet_uid,
"observed_ifaces": [],
"verdict": "pending",
"verdict_reason": None,
"verdict_confidence": None,
@@ -145,7 +149,7 @@ class PacketTracker:
payload = entry["payload"]
changed = False
for key, value in pkt_info.items():
if key == "observed_ifaces":
if key == "iface":
continue
if value is None:
continue
@@ -157,16 +161,10 @@ class PacketTracker:
changed = True
iface = pkt_info.get("iface")
if iface:
observed_ifaces = payload.setdefault("observed_ifaces", [])
if iface not in observed_ifaces:
observed_ifaces.append(iface)
changed = True
if not payload.get("ingress_if"):
payload["ingress_if"] = iface
payload["iface"] = iface
payload["ingress_seen_at"] = _utcnow()
changed = True
if iface and not payload.get("ingress_if"):
payload["ingress_if"] = iface
payload["ingress_seen_at"] = _utcnow()
changed = True
payload["last_observed_at"] = now_ts
entry["last_observed_at"] = now_ts
@@ -208,7 +206,6 @@ class PacketTracker:
payload["verdict_reason"] = reject_reason
payload["verdict_confidence"] = "medium"
payload["verdict_seen_at"] = _utcnow()
payload["direction"] = "rejected"
match["last_observed_at"] = now_ts
match["dirty"] = True
match["finalized"] = True
@@ -262,7 +259,6 @@ class PacketTracker:
entry["payload"]["verdict_reason"] = "timeout"
entry["payload"]["verdict_confidence"] = "low"
entry["payload"]["verdict_seen_at"] = _utcnow()
entry["payload"]["direction"] = "observed"
entry["dirty"] = True
should_flush = entry["dirty"] and (
@@ -288,8 +284,6 @@ class PacketTracker:
def _persist(self, entry: Dict[str, Any]) -> None:
payload = dict(entry["payload"])
if payload.get("observed_ifaces") == []:
payload["observed_ifaces"] = None
web_loop = getattr(shared_objects, "web_loop", None)
web_db = getattr(shared_objects, "db", None)

77
frontend/src/components/PacketViewer.tsx
View File

@@ -85,17 +85,19 @@ function colorForName(name: string) {
}
function packetKey(packet: PacketRow) {
return String(packet.packet_uid ?? packet.id ?? `${packet.iface ?? 'if'}:${packet.timestamp ?? ''}`);
return String(packet.packet_uid ?? packet.id ?? `${packet.ingress_if ?? 'if'}:${packet.timestamp ?? ''}`);
}
function parseIfaces(ifaceField?: string | string[] | null, observedIfaces?: string[] | null) {
if (Array.isArray(observedIfaces) && observedIfaces.length > 0) return observedIfaces;
if (!ifaceField) return [];
if (Array.isArray(ifaceField)) return ifaceField;
return String(ifaceField)
.split(/[,\|;]+/)
.map((s) => s.trim())
.filter(Boolean);
function formatEthType(packet: PacketRow) {
if (packet.eth_type) return String(packet.eth_type);
if (typeof packet.eth_type_raw === 'number') return `0x${packet.eth_type_raw.toString(16)}`;
return '-';
}
function formatIpProto(packet: PacketRow) {
if (packet.ip_proto) return String(packet.ip_proto);
if (typeof packet.ip_proto_raw === 'number') return String(packet.ip_proto_raw);
return '-';
}
/** Format timestamp to European locale with ms */
@@ -206,7 +208,7 @@ export default function PacketViewer(): ReactElement {
} else {
pushNew(data.packets as PacketRow[]);
}
} else if (data && (data.iface || data.raw_b64 || data.id || data.timestamp)) {
} else if (data && (data.packet_uid || data.raw_b64 || data.id || data.timestamp)) {
if (paused) {
queuedDuringPause.current.unshift(data as PacketRow);
} else {
@@ -300,37 +302,32 @@ export default function PacketViewer(): ReactElement {
width: 120,
render: (val: any) => <Text>{formatTimestamp(val)}</Text>,
},
{
title: 'Ifaces',
dataIndex: 'iface',
key: 'iface',
width: 100,
render: (_: any, rec: PacketRow) => {
const ifaces = parseIfaces(rec.iface, rec.observed_ifaces);
if (ifaces.length === 0) return <Text type="secondary">-</Text>;
return (
<Space wrap size={[6, 6]}>
{ifaces.map((name) => {
const { background, color } = colorForName(name);
// tag style: use background and computed text color for accessibility
return (
<Tooltip key={name} title={name}>
<Tag style={{ background, color, fontWeight: 600, borderRadius: 6, fontSize: 15 }}>{name}</Tag>
</Tooltip>
);
})}
</Space>
);
},
},
{
title: 'Path',
key: 'path',
width: 150,
render: (_: any, rec: PacketRow) => (
<Space direction="vertical" size={0}>
<Text>{rec.ingress_if ?? '-'}</Text>
<Text type="secondary">{rec.egress_if ?? '-'}</Text>
<Space wrap size={[6, 6]}>
{[rec.ingress_if, rec.egress_if].filter(Boolean).length > 0 ? (
<>
{rec.ingress_if && (
<Tooltip title={`ingress: ${rec.ingress_if}`}>
<Tag style={{ ...colorForName(rec.ingress_if), fontWeight: 600, borderRadius: 6, fontSize: 15 }}>
{rec.ingress_if}
</Tag>
</Tooltip>
)}
{rec.egress_if && (
<Tooltip title={`egress: ${rec.egress_if}`}>
<Tag style={{ ...colorForName(rec.egress_if), fontWeight: 600, borderRadius: 6, fontSize: 15 }}>
{rec.egress_if}
</Tag>
</Tooltip>
)}
</>
) : (
<Text type="secondary">-</Text>
)}
</Space>
),
},
@@ -366,12 +363,18 @@ export default function PacketViewer(): ReactElement {
</div>
),
},
{
title: 'L2',
key: 'eth_type',
width: 100,
render: (_: any, rec: PacketRow) => <Text>{formatEthType(rec)}</Text>,
},
{
title: 'Protocol',
dataIndex: 'ip_proto',
key: 'ip_proto',
width: 110,
render: (v: any) => <Text>{v ?? '-'}</Text>,
render: (_: any, rec: PacketRow) => <Text>{formatIpProto(rec)}</Text>,
},
{
title: 'App',

5
frontend/src/types/packets.ts
View File

@@ -2,13 +2,13 @@ export interface PacketRow {
id?: number | string;
timestamp?: string;
packet_uid?: string;
iface?: string | string[];
ingress_if?: string | null;
egress_if?: string | null;
observed_ifaces?: string[] | null;
src_mac?: string | null;
dst_mac?: string | null;
eth_type_raw?: number | null;
eth_type?: string | number | null;
ip_proto_raw?: number | null;
ip_proto?: string | number | null;
src_ip?: string | null;
dst_ip?: string | null;
@@ -26,7 +26,6 @@ export interface PacketRow {
app_risk_score?: number | null;
dpi_metadata?: Record<string, unknown> | null;
telemetry_metadata?: Record<string, unknown> | null;
direction?: string | null;
verdict?: string | null;
verdict_reason?: string | null;
verdict_confidence?: string | null;

11
setup_database.sh
View File

@@ -47,11 +47,8 @@ CREATE TABLE IF NOT EXISTS packets (
packet_uid TEXT UNIQUE,
-- Interface metadata
iface VARCHAR(64),
ingress_if VARCHAR(64),
egress_if VARCHAR(64),
observed_ifaces TEXT[],
direction VARCHAR(64),
verdict VARCHAR(32),
verdict_reason VARCHAR(128),
verdict_confidence VARCHAR(32),
@@ -62,6 +59,7 @@ CREATE TABLE IF NOT EXISTS packets (
-- Ethernet
src_mac MACADDR,
dst_mac MACADDR,
eth_type_raw INTEGER,
eth_type VARCHAR(64),
-- VLAN
@@ -70,6 +68,7 @@ CREATE TABLE IF NOT EXISTS packets (
-- IP layer
src_ip INET,
dst_ip INET,
ip_proto_raw INTEGER,
ip_proto VARCHAR(64),
-- Transport layer
@@ -95,6 +94,7 @@ CREATE TABLE IF NOT EXISTS packets (
);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS packet_uid TEXT;
ALTER TABLE packets DROP COLUMN IF EXISTS iface;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_protocol VARCHAR(128);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_master_protocol VARCHAR(128);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_category VARCHAR(128);
@@ -103,9 +103,11 @@ ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_hostname VARCHAR(255);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_is_encrypted BOOLEAN;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_risk_score INTEGER;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS dpi_metadata JSONB;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS eth_type_raw INTEGER;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS ip_proto_raw INTEGER;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS ingress_if VARCHAR(64);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS egress_if VARCHAR(64);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS observed_ifaces TEXT[];
ALTER TABLE packets DROP COLUMN IF EXISTS observed_ifaces;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict VARCHAR(32);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_reason VARCHAR(128);
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_confidence VARCHAR(32);
@@ -113,6 +115,7 @@ ALTER TABLE packets ADD COLUMN IF NOT EXISTS ingress_seen_at TIMESTAMPTZ;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS egress_seen_at TIMESTAMPTZ;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_seen_at TIMESTAMPTZ;
ALTER TABLE packets ADD COLUMN IF NOT EXISTS telemetry_metadata JSONB;
ALTER TABLE packets DROP COLUMN IF EXISTS direction;
CREATE UNIQUE INDEX IF NOT EXISTS idx_packets_packet_uid ON packets(packet_uid);
EOF