diff --git a/backend/src/config.py b/backend/src/config.py index 8aae24e..724fd34 100644 --- a/backend/src/config.py +++ b/backend/src/config.py @@ -56,6 +56,7 @@ class BackendSettings: telemetry_reader_join_timeout_seconds: float tshark_enabled: bool tshark_display_filter: str + tshark_try_heuristic_first: bool tshark_cache_ttl_seconds: float tshark_match_window_ms: int tshark_reader_join_timeout_seconds: float @@ -89,6 +90,7 @@ def load_settings() -> BackendSettings: telemetry_reader_join_timeout_seconds=_env_float("BACKEND_TELEMETRY_READER_JOIN_TIMEOUT_SECONDS", 2.0), tshark_enabled=_env_bool("BACKEND_TSHARK_ENABLED", True), tshark_display_filter=_env_str("BACKEND_TSHARK_DISPLAY_FILTER", ""), + tshark_try_heuristic_first=_env_bool("BACKEND_TSHARK_TRY_HEURISTIC_FIRST", True), tshark_cache_ttl_seconds=_env_float("BACKEND_TSHARK_CACHE_TTL_SECONDS", 5.0), tshark_match_window_ms=_env_int("BACKEND_TSHARK_MATCH_WINDOW_MS", 5_000), tshark_reader_join_timeout_seconds=_env_float("BACKEND_TSHARK_READER_JOIN_TIMEOUT_SECONDS", 2.0), diff --git a/backend/src/network_sniffer.py b/backend/src/network_sniffer.py index 6b9db47..7b05cda 100644 --- a/backend/src/network_sniffer.py +++ b/backend/src/network_sniffer.py @@ -174,6 +174,15 @@ def _merge_enrichment(pkt_info: PacketInfo, enrichment: Dict[str, Any]) -> None: elif current is None: pkt_info["dpi_metadata"] = value continue + if key == "capture_sources": + current_sources = list(pkt_info.get("capture_sources") or []) + incoming_sources = [str(source) for source in value if source] + merged_sources = current_sources[:] + for source in incoming_sources: + if source not in merged_sources: + merged_sources.append(source) + pkt_info["capture_sources"] = merged_sources + continue if pkt_info.get(key) is None: pkt_info[key] = value diff --git a/backend/src/utilities/database.py b/backend/src/utilities/database.py index 9f290d0..dfcd6b8 100644 --- a/backend/src/utilities/database.py +++ b/backend/src/utilities/database.py @@ -288,6 +288,7 @@ class DatabasePool: lower_bound = datetime.fromtimestamp(max(observed_at_ms - window_ms, 0) / 1000.0, tz=timezone.utc) upper_bound = datetime.fromtimestamp(max(observed_at_ms + window_ms, 0) / 1000.0, tz=timezone.utc) dpi_metadata = enrichment.get("dpi_metadata") + capture_sources = [str(source) for source in enrichment.get("capture_sources", []) if source] try: async with self._pool.acquire() as conn: @@ -306,7 +307,16 @@ class DatabasePool: WHEN $16::jsonb IS NULL THEN packets.dpi_metadata WHEN packets.dpi_metadata IS NULL THEN $16::jsonb ELSE packets.dpi_metadata || $16::jsonb - END + END, + capture_sources = ( + SELECT ARRAY( + SELECT DISTINCT source + FROM unnest( + COALESCE(packets.capture_sources, ARRAY[]::text[]) || + COALESCE($17::text[], ARRAY[]::text[]) + ) AS source + ) + ) WHERE ip_proto_raw = $1 AND (capture_iface = $2 OR ingress_if = $2 OR egress_if = $2) @@ -324,6 +334,7 @@ class DatabasePool: OR packets.app_hostname IS NULL OR packets.app_is_encrypted IS NULL OR ($16::jsonb IS NOT NULL) + OR (COALESCE(array_length($17::text[], 1), 0) > 0) ) RETURNING * """, @@ -343,6 +354,7 @@ class DatabasePool: enrichment.get("app_hostname"), enrichment.get("app_is_encrypted"), json.dumps(dpi_metadata) if dpi_metadata is not None else None, + capture_sources if capture_sources else None, ) except Exception: logger.exception("DB packet metadata backfill failed") diff --git a/backend/src/utilities/packet_tracker.py b/backend/src/utilities/packet_tracker.py index 791516f..3dbb6d7 100644 --- a/backend/src/utilities/packet_tracker.py +++ b/backend/src/utilities/packet_tracker.py @@ -84,7 +84,11 @@ class PacketTracker: now_ts = time.time() correlation_key = self._ensure_correlation(pkt_info) pkt_info["raw_present"] = pkt_info.get("raw") is not None - pkt_info["capture_sources"] = [pkt_info.get("capture_source") or "af_packet"] + existing_sources = list(pkt_info.get("capture_sources") or []) + primary_source = pkt_info.get("capture_source") or "af_packet" + if primary_source not in existing_sources: + existing_sources.insert(0, primary_source) + pkt_info["capture_sources"] = existing_sources with self._lock: entry = self._entries.get(correlation_key) diff --git a/backend/src/utilities/tshark_manager.py b/backend/src/utilities/tshark_manager.py index c980d19..d877bae 100644 --- a/backend/src/utilities/tshark_manager.py +++ b/backend/src/utilities/tshark_manager.py @@ -22,6 +22,8 @@ _FIELDS: List[str] = [ "frame.time_epoch", "frame.interface_name", "frame.len", + "_ws.col.Protocol", + "_ws.col.Info", "ip.src", "ipv6.src", "ip.dst", @@ -158,6 +160,8 @@ def _parse_line(line: str, fallback_iface: str) -> Optional[Dict[str, Any]]: timestamp = _safe_float(row["frame.time_epoch"]) length = _safe_int(row["frame.len"]) + protocol_col = _safe_text(row["_ws.col.Protocol"]) + info_col = _safe_text(row["_ws.col.Info"]) protocol = _safe_int(row["ip.proto"]) or _safe_int(row["ipv6.nxt"]) src_ip = _safe_text(row["ip.src"]) or _safe_text(row["ipv6.src"]) dst_ip = _safe_text(row["ip.dst"]) or _safe_text(row["ipv6.dst"]) @@ -178,6 +182,8 @@ def _parse_line(line: str, fallback_iface: str) -> Optional[Dict[str, Any]]: "dst_ip": dst_ip, "src_port": src_port, "dst_port": dst_port, + "protocol_col": protocol_col, + "info_col": info_col, "frame_protocols": _safe_text(row["frame.protocols"]), "http": _jsonable( { @@ -216,6 +222,8 @@ def _build_enrichment(event: Dict[str, Any]) -> Dict[str, Any]: tls_meta = dict(event.get("tls") or {}) dns_meta = dict(event.get("dns") or {}) protocols = str(event.get("frame_protocols") or "") + protocol_col = _safe_text(event.get("protocol_col")) + info_col = _safe_text(event.get("info_col")) app_protocol: Optional[str] = None app_category: Optional[str] = None @@ -237,10 +245,14 @@ def _build_enrichment(event: Dict[str, Any]) -> Dict[str, Any]: app_category = "Encrypted" app_hostname = tls_meta.get("server_name") app_is_encrypted = True + elif protocol_col and protocol_col.upper() not in {"TCP", "UDP", "IP", "IPV6", "ETH", "ARP"}: + app_protocol = protocol_col tshark_meta = _jsonable( { "observed_at_ms": event.get("observed_at_ms"), + "protocol": protocol_col, + "info": info_col, "frame_protocols": event.get("frame_protocols"), "length": event.get("length"), } @@ -263,14 +275,23 @@ def _build_enrichment(event: Dict[str, Any]) -> Dict[str, Any]: "app_hostname": app_hostname, "app_is_encrypted": app_is_encrypted, "dpi_metadata": dpi_metadata or None, + "capture_sources": ["tshark"], } def _has_useful_enrichment(enrichment: Dict[str, Any]) -> bool: - return any( - enrichment.get(key) is not None - for key in ("app_protocol", "app_hostname", "app_is_encrypted", "dpi_metadata") - ) + if enrichment.get("app_protocol") is not None: + return True + if enrichment.get("app_hostname") is not None: + return True + dpi_metadata = enrichment.get("dpi_metadata") or {} + if not isinstance(dpi_metadata, dict): + return False + for key in ("http", "tls", "dns"): + value = dpi_metadata.get(key) + if isinstance(value, dict) and value: + return True + return False class TsharkManager: @@ -389,6 +410,9 @@ class TsharkManager: "-E", "occurrence=f", ] + if settings.tshark_try_heuristic_first: + cmd.extend(["-o", "tcp.try_heuristic_first:true"]) + cmd.extend(["-o", "udp.try_heuristic_first:true"]) if settings.tshark_display_filter: cmd.extend(["-Y", settings.tshark_display_filter]) for field in _FIELDS: @@ -485,6 +509,8 @@ class TsharkManager: self._stats["events_total"] += 1 self._stats["last_event_by_iface"][iface] = { "observed_at_ms": event.get("observed_at_ms"), + "protocol_col": event.get("protocol_col"), + "info_col": event.get("info_col"), "protocol": event.get("protocol"), "src_ip": event.get("src_ip"), "dst_ip": event.get("dst_ip"), diff --git a/setup_build_server.sh b/setup_build_server.sh index 17e8394..f01eb4f 100755 --- a/setup_build_server.sh +++ b/setup_build_server.sh @@ -11,7 +11,6 @@ BACKEND_DIR="$APP_DIR/backend" BACKEND_SERVICE="mitm-backend" NGINX_SITE="/etc/nginx/sites-available/mitm-webserver" USER_ROOT="root" -BACKEND_ENV_FILE="$BACKEND_DIR/.env" GITEA_RUNNER_URL="https://gitea.malmert.de//api/v1/repos/marcus/mitm-webserver/actions/runners/register" RUNNER_TOKEN="cThC2xmAZWaOAqRRMENuVVTJckxaHiJxVGx2NCQY" @@ -71,19 +70,6 @@ cd "$BACKEND_DIR" pip install -r requirements.txt deactivate -# ----------------------------- -# Backend Environment Defaults -# ----------------------------- -echo "==> Write backend environment defaults" -cat >"$BACKEND_ENV_FILE" <