tshark test 6

backend/src/config.py

@@ -56,6 +56,7 @@ class BackendSettings:
     telemetry_reader_join_timeout_seconds: float
     tshark_enabled: bool
     tshark_display_filter: str
+    tshark_try_heuristic_first: bool
     tshark_cache_ttl_seconds: float
     tshark_match_window_ms: int
     tshark_reader_join_timeout_seconds: float
@@ -89,6 +90,7 @@ def load_settings() -> BackendSettings:
         telemetry_reader_join_timeout_seconds=_env_float("BACKEND_TELEMETRY_READER_JOIN_TIMEOUT_SECONDS", 2.0),
         tshark_enabled=_env_bool("BACKEND_TSHARK_ENABLED", True),
         tshark_display_filter=_env_str("BACKEND_TSHARK_DISPLAY_FILTER", ""),
+        tshark_try_heuristic_first=_env_bool("BACKEND_TSHARK_TRY_HEURISTIC_FIRST", True),
         tshark_cache_ttl_seconds=_env_float("BACKEND_TSHARK_CACHE_TTL_SECONDS", 5.0),
         tshark_match_window_ms=_env_int("BACKEND_TSHARK_MATCH_WINDOW_MS", 5_000),
         tshark_reader_join_timeout_seconds=_env_float("BACKEND_TSHARK_READER_JOIN_TIMEOUT_SECONDS", 2.0),

backend/src/network_sniffer.py

@@ -174,6 +174,15 @@ def _merge_enrichment(pkt_info: PacketInfo, enrichment: Dict[str, Any]) -> None:
             elif current is None:
                 pkt_info["dpi_metadata"] = value
             continue
+        if key == "capture_sources":
+            current_sources = list(pkt_info.get("capture_sources") or [])
+            incoming_sources = [str(source) for source in value if source]
+            merged_sources = current_sources[:]
+            for source in incoming_sources:
+                if source not in merged_sources:
+                    merged_sources.append(source)
+            pkt_info["capture_sources"] = merged_sources
+            continue
         if pkt_info.get(key) is None:
             pkt_info[key] = value
 

backend/src/utilities/database.py

@@ -288,6 +288,7 @@ class DatabasePool:
         lower_bound = datetime.fromtimestamp(max(observed_at_ms - window_ms, 0) / 1000.0, tz=timezone.utc)
         upper_bound = datetime.fromtimestamp(max(observed_at_ms + window_ms, 0) / 1000.0, tz=timezone.utc)
         dpi_metadata = enrichment.get("dpi_metadata")
+        capture_sources = [str(source) for source in enrichment.get("capture_sources", []) if source]
 
         try:
             async with self._pool.acquire() as conn:
@@ -306,7 +307,16 @@ class DatabasePool:
                             WHEN $16::jsonb IS NULL THEN packets.dpi_metadata
                             WHEN packets.dpi_metadata IS NULL THEN $16::jsonb
                             ELSE packets.dpi_metadata || $16::jsonb
-                        END
+                        END,
+                        capture_sources = (
+                            SELECT ARRAY(
+                                SELECT DISTINCT source
+                                FROM unnest(
+                                    COALESCE(packets.capture_sources, ARRAY[]::text[]) ||
+                                    COALESCE($17::text[], ARRAY[]::text[])
+                                ) AS source
+                            )
+                        )
                     WHERE
                         ip_proto_raw = $1
                         AND (capture_iface = $2 OR ingress_if = $2 OR egress_if = $2)
@@ -324,6 +334,7 @@ class DatabasePool:
                             OR packets.app_hostname IS NULL
                             OR packets.app_is_encrypted IS NULL
                             OR ($16::jsonb IS NOT NULL)
+                            OR (COALESCE(array_length($17::text[], 1), 0) > 0)
                         )
                     RETURNING *
                     """,
@@ -343,6 +354,7 @@ class DatabasePool:
                     enrichment.get("app_hostname"),
                     enrichment.get("app_is_encrypted"),
                     json.dumps(dpi_metadata) if dpi_metadata is not None else None,
+                    capture_sources if capture_sources else None,
                 )
         except Exception:
             logger.exception("DB packet metadata backfill failed")

backend/src/utilities/packet_tracker.py

@@ -84,7 +84,11 @@ class PacketTracker:
         now_ts = time.time()
         correlation_key = self._ensure_correlation(pkt_info)
         pkt_info["raw_present"] = pkt_info.get("raw") is not None
-        pkt_info["capture_sources"] = [pkt_info.get("capture_source") or "af_packet"]
+        existing_sources = list(pkt_info.get("capture_sources") or [])
+        primary_source = pkt_info.get("capture_source") or "af_packet"
+        if primary_source not in existing_sources:
+            existing_sources.insert(0, primary_source)
+        pkt_info["capture_sources"] = existing_sources
 
         with self._lock:
             entry = self._entries.get(correlation_key)

backend/src/utilities/tshark_manager.py

@@ -22,6 +22,8 @@ _FIELDS: List[str] = [
     "frame.time_epoch",
     "frame.interface_name",
     "frame.len",
+    "_ws.col.Protocol",
+    "_ws.col.Info",
     "ip.src",
     "ipv6.src",
     "ip.dst",
@@ -158,6 +160,8 @@ def _parse_line(line: str, fallback_iface: str) -> Optional[Dict[str, Any]]:
 
     timestamp = _safe_float(row["frame.time_epoch"])
     length = _safe_int(row["frame.len"])
+    protocol_col = _safe_text(row["_ws.col.Protocol"])
+    info_col = _safe_text(row["_ws.col.Info"])
     protocol = _safe_int(row["ip.proto"]) or _safe_int(row["ipv6.nxt"])
     src_ip = _safe_text(row["ip.src"]) or _safe_text(row["ipv6.src"])
     dst_ip = _safe_text(row["ip.dst"]) or _safe_text(row["ipv6.dst"])
@@ -178,6 +182,8 @@ def _parse_line(line: str, fallback_iface: str) -> Optional[Dict[str, Any]]:
         "dst_ip": dst_ip,
         "src_port": src_port,
         "dst_port": dst_port,
+        "protocol_col": protocol_col,
+        "info_col": info_col,
         "frame_protocols": _safe_text(row["frame.protocols"]),
         "http": _jsonable(
             {
@@ -216,6 +222,8 @@ def _build_enrichment(event: Dict[str, Any]) -> Dict[str, Any]:
     tls_meta = dict(event.get("tls") or {})
     dns_meta = dict(event.get("dns") or {})
     protocols = str(event.get("frame_protocols") or "")
+    protocol_col = _safe_text(event.get("protocol_col"))
+    info_col = _safe_text(event.get("info_col"))
 
     app_protocol: Optional[str] = None
     app_category: Optional[str] = None
@@ -237,10 +245,14 @@ def _build_enrichment(event: Dict[str, Any]) -> Dict[str, Any]:
         app_category = "Encrypted"
         app_hostname = tls_meta.get("server_name")
         app_is_encrypted = True
+    elif protocol_col and protocol_col.upper() not in {"TCP", "UDP", "IP", "IPV6", "ETH", "ARP"}:
+        app_protocol = protocol_col
 
     tshark_meta = _jsonable(
         {
             "observed_at_ms": event.get("observed_at_ms"),
+            "protocol": protocol_col,
+            "info": info_col,
             "frame_protocols": event.get("frame_protocols"),
             "length": event.get("length"),
         }
@@ -263,14 +275,23 @@ def _build_enrichment(event: Dict[str, Any]) -> Dict[str, Any]:
         "app_hostname": app_hostname,
         "app_is_encrypted": app_is_encrypted,
         "dpi_metadata": dpi_metadata or None,
+        "capture_sources": ["tshark"],
     }
 
 
 def _has_useful_enrichment(enrichment: Dict[str, Any]) -> bool:
-    return any(
-        enrichment.get(key) is not None
-        for key in ("app_protocol", "app_hostname", "app_is_encrypted", "dpi_metadata")
-    )
+    if enrichment.get("app_protocol") is not None:
+        return True
+    if enrichment.get("app_hostname") is not None:
+        return True
+    dpi_metadata = enrichment.get("dpi_metadata") or {}
+    if not isinstance(dpi_metadata, dict):
+        return False
+    for key in ("http", "tls", "dns"):
+        value = dpi_metadata.get(key)
+        if isinstance(value, dict) and value:
+            return True
+    return False
 
 
 class TsharkManager:
@@ -389,6 +410,9 @@ class TsharkManager:
             "-E",
             "occurrence=f",
         ]
+        if settings.tshark_try_heuristic_first:
+            cmd.extend(["-o", "tcp.try_heuristic_first:true"])
+            cmd.extend(["-o", "udp.try_heuristic_first:true"])
         if settings.tshark_display_filter:
             cmd.extend(["-Y", settings.tshark_display_filter])
         for field in _FIELDS:
@@ -485,6 +509,8 @@ class TsharkManager:
                 self._stats["events_total"] += 1
                 self._stats["last_event_by_iface"][iface] = {
                     "observed_at_ms": event.get("observed_at_ms"),
+                    "protocol_col": event.get("protocol_col"),
+                    "info_col": event.get("info_col"),
                     "protocol": event.get("protocol"),
                     "src_ip": event.get("src_ip"),
                     "dst_ip": event.get("dst_ip"),

setup_build_server.sh

@@ -11,7 +11,6 @@ BACKEND_DIR="$APP_DIR/backend"
 BACKEND_SERVICE="mitm-backend"
 NGINX_SITE="/etc/nginx/sites-available/mitm-webserver"
 USER_ROOT="root"
-BACKEND_ENV_FILE="$BACKEND_DIR/.env"
 
 GITEA_RUNNER_URL="https://gitea.malmert.de//api/v1/repos/marcus/mitm-webserver/actions/runners/register"
 RUNNER_TOKEN="cThC2xmAZWaOAqRRMENuVVTJckxaHiJxVGx2NCQY"
@@ -71,19 +70,6 @@ cd "$BACKEND_DIR"
 pip install -r requirements.txt
 deactivate
 
-# -----------------------------
-# Backend Environment Defaults
-# -----------------------------
-echo "==> Write backend environment defaults"
-cat >"$BACKEND_ENV_FILE" <<EOL
-BACKEND_TSHARK_ENABLED=true
-BACKEND_TSHARK_DISPLAY_FILTER=
-BACKEND_TSHARK_CACHE_TTL_SECONDS=5.0
-BACKEND_TSHARK_MATCH_WINDOW_MS=5000
-BACKEND_TSHARK_READER_JOIN_TIMEOUT_SECONDS=2.0
-BACKEND_TSHARK_PROCESS_STOP_TIMEOUT_SECONDS=3.0
-EOL
-
 # -----------------------------
 # Systemd Service für Backend
 # -----------------------------