tshark full usage

backend/src/utilities/database.py

@@ -375,6 +375,124 @@ class DatabasePool:
 
         return updated_count
 
+    async def backfill_stream_metadata(
+        self,
+        *,
+        iface: str,
+        protocol: int,
+        stream_kind: str,
+        stream_id: int,
+        observed_at_ms: int,
+        enrichment: Dict[str, Any],
+        window_ms: int,
+    ) -> int:
+        """Propagate tshark stream context across packets already tagged with the same stream id."""
+        if self._pool is None:
+            await self.init_pool()
+
+        lower_bound = datetime.fromtimestamp(max(observed_at_ms - window_ms, 0) / 1000.0, tz=timezone.utc)
+        upper_bound = datetime.fromtimestamp(max(observed_at_ms + window_ms, 0) / 1000.0, tz=timezone.utc)
+        capture_sources = [str(source) for source in enrichment.get("capture_sources", []) if source]
+        stream_kind = str(stream_kind).lower()
+        if stream_kind not in {"tcp", "udp"}:
+            return 0
+
+        try:
+            async with self._pool.acquire() as conn:
+                rows = await conn.fetch(
+                    """
+                    UPDATE packets
+                    SET
+                        updated_at = NOW(),
+                        app_protocol = CASE
+                            WHEN packets.app_protocol IS NULL OR packets.app_protocol IN ('TCP', 'UDP', 'IP', 'IPv6', 'ETH')
+                                THEN COALESCE($7, packets.app_protocol)
+                            ELSE packets.app_protocol
+                        END,
+                        app_master_protocol = CASE
+                            WHEN packets.app_master_protocol IS NULL OR packets.app_master_protocol IN ('TCP', 'UDP', 'IP', 'IPv6', 'ETH')
+                                THEN COALESCE($8, packets.app_master_protocol)
+                            ELSE packets.app_master_protocol
+                        END,
+                        app_category = CASE
+                            WHEN packets.app_category IS NULL OR packets.app_category IN ('Transport', 'Network', 'Protocol')
+                                THEN COALESCE($9, packets.app_category)
+                            ELSE packets.app_category
+                        END,
+                        app_confidence = CASE
+                            WHEN packets.app_protocol IS NULL OR packets.app_protocol IN ('TCP', 'UDP', 'IP', 'IPv6', 'ETH')
+                                THEN COALESCE($10, packets.app_confidence)
+                            ELSE packets.app_confidence
+                        END,
+                        app_hostname = COALESCE(packets.app_hostname, $11),
+                        app_is_encrypted = COALESCE(packets.app_is_encrypted, $12),
+                        capture_sources = (
+                            SELECT ARRAY(
+                                SELECT DISTINCT source
+                                FROM unnest(
+                                    COALESCE(packets.capture_sources, ARRAY[]::text[]) ||
+                                    COALESCE($13::text[], ARRAY[]::text[])
+                                ) AS source
+                            )
+                        )
+                    WHERE
+                        ip_proto_raw = $1
+                        AND (capture_iface = $2 OR ingress_if = $2 OR egress_if = $2)
+                        AND timestamp BETWEEN $5 AND $6
+                        AND (
+                            CASE
+                                WHEN $3 = 'tcp' THEN COALESCE(packets.dpi_metadata -> 'tcp' ->> 'stream', '')
+                                WHEN $3 = 'udp' THEN COALESCE(packets.dpi_metadata -> 'udp' ->> 'stream', '')
+                                ELSE ''
+                            END
+                        ) = $4
+                        AND (
+                            packets.app_protocol IS NULL
+                            OR packets.app_protocol IN ('TCP', 'UDP', 'IP', 'IPv6', 'ETH')
+                            OR packets.app_master_protocol IS NULL
+                            OR packets.app_master_protocol IN ('TCP', 'UDP', 'IP', 'IPv6', 'ETH')
+                            OR packets.app_category IS NULL
+                            OR packets.app_category IN ('Transport', 'Network', 'Protocol')
+                            OR packets.app_confidence IS NULL
+                            OR packets.app_hostname IS NULL
+                            OR packets.app_is_encrypted IS NULL
+                            OR (COALESCE(array_length($13::text[], 1), 0) > 0)
+                        )
+                    RETURNING *
+                    """,
+                    protocol,
+                    iface,
+                    stream_kind,
+                    str(stream_id),
+                    lower_bound,
+                    upper_bound,
+                    enrichment.get("app_protocol"),
+                    enrichment.get("app_master_protocol"),
+                    enrichment.get("app_category"),
+                    enrichment.get("app_confidence"),
+                    enrichment.get("app_hostname"),
+                    enrichment.get("app_is_encrypted"),
+                    capture_sources if capture_sources else None,
+                )
+        except Exception:
+            logger.exception("DB stream metadata backfill failed")
+            return 0
+
+        if not rows:
+            return 0
+
+        updated_count = 0
+        for row in rows:
+            serialized = _serialize_row_for_broadcast(dict(row))
+            updated_count += 1
+            if self.broadcaster:
+                try:
+                    self.broadcaster.sync_publish(serialized)
+                except Exception:
+                    logger.exception("Failed to publish stream-context packet row")
+
+        return updated_count
+
     async def fetch_latest(self, limit: int) -> List[PacketDBModel]:
         """Fetch newest packet rows as validated `PacketDBModel` instances."""
         if self._pool is None: