test visualization with d3js

backend/src/api/analysis_api.py

@@ -20,11 +20,30 @@ class InterfaceHostEvidence(BaseModel):
     destination_on_egress_count: int = Field(..., description="Packets where this endpoint appeared as the destination on egress.")
 
 
+class ProtocolEvidence(BaseModel):
+    protocol: str = Field(..., description="Detected application or fallback transport/network protocol.")
+    packet_count: int = Field(..., description="Packet observations supporting this interface-host-protocol mapping.")
+    last_seen: datetime = Field(..., description="Most recent packet timestamp supporting this protocol mapping.")
+    accept_count: int = Field(0, description="Packets with verdict=accept for this protocol.")
+    drop_count: int = Field(0, description="Packets with verdict=drop for this protocol.")
+    reject_count: int = Field(0, description="Packets with verdict=reject for this protocol.")
+    unknown_count: int = Field(0, description="Packets with verdict pending/unknown or without a verdict.")
+
+
+class InterfaceHostProtocolEvidence(InterfaceHostEvidence):
+    protocols: List[ProtocolEvidence] = Field(default_factory=list, description="Protocols observed for this host on the interface.")
+
+
 class InterfaceAttachment(BaseModel):
     interface: str = Field(..., description="MITM machine interface name.")
     hosts: List[InterfaceHostEvidence] = Field(default_factory=list, description="Endpoints inferred to be attached to this interface.")
 
 
+class InterfaceProtocolAttachment(BaseModel):
+    interface: str = Field(..., description="MITM machine interface name.")
+    hosts: List[InterfaceHostProtocolEvidence] = Field(default_factory=list, description="Endpoints inferred to be attached to this interface, with protocol breakdown.")
+
+
 class InterfaceHostAnalysisResponse(BaseModel):
     since: Optional[datetime] = Field(None, description="Only packets at or after this timestamp were analyzed.")
     interfaces: List[InterfaceAttachment] = Field(default_factory=list)
@@ -37,6 +56,18 @@ class InterfaceHostAnalysisResponse(BaseModel):
     )
 
 
+class InterfaceHostProtocolAnalysisResponse(BaseModel):
+    since: Optional[datetime] = Field(None, description="Only packets at or after this timestamp were analyzed.")
+    interfaces: List[InterfaceProtocolAttachment] = Field(default_factory=list)
+    notes: List[str] = Field(
+        default_factory=lambda: [
+            "This is an inference from observed packet direction, not a kernel neighbor-table lookup.",
+            "Each host can carry multiple protocols; protocols prefer app_protocol and fall back to lower-layer protocol names.",
+            "Verdict counts are packet counts grouped per interface, host, and protocol.",
+        ]
+    )
+
+
 @router.get("/interface-hosts", response_model=InterfaceHostAnalysisResponse)
 async def analysis_interface_hosts(
     since_minutes: Optional[int] = Query(
@@ -68,3 +99,46 @@ async def analysis_interface_hosts(
 
     interfaces = [InterfaceAttachment(**row) for row in rows]
     return InterfaceHostAnalysisResponse(since=since, interfaces=interfaces)
+
+
+@router.get("/interface-host-protocols", response_model=InterfaceHostProtocolAnalysisResponse)
+async def analysis_interface_host_protocols(
+    since_minutes: Optional[int] = Query(
+        60,
+        ge=1,
+        le=60 * 24 * 30,
+        description="Analyze only packets seen within the last N minutes. Set to a large value to cover more history.",
+    ),
+    limit_per_interface: int = Query(
+        50,
+        ge=1,
+        le=1000,
+        description="Maximum number of inferred hosts returned per interface.",
+    ),
+    limit_protocols_per_host: int = Query(
+        12,
+        ge=1,
+        le=100,
+        description="Maximum number of top protocols returned per inferred host.",
+    ),
+) -> InterfaceHostProtocolAnalysisResponse:
+    """Infer interface-host attachment and break observed traffic down by protocol."""
+    db = shared.db
+    if db is None:
+        raise HTTPException(status_code=503, detail="Database not available")
+
+    since: Optional[datetime] = None
+    if since_minutes is not None:
+        since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
+
+    try:
+        rows = await db.infer_interface_host_protocols(
+            since=since,
+            limit_per_interface=limit_per_interface,
+            limit_protocols_per_host=limit_protocols_per_host,
+        )
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail="Failed to infer interface host protocol mapping") from exc
+
+    interfaces = [InterfaceProtocolAttachment(**row) for row in rows]
+    return InterfaceHostProtocolAnalysisResponse(since=since, interfaces=interfaces)

backend/src/utilities/database.py

@@ -733,6 +733,197 @@ class DatabasePool:
             for iface, hosts in sorted(grouped.items())
         ]
 
+    async def infer_interface_host_protocols(
+        self,
+        *,
+        since: Optional[datetime] = None,
+        limit_per_interface: int = 50,
+        limit_protocols_per_host: int = 12,
+    ) -> List[Dict[str, Any]]:
+        """Infer interface-host attachment and aggregate observed protocols and verdicts."""
+        if self._pool is None:
+            await self.init_pool()
+
+        async with self._pool.acquire() as conn:
+            rows = await conn.fetch(
+                """
+                WITH observations AS (
+                    SELECT
+                        ingress_if AS iface,
+                        src_ip::text AS ip_address,
+                        src_mac::text AS mac_address,
+                        COALESCE(NULLIF(app_protocol, ''), NULLIF(app_master_protocol, ''), NULLIF(ip_proto, ''), NULLIF(eth_type, ''), 'UNKNOWN') AS protocol_name,
+                        COALESCE(NULLIF(verdict, ''), 'unknown') AS verdict_name,
+                        timestamp,
+                        'source_on_ingress' AS evidence
+                    FROM packets
+                    WHERE ingress_if IS NOT NULL
+                      AND (src_ip IS NOT NULL OR src_mac IS NOT NULL)
+                      AND ($1::timestamptz IS NULL OR timestamp >= $1)
+
+                    UNION ALL
+
+                    SELECT
+                        egress_if AS iface,
+                        dst_ip::text AS ip_address,
+                        dst_mac::text AS mac_address,
+                        COALESCE(NULLIF(app_protocol, ''), NULLIF(app_master_protocol, ''), NULLIF(ip_proto, ''), NULLIF(eth_type, ''), 'UNKNOWN') AS protocol_name,
+                        COALESCE(NULLIF(verdict, ''), 'unknown') AS verdict_name,
+                        timestamp,
+                        'destination_on_egress' AS evidence
+                    FROM packets
+                    WHERE egress_if IS NOT NULL
+                      AND (dst_ip IS NOT NULL OR dst_mac IS NOT NULL)
+                      AND ($1::timestamptz IS NULL OR timestamp >= $1)
+                ),
+                filtered AS (
+                    SELECT *
+                    FROM observations
+                    WHERE iface IS NOT NULL
+                      AND COALESCE(mac_address, '') <> 'ff:ff:ff:ff:ff:ff'
+                      AND (
+                          COALESCE(ip_address, '') <> ''
+                          OR COALESCE(mac_address, '') <> ''
+                      )
+                ),
+                host_aggregated AS (
+                    SELECT
+                        iface,
+                        ip_address,
+                        mac_address,
+                        COUNT(*) AS packet_count,
+                        MAX(timestamp) AS last_seen,
+                        SUM(CASE WHEN evidence = 'source_on_ingress' THEN 1 ELSE 0 END) AS source_on_ingress_count,
+                        SUM(CASE WHEN evidence = 'destination_on_egress' THEN 1 ELSE 0 END) AS destination_on_egress_count
+                    FROM filtered
+                    GROUP BY iface, ip_address, mac_address
+                ),
+                selected_hosts AS (
+                    SELECT *
+                    FROM (
+                        SELECT
+                            *,
+                            ROW_NUMBER() OVER (
+                                PARTITION BY iface
+                                ORDER BY packet_count DESC, last_seen DESC, ip_address, mac_address
+                            ) AS row_num
+                        FROM host_aggregated
+                    ) ranked_hosts
+                    WHERE row_num <= $2
+                ),
+                protocol_aggregated AS (
+                    SELECT
+                        filtered.iface,
+                        filtered.ip_address,
+                        filtered.mac_address,
+                        filtered.protocol_name,
+                        COUNT(*) AS packet_count,
+                        MAX(filtered.timestamp) AS last_seen,
+                        SUM(CASE WHEN filtered.verdict_name = 'accept' THEN 1 ELSE 0 END) AS accept_count,
+                        SUM(CASE WHEN filtered.verdict_name = 'drop' THEN 1 ELSE 0 END) AS drop_count,
+                        SUM(CASE WHEN filtered.verdict_name = 'reject' THEN 1 ELSE 0 END) AS reject_count,
+                        SUM(CASE WHEN filtered.verdict_name NOT IN ('accept', 'drop', 'reject') THEN 1 ELSE 0 END) AS unknown_count
+                    FROM filtered
+                    INNER JOIN selected_hosts
+                        ON selected_hosts.iface = filtered.iface
+                       AND selected_hosts.ip_address IS NOT DISTINCT FROM filtered.ip_address
+                       AND selected_hosts.mac_address IS NOT DISTINCT FROM filtered.mac_address
+                    GROUP BY filtered.iface, filtered.ip_address, filtered.mac_address, filtered.protocol_name
+                ),
+                ranked_protocols AS (
+                    SELECT *
+                    FROM (
+                        SELECT
+                            *,
+                            ROW_NUMBER() OVER (
+                                PARTITION BY iface, ip_address, mac_address
+                                ORDER BY packet_count DESC, last_seen DESC, protocol_name
+                            ) AS row_num
+                        FROM protocol_aggregated
+                    ) ranked
+                    WHERE row_num <= $3
+                )
+                SELECT
+                    selected_hosts.iface,
+                    selected_hosts.ip_address,
+                    selected_hosts.mac_address,
+                    selected_hosts.packet_count AS host_packet_count,
+                    selected_hosts.last_seen AS host_last_seen,
+                    selected_hosts.source_on_ingress_count,
+                    selected_hosts.destination_on_egress_count,
+                    ranked_protocols.protocol_name,
+                    ranked_protocols.packet_count AS protocol_packet_count,
+                    ranked_protocols.last_seen AS protocol_last_seen,
+                    ranked_protocols.accept_count,
+                    ranked_protocols.drop_count,
+                    ranked_protocols.reject_count,
+                    ranked_protocols.unknown_count
+                FROM selected_hosts
+                LEFT JOIN ranked_protocols
+                    ON ranked_protocols.iface = selected_hosts.iface
+                   AND ranked_protocols.ip_address IS NOT DISTINCT FROM selected_hosts.ip_address
+                   AND ranked_protocols.mac_address IS NOT DISTINCT FROM selected_hosts.mac_address
+                ORDER BY
+                    selected_hosts.iface,
+                    selected_hosts.packet_count DESC,
+                    selected_hosts.last_seen DESC,
+                    selected_hosts.ip_address,
+                    selected_hosts.mac_address,
+                    ranked_protocols.packet_count DESC NULLS LAST,
+                    ranked_protocols.last_seen DESC NULLS LAST,
+                    ranked_protocols.protocol_name
+                """,
+                since,
+                limit_per_interface,
+                limit_protocols_per_host,
+            )
+
+        grouped: Dict[str, Dict[str, Dict[str, Any]]] = {}
+        for row in rows:
+            record = dict(row)
+            iface = str(record["iface"])
+            host_key = f"{record.get('ip_address') or 'no-ip'}|{record.get('mac_address') or 'no-mac'}"
+
+            iface_hosts = grouped.setdefault(iface, {})
+            host_record = iface_hosts.get(host_key)
+            if host_record is None:
+                host_record = {
+                    "ip_address": record.get("ip_address"),
+                    "mac_address": record.get("mac_address"),
+                    "packet_count": int(record.get("host_packet_count") or 0),
+                    "last_seen": record["host_last_seen"].isoformat() if hasattr(record.get("host_last_seen"), "isoformat") else record.get("host_last_seen"),
+                    "source_on_ingress_count": int(record.get("source_on_ingress_count") or 0),
+                    "destination_on_egress_count": int(record.get("destination_on_egress_count") or 0),
+                    "protocols": [],
+                }
+                iface_hosts[host_key] = host_record
+
+            protocol_name = record.get("protocol_name")
+            if protocol_name not in (None, ""):
+                host_record["protocols"].append(
+                    {
+                        "protocol": str(protocol_name),
+                        "packet_count": int(record.get("protocol_packet_count") or 0),
+                        "last_seen": record["protocol_last_seen"].isoformat()
+                        if hasattr(record.get("protocol_last_seen"), "isoformat")
+                        else record.get("protocol_last_seen"),
+                        "accept_count": int(record.get("accept_count") or 0),
+                        "drop_count": int(record.get("drop_count") or 0),
+                        "reject_count": int(record.get("reject_count") or 0),
+                        "unknown_count": int(record.get("unknown_count") or 0),
+                    }
+                )
+
+        result: List[Dict[str, Any]] = []
+        for iface, hosts in sorted(grouped.items()):
+            sorted_hosts = sorted(
+                hosts.values(),
+                key=lambda item: (-int(item.get("packet_count") or 0), str(item.get("last_seen") or ""), str(item.get("ip_address") or ""), str(item.get("mac_address") or "")),
+            )
+            result.append({"interface": iface, "hosts": sorted_hosts})
+
+        return result
+
     async def clear_all_packets(self, reset_identity: bool = True) -> bool:
         """Truncate the packet table and optionally reset identity counters."""
         if self._pool is None: