test visualization with d3js
This commit is contained in:
@@ -733,6 +733,197 @@ class DatabasePool:
|
||||
for iface, hosts in sorted(grouped.items())
|
||||
]
|
||||
|
||||
async def infer_interface_host_protocols(
|
||||
self,
|
||||
*,
|
||||
since: Optional[datetime] = None,
|
||||
limit_per_interface: int = 50,
|
||||
limit_protocols_per_host: int = 12,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Infer interface-host attachment and aggregate observed protocols and verdicts."""
|
||||
if self._pool is None:
|
||||
await self.init_pool()
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
WITH observations AS (
|
||||
SELECT
|
||||
ingress_if AS iface,
|
||||
src_ip::text AS ip_address,
|
||||
src_mac::text AS mac_address,
|
||||
COALESCE(NULLIF(app_protocol, ''), NULLIF(app_master_protocol, ''), NULLIF(ip_proto, ''), NULLIF(eth_type, ''), 'UNKNOWN') AS protocol_name,
|
||||
COALESCE(NULLIF(verdict, ''), 'unknown') AS verdict_name,
|
||||
timestamp,
|
||||
'source_on_ingress' AS evidence
|
||||
FROM packets
|
||||
WHERE ingress_if IS NOT NULL
|
||||
AND (src_ip IS NOT NULL OR src_mac IS NOT NULL)
|
||||
AND ($1::timestamptz IS NULL OR timestamp >= $1)
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
egress_if AS iface,
|
||||
dst_ip::text AS ip_address,
|
||||
dst_mac::text AS mac_address,
|
||||
COALESCE(NULLIF(app_protocol, ''), NULLIF(app_master_protocol, ''), NULLIF(ip_proto, ''), NULLIF(eth_type, ''), 'UNKNOWN') AS protocol_name,
|
||||
COALESCE(NULLIF(verdict, ''), 'unknown') AS verdict_name,
|
||||
timestamp,
|
||||
'destination_on_egress' AS evidence
|
||||
FROM packets
|
||||
WHERE egress_if IS NOT NULL
|
||||
AND (dst_ip IS NOT NULL OR dst_mac IS NOT NULL)
|
||||
AND ($1::timestamptz IS NULL OR timestamp >= $1)
|
||||
),
|
||||
filtered AS (
|
||||
SELECT *
|
||||
FROM observations
|
||||
WHERE iface IS NOT NULL
|
||||
AND COALESCE(mac_address, '') <> 'ff:ff:ff:ff:ff:ff'
|
||||
AND (
|
||||
COALESCE(ip_address, '') <> ''
|
||||
OR COALESCE(mac_address, '') <> ''
|
||||
)
|
||||
),
|
||||
host_aggregated AS (
|
||||
SELECT
|
||||
iface,
|
||||
ip_address,
|
||||
mac_address,
|
||||
COUNT(*) AS packet_count,
|
||||
MAX(timestamp) AS last_seen,
|
||||
SUM(CASE WHEN evidence = 'source_on_ingress' THEN 1 ELSE 0 END) AS source_on_ingress_count,
|
||||
SUM(CASE WHEN evidence = 'destination_on_egress' THEN 1 ELSE 0 END) AS destination_on_egress_count
|
||||
FROM filtered
|
||||
GROUP BY iface, ip_address, mac_address
|
||||
),
|
||||
selected_hosts AS (
|
||||
SELECT *
|
||||
FROM (
|
||||
SELECT
|
||||
*,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY iface
|
||||
ORDER BY packet_count DESC, last_seen DESC, ip_address, mac_address
|
||||
) AS row_num
|
||||
FROM host_aggregated
|
||||
) ranked_hosts
|
||||
WHERE row_num <= $2
|
||||
),
|
||||
protocol_aggregated AS (
|
||||
SELECT
|
||||
filtered.iface,
|
||||
filtered.ip_address,
|
||||
filtered.mac_address,
|
||||
filtered.protocol_name,
|
||||
COUNT(*) AS packet_count,
|
||||
MAX(filtered.timestamp) AS last_seen,
|
||||
SUM(CASE WHEN filtered.verdict_name = 'accept' THEN 1 ELSE 0 END) AS accept_count,
|
||||
SUM(CASE WHEN filtered.verdict_name = 'drop' THEN 1 ELSE 0 END) AS drop_count,
|
||||
SUM(CASE WHEN filtered.verdict_name = 'reject' THEN 1 ELSE 0 END) AS reject_count,
|
||||
SUM(CASE WHEN filtered.verdict_name NOT IN ('accept', 'drop', 'reject') THEN 1 ELSE 0 END) AS unknown_count
|
||||
FROM filtered
|
||||
INNER JOIN selected_hosts
|
||||
ON selected_hosts.iface = filtered.iface
|
||||
AND selected_hosts.ip_address IS NOT DISTINCT FROM filtered.ip_address
|
||||
AND selected_hosts.mac_address IS NOT DISTINCT FROM filtered.mac_address
|
||||
GROUP BY filtered.iface, filtered.ip_address, filtered.mac_address, filtered.protocol_name
|
||||
),
|
||||
ranked_protocols AS (
|
||||
SELECT *
|
||||
FROM (
|
||||
SELECT
|
||||
*,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY iface, ip_address, mac_address
|
||||
ORDER BY packet_count DESC, last_seen DESC, protocol_name
|
||||
) AS row_num
|
||||
FROM protocol_aggregated
|
||||
) ranked
|
||||
WHERE row_num <= $3
|
||||
)
|
||||
SELECT
|
||||
selected_hosts.iface,
|
||||
selected_hosts.ip_address,
|
||||
selected_hosts.mac_address,
|
||||
selected_hosts.packet_count AS host_packet_count,
|
||||
selected_hosts.last_seen AS host_last_seen,
|
||||
selected_hosts.source_on_ingress_count,
|
||||
selected_hosts.destination_on_egress_count,
|
||||
ranked_protocols.protocol_name,
|
||||
ranked_protocols.packet_count AS protocol_packet_count,
|
||||
ranked_protocols.last_seen AS protocol_last_seen,
|
||||
ranked_protocols.accept_count,
|
||||
ranked_protocols.drop_count,
|
||||
ranked_protocols.reject_count,
|
||||
ranked_protocols.unknown_count
|
||||
FROM selected_hosts
|
||||
LEFT JOIN ranked_protocols
|
||||
ON ranked_protocols.iface = selected_hosts.iface
|
||||
AND ranked_protocols.ip_address IS NOT DISTINCT FROM selected_hosts.ip_address
|
||||
AND ranked_protocols.mac_address IS NOT DISTINCT FROM selected_hosts.mac_address
|
||||
ORDER BY
|
||||
selected_hosts.iface,
|
||||
selected_hosts.packet_count DESC,
|
||||
selected_hosts.last_seen DESC,
|
||||
selected_hosts.ip_address,
|
||||
selected_hosts.mac_address,
|
||||
ranked_protocols.packet_count DESC NULLS LAST,
|
||||
ranked_protocols.last_seen DESC NULLS LAST,
|
||||
ranked_protocols.protocol_name
|
||||
""",
|
||||
since,
|
||||
limit_per_interface,
|
||||
limit_protocols_per_host,
|
||||
)
|
||||
|
||||
grouped: Dict[str, Dict[str, Dict[str, Any]]] = {}
|
||||
for row in rows:
|
||||
record = dict(row)
|
||||
iface = str(record["iface"])
|
||||
host_key = f"{record.get('ip_address') or 'no-ip'}|{record.get('mac_address') or 'no-mac'}"
|
||||
|
||||
iface_hosts = grouped.setdefault(iface, {})
|
||||
host_record = iface_hosts.get(host_key)
|
||||
if host_record is None:
|
||||
host_record = {
|
||||
"ip_address": record.get("ip_address"),
|
||||
"mac_address": record.get("mac_address"),
|
||||
"packet_count": int(record.get("host_packet_count") or 0),
|
||||
"last_seen": record["host_last_seen"].isoformat() if hasattr(record.get("host_last_seen"), "isoformat") else record.get("host_last_seen"),
|
||||
"source_on_ingress_count": int(record.get("source_on_ingress_count") or 0),
|
||||
"destination_on_egress_count": int(record.get("destination_on_egress_count") or 0),
|
||||
"protocols": [],
|
||||
}
|
||||
iface_hosts[host_key] = host_record
|
||||
|
||||
protocol_name = record.get("protocol_name")
|
||||
if protocol_name not in (None, ""):
|
||||
host_record["protocols"].append(
|
||||
{
|
||||
"protocol": str(protocol_name),
|
||||
"packet_count": int(record.get("protocol_packet_count") or 0),
|
||||
"last_seen": record["protocol_last_seen"].isoformat()
|
||||
if hasattr(record.get("protocol_last_seen"), "isoformat")
|
||||
else record.get("protocol_last_seen"),
|
||||
"accept_count": int(record.get("accept_count") or 0),
|
||||
"drop_count": int(record.get("drop_count") or 0),
|
||||
"reject_count": int(record.get("reject_count") or 0),
|
||||
"unknown_count": int(record.get("unknown_count") or 0),
|
||||
}
|
||||
)
|
||||
|
||||
result: List[Dict[str, Any]] = []
|
||||
for iface, hosts in sorted(grouped.items()):
|
||||
sorted_hosts = sorted(
|
||||
hosts.values(),
|
||||
key=lambda item: (-int(item.get("packet_count") or 0), str(item.get("last_seen") or ""), str(item.get("ip_address") or ""), str(item.get("mac_address") or "")),
|
||||
)
|
||||
result.append({"interface": iface, "hosts": sorted_hosts})
|
||||
|
||||
return result
|
||||
|
||||
async def clear_all_packets(self, reset_identity: bool = True) -> bool:
|
||||
"""Truncate the packet table and optionally reset identity counters."""
|
||||
if self._pool is None:
|
||||
|
||||
Reference in New Issue
Block a user