Files
mitm-webserver/backend/src/api/analysis_api.py
malmert ae432b7437
All checks were successful
Build and Deploy MITM Webserver / traffic_target (push) Successful in 0s
Build and Deploy MITM Webserver / build (push) Successful in 11s
test visuals
2026-03-31 22:37:00 +02:00

552 lines
22 KiB
Python

"""Analysis endpoints derived from captured packet history."""
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
import src.shared_objects as shared
router = APIRouter()
class InterfaceHostEvidence(BaseModel):
ip_address: Optional[str] = Field(None, description="Observed IP address for the host.")
mac_address: Optional[str] = Field(None, description="Observed MAC address for the host.")
packet_count: int = Field(..., description="How many packet observations supported this mapping.")
last_seen: datetime = Field(..., description="Most recent packet timestamp supporting this mapping.")
source_on_ingress_count: int = Field(..., description="Packets where this endpoint appeared as the source on ingress.")
destination_on_egress_count: int = Field(..., description="Packets where this endpoint appeared as the destination on egress.")
class ProtocolLayerPathEvidence(BaseModel):
ethernet_protocol: Optional[str] = Field(None, description="Ethernet protocol label for this path, if known.")
ip_protocol: Optional[str] = Field(None, description="IP protocol label for this path, if known.")
packet_count: int = Field(..., description="Packet observations supporting this path.")
last_seen: datetime = Field(..., description="Most recent packet timestamp supporting this path.")
accept_count: int = Field(0, description="Packets with verdict=accept for this path.")
drop_count: int = Field(0, description="Packets with verdict=drop for this path.")
reject_count: int = Field(0, description="Packets with verdict=reject for this path.")
unknown_count: int = Field(0, description="Packets with verdict pending/unknown or without a verdict.")
class ProtocolEvidence(BaseModel):
protocol: str = Field(..., description="Detected application or fallback transport/network protocol.")
packet_count: int = Field(..., description="Packet observations supporting this interface-host-protocol mapping.")
last_seen: datetime = Field(..., description="Most recent packet timestamp supporting this protocol mapping.")
accept_count: int = Field(0, description="Packets with verdict=accept for this protocol.")
drop_count: int = Field(0, description="Packets with verdict=drop for this protocol.")
reject_count: int = Field(0, description="Packets with verdict=reject for this protocol.")
unknown_count: int = Field(0, description="Packets with verdict pending/unknown or without a verdict.")
ethernet_protocol: Optional[str] = Field(None, description="Dominant Ethernet protocol associated with this protocol evidence.")
ip_protocol: Optional[str] = Field(None, description="Dominant IP protocol associated with this protocol evidence.")
layer_paths: List[ProtocolLayerPathEvidence] = Field(
default_factory=list,
description="Optional Ethernet/IP breakdown contributing to this protocol evidence.",
)
class InterfaceHostProtocolEvidence(InterfaceHostEvidence):
protocols: List[ProtocolEvidence] = Field(default_factory=list, description="Protocols observed for this host on the interface.")
class InterfaceAttachment(BaseModel):
interface: str = Field(..., description="MITM machine interface name.")
hosts: List[InterfaceHostEvidence] = Field(default_factory=list, description="Endpoints inferred to be attached to this interface.")
class InterfaceProtocolAttachment(BaseModel):
interface: str = Field(..., description="MITM machine interface name.")
hosts: List[InterfaceHostProtocolEvidence] = Field(default_factory=list, description="Endpoints inferred to be attached to this interface, with protocol breakdown.")
class InterfaceHostAnalysisResponse(BaseModel):
since: Optional[datetime] = Field(None, description="Only packets at or after this timestamp were analyzed.")
interfaces: List[InterfaceAttachment] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"This is an inference from observed packet direction, not a kernel neighbor-table lookup.",
"A host is inferred on an interface when it appears as source on ingress or as destination on egress on that interface.",
"Broadcast and obviously incomplete endpoint records are ignored.",
]
)
class InterfaceHostProtocolAnalysisResponse(BaseModel):
since: Optional[datetime] = Field(None, description="Only packets at or after this timestamp were analyzed.")
interfaces: List[InterfaceProtocolAttachment] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"This is an inference from observed packet direction, not a kernel neighbor-table lookup.",
"Each host can carry multiple protocols; protocols prefer app_protocol and fall back to lower-layer protocol names.",
"Verdict counts are packet counts grouped per interface, host, and protocol.",
]
)
class InterfaceProtocolPathEvidence(BaseModel):
ingress_interface: Optional[str] = Field(None, description="Observed ingress interface for the packet path.")
egress_interface: Optional[str] = Field(None, description="Observed egress interface for the packet path.")
src_ip_address: Optional[str] = Field(None, description="Observed source IP address.")
src_mac_address: Optional[str] = Field(None, description="Observed source MAC address.")
dst_ip_address: Optional[str] = Field(None, description="Observed destination IP address.")
dst_mac_address: Optional[str] = Field(None, description="Observed destination MAC address.")
protocol: str = Field(..., description="Detected application or fallback protocol for the packet path.")
ethernet_protocol: Optional[str] = Field(None, description="Dominant Ethernet protocol associated with this path.")
ip_protocol: Optional[str] = Field(None, description="Dominant IP protocol associated with this path.")
packet_count: int = Field(..., description="Packet observations supporting this end-to-end path.")
last_seen: datetime = Field(..., description="Most recent packet timestamp supporting this path.")
accept_count: int = Field(0, description="Packets with verdict=accept for this path.")
drop_count: int = Field(0, description="Packets with verdict=drop for this path.")
reject_count: int = Field(0, description="Packets with verdict=reject for this path.")
unknown_count: int = Field(0, description="Packets with verdict pending/unknown or without a verdict.")
class InterfaceProtocolPathAnalysisResponse(BaseModel):
since: Optional[datetime] = Field(None, description="Only packets at or after this timestamp were analyzed.")
paths: List[InterfaceProtocolPathEvidence] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"This Sankey view is built from packet paths, not from inferred interface-host attachment.",
"Each row represents a grouped ingress -> source endpoint -> protocol -> destination endpoint -> egress path.",
"Protocols prefer app_protocol and fall back to lower-layer protocol names.",
]
)
class ConversationEvidence(BaseModel):
ingress_interface: Optional[str] = None
egress_interface: Optional[str] = None
src_ip_address: Optional[str] = None
src_mac_address: Optional[str] = None
dst_ip_address: Optional[str] = None
dst_mac_address: Optional[str] = None
src_port: Optional[int] = None
dst_port: Optional[int] = None
protocol: str
ethernet_protocol: Optional[str] = None
ip_protocol: Optional[str] = None
hostnames: List[str] = Field(default_factory=list)
packet_count: int
byte_count: int
first_seen: datetime
last_seen: datetime
accept_count: int = 0
drop_count: int = 0
reject_count: int = 0
unknown_count: int = 0
class ConversationAnalysisResponse(BaseModel):
since: Optional[datetime] = None
conversations: List[ConversationEvidence] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"Conversations group directional traffic by source, destination, ports, and detected protocol.",
"Byte counts come from packet lengths observed by the MITM and are useful for comparing session size.",
"Hostname hints are inferred from app_hostname when present, including DNS, HTTP Host, and TLS SNI.",
]
)
class LabelCountEvidence(BaseModel):
label: str
packet_count: int
class HostPeerEvidence(BaseModel):
ip_address: Optional[str] = None
mac_address: Optional[str] = None
packet_count: int
byte_count: int
last_seen: datetime
protocols: List[str] = Field(default_factory=list)
class HostServiceEvidence(BaseModel):
port: Optional[int] = None
protocol: str
packet_count: int
byte_count: int
last_seen: datetime
hostnames: List[str] = Field(default_factory=list)
class HostIntelligenceEvidence(BaseModel):
ip_address: Optional[str] = None
mac_address: Optional[str] = None
packet_count: int
byte_count: int
first_seen: datetime
last_seen: datetime
interfaces: List[str] = Field(default_factory=list)
source_count: int
destination_count: int
hostnames: List[str] = Field(default_factory=list)
top_protocols: List[LabelCountEvidence] = Field(default_factory=list)
peers: List[HostPeerEvidence] = Field(default_factory=list)
services: List[HostServiceEvidence] = Field(default_factory=list)
class HostIntelligenceAnalysisResponse(BaseModel):
since: Optional[datetime] = None
hosts: List[HostIntelligenceEvidence] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"Host intelligence merges packet direction, protocol usage, peer relationships, and hostname enrichment.",
"Services are inferred from traffic where the host appears as the destination on a specific port.",
"Hostname hints come from detected app_hostname values and help turn IPs into recognizable assets.",
]
)
class DiscoveryActivityEvidence(BaseModel):
category: str
protocol: str
ingress_interface: Optional[str] = None
egress_interface: Optional[str] = None
src_ip_address: Optional[str] = None
src_mac_address: Optional[str] = None
dst_ip_address: Optional[str] = None
dst_mac_address: Optional[str] = None
src_port: Optional[int] = None
dst_port: Optional[int] = None
hostnames: List[str] = Field(default_factory=list)
packet_count: int
byte_count: int
first_seen: datetime
last_seen: datetime
class DiscoveryAnalysisResponse(BaseModel):
since: Optional[datetime] = None
activities: List[DiscoveryActivityEvidence] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"Discovery traffic highlights local network learning and service advertisement protocols.",
"This includes ARP, DHCP, mDNS, SSDP, LLMNR, NBNS, and selected ICMPv6 discovery traffic.",
"These views are useful for mapping who is present on the segment and which naming systems are active.",
]
)
class ScanCandidateEvidence(BaseModel):
src_ip_address: Optional[str] = None
src_mac_address: Optional[str] = None
packet_count: int
target_host_count: int
target_port_count: int
first_seen: datetime
last_seen: datetime
class BeaconCandidateEvidence(BaseModel):
src_ip_address: Optional[str] = None
src_mac_address: Optional[str] = None
dst_ip_address: Optional[str] = None
dst_mac_address: Optional[str] = None
dst_port: Optional[int] = None
protocol: str
packet_count: int
avg_interval_seconds: float
jitter_ratio: float
interval_samples: List[float] = Field(default_factory=list)
first_seen: datetime
last_seen: datetime
class RareServiceEvidence(BaseModel):
dst_ip_address: Optional[str] = None
dst_mac_address: Optional[str] = None
dst_port: Optional[int] = None
protocol: str
packet_count: int
client_count: int
hostnames: List[str] = Field(default_factory=list)
last_seen: datetime
class ResetHeavyPathEvidence(BaseModel):
src_ip_address: Optional[str] = None
src_mac_address: Optional[str] = None
dst_ip_address: Optional[str] = None
dst_mac_address: Optional[str] = None
dst_port: Optional[int] = None
total_packets: int
reset_count: int
reset_ratio: float
last_seen: datetime
class DropHeavyPathEvidence(BaseModel):
src_ip_address: Optional[str] = None
src_mac_address: Optional[str] = None
dst_ip_address: Optional[str] = None
dst_mac_address: Optional[str] = None
protocol: str
total_packets: int
drop_count: int
reject_count: int
failure_ratio: float
last_seen: datetime
class AnomalyAnalysisResponse(BaseModel):
since: Optional[datetime] = None
scan_candidates: List[ScanCandidateEvidence] = Field(default_factory=list)
beacon_candidates: List[BeaconCandidateEvidence] = Field(default_factory=list)
rare_services: List[RareServiceEvidence] = Field(default_factory=list)
reset_heavy_paths: List[ResetHeavyPathEvidence] = Field(default_factory=list)
drop_heavy_paths: List[DropHeavyPathEvidence] = Field(default_factory=list)
notes: List[str] = Field(
default_factory=lambda: [
"Anomaly views are heuristic and intended as leads for investigation, not final verdicts.",
"Scan candidates are sources touching many hosts or ports, beacon candidates are conversations with regular intervals.",
"Rare services, reset-heavy paths, and drop-heavy paths help surface unusual or unhealthy communication.",
]
)
@router.get("/interface-hosts", response_model=InterfaceHostAnalysisResponse)
async def analysis_interface_hosts(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit_per_interface: int = Query(
100,
ge=1,
le=1000,
description="Maximum number of inferred hosts returned per interface.",
),
) -> InterfaceHostAnalysisResponse:
"""Infer which IP/MAC endpoints are likely attached to each MITM-side interface."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
rows = await db.infer_interface_hosts(since=since, limit_per_interface=limit_per_interface)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to infer interface host mapping: {exc}") from exc
interfaces = [InterfaceAttachment(**row) for row in rows]
return InterfaceHostAnalysisResponse(since=since, interfaces=interfaces)
@router.get("/interface-host-protocols", response_model=InterfaceHostProtocolAnalysisResponse)
async def analysis_interface_host_protocols(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit_per_interface: int = Query(
50,
ge=1,
le=1000,
description="Maximum number of inferred hosts returned per interface.",
),
limit_protocols_per_host: int = Query(
12,
ge=1,
le=100,
description="Maximum number of top protocols returned per inferred host.",
),
) -> InterfaceHostProtocolAnalysisResponse:
"""Infer interface-host attachment and break observed traffic down by protocol."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
rows = await db.infer_interface_host_protocols(
since=since,
limit_per_interface=limit_per_interface,
limit_protocols_per_host=limit_protocols_per_host,
)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to infer interface host protocol mapping: {exc}") from exc
interfaces = [InterfaceProtocolAttachment(**row) for row in rows]
return InterfaceHostProtocolAnalysisResponse(since=since, interfaces=interfaces)
@router.get("/interface-protocol-paths", response_model=InterfaceProtocolPathAnalysisResponse)
async def analysis_interface_protocol_paths(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit_paths: int = Query(
500,
ge=1,
le=5000,
description="Maximum number of grouped packet paths returned for the Sankey view.",
),
) -> InterfaceProtocolPathAnalysisResponse:
"""Aggregate directional packet paths for the Sankey diagram."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
rows = await db.infer_interface_protocol_paths(
since=since,
limit_paths=limit_paths,
)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to infer interface protocol paths: {exc}") from exc
paths = [InterfaceProtocolPathEvidence(**row) for row in rows]
return InterfaceProtocolPathAnalysisResponse(since=since, paths=paths)
@router.get("/conversations", response_model=ConversationAnalysisResponse)
async def analysis_conversations(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit: int = Query(
300,
ge=1,
le=5000,
description="Maximum number of conversations returned.",
),
) -> ConversationAnalysisResponse:
"""Aggregate directional conversations between observed endpoints."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
rows = await db.analyze_conversations(since=since, limit=limit)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to analyze conversations: {exc}") from exc
conversations = [ConversationEvidence(**row) for row in rows]
return ConversationAnalysisResponse(since=since, conversations=conversations)
@router.get("/host-intelligence", response_model=HostIntelligenceAnalysisResponse)
async def analysis_host_intelligence(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit_hosts: int = Query(
40,
ge=1,
le=500,
description="Maximum number of hosts returned in the intelligence view.",
),
) -> HostIntelligenceAnalysisResponse:
"""Build host-centric intelligence including peers, services, and hostname hints."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
rows = await db.analyze_host_intelligence(since=since, limit_hosts=limit_hosts)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to analyze host intelligence: {exc}") from exc
hosts = [HostIntelligenceEvidence(**row) for row in rows]
return HostIntelligenceAnalysisResponse(since=since, hosts=hosts)
@router.get("/discovery", response_model=DiscoveryAnalysisResponse)
async def analysis_discovery(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit: int = Query(
300,
ge=1,
le=5000,
description="Maximum number of grouped discovery activities returned.",
),
) -> DiscoveryAnalysisResponse:
"""Highlight local discovery, naming, and service advertisement traffic."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
rows = await db.analyze_discovery_activity(since=since, limit=limit)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to analyze discovery activity: {exc}") from exc
activities = [DiscoveryActivityEvidence(**row) for row in rows]
return DiscoveryAnalysisResponse(since=since, activities=activities)
@router.get("/anomalies", response_model=AnomalyAnalysisResponse)
async def analysis_anomalies(
since_minutes: Optional[int] = Query(
None,
ge=1,
le=60 * 24 * 30,
description="Analyze only packets seen within the last N minutes. Omit to cover all captured history.",
),
limit: int = Query(
50,
ge=1,
le=500,
description="Maximum number of anomaly candidates returned per category.",
),
) -> AnomalyAnalysisResponse:
"""Return heuristic anomaly candidates for scans, beaconing, resets, and failures."""
db = shared.db
if db is None:
raise HTTPException(status_code=503, detail="Database not available")
since: Optional[datetime] = None
if since_minutes is not None:
since = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
try:
result = await db.analyze_anomalies(since=since, limit=limit)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to analyze anomalies: {exc}") from exc
return AnomalyAnalysisResponse(since=since, **result)