marcus/mitm-webserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor: Remove NFStream and flow identity utilities; introduce Tshark manager
All checks were successful
Build and Deploy MITM Webserver / build (push) Successful in 11s
Details

Browse Source 
- Deleted flow_identity.py and nfstream_flow_worker.py as they are no longer needed.
- Removed nfstream_manager.py and its associated logic for managing NFStream workers.
- Added tshark_manager.py to manage tshark packet enrichment and matching.
- Updated setup_build_server.sh to include default environment variables for tshark.
- Implemented packet signature generation and enrichment logic in the new TsharkManager class.
This commit is contained in:
malmert
2026-03-07 19:51:29 +01:00
parent dfad09fa21
commit 22bb6b8526
11 changed files with 603 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
backend/src/utilities/database.py
View File

@@ -267,7 +267,7 @@ class DatabasePool:
except Exception:
logger.exception("Failed to publish pkt_info to broadcaster")
async def backfill_flow_metadata(
async def backfill_packet_metadata(
self,
*,
iface: str,
@@ -276,17 +276,17 @@ class DatabasePool:
src_port: int,
dst_port: int,
protocol: int,
first_seen_ms: int,
last_seen_ms: int,
length: int,
observed_at_ms: int,
enrichment: Dict[str, Any],
window_ms: int,
) -> int:
"""Update recent packet rows for a flow after enrichment arrives asynchronously."""
"""Update recent packet rows after tshark metadata arrives asynchronously."""
if self._pool is None:
await self.init_pool()
lower_bound = datetime.fromtimestamp(max(first_seen_ms - window_ms, 0) / 1000.0, tz=timezone.utc)
upper_bound = datetime.fromtimestamp(max(last_seen_ms + window_ms, 0) / 1000.0, tz=timezone.utc)
lower_bound = datetime.fromtimestamp(max(observed_at_ms - window_ms, 0) / 1000.0, tz=timezone.utc)
upper_bound = datetime.fromtimestamp(max(observed_at_ms + window_ms, 0) / 1000.0, tz=timezone.utc)
dpi_metadata = enrichment.get("dpi_metadata")
try:
@@ -303,19 +303,19 @@ class DatabasePool:
app_hostname = COALESCE(packets.app_hostname, $13),
app_is_encrypted = COALESCE(packets.app_is_encrypted, $14),
dpi_metadata = CASE
WHEN $15::jsonb IS NULL THEN packets.dpi_metadata
WHEN packets.dpi_metadata IS NULL THEN $15::jsonb
ELSE packets.dpi_metadata || $15::jsonb
WHEN $16::jsonb IS NULL THEN packets.dpi_metadata
WHEN packets.dpi_metadata IS NULL THEN $16::jsonb
ELSE packets.dpi_metadata || $16::jsonb
END
WHERE
ip_proto_raw = $1
AND (capture_iface = $2 OR ingress_if = $2 OR egress_if = $2)
AND timestamp BETWEEN $7 AND $8
AND (
(src_ip = $3::inet AND dst_ip = $4::inet AND src_port = $5 AND dst_port = $6)
OR
(src_ip = $4::inet AND dst_ip = $3::inet AND src_port = $6 AND dst_port = $5)
)
AND src_ip = $3::inet
AND dst_ip = $4::inet
AND src_port = $5
AND dst_port = $6
AND length = $7
AND timestamp BETWEEN $8 AND $9
AND (
packets.app_protocol IS NULL
OR packets.app_master_protocol IS NULL
@@ -323,7 +323,7 @@ class DatabasePool:
OR packets.app_confidence IS NULL
OR packets.app_hostname IS NULL
OR packets.app_is_encrypted IS NULL
OR ($15::jsonb IS NOT NULL)
OR ($16::jsonb IS NOT NULL)
)
RETURNING *
""",
@@ -333,6 +333,7 @@ class DatabasePool:
dst_ip,
src_port,
dst_port,
length,
lower_bound,
upper_bound,
enrichment.get("app_protocol"),
@@ -344,7 +345,7 @@ class DatabasePool:
json.dumps(dpi_metadata) if dpi_metadata is not None else None,
)
except Exception:
logger.exception("DB flow metadata backfill failed")
logger.exception("DB packet metadata backfill failed")
return 0
if not rows:
@@ -358,7 +359,7 @@ class DatabasePool:
try:
self.broadcaster.sync_publish(serialized)
except Exception:
logger.exception("Failed to publish flow-enriched packet row")
logger.exception("Failed to publish tshark-enriched packet row")
return updated_count