try to deduplicate db rows

setup_database.sh

@@ -48,6 +48,7 @@ CREATE TABLE IF NOT EXISTS packets (
     correlation_key TEXT UNIQUE,
     packet_id TEXT,
     packet_uid TEXT,
+    flow_id TEXT,
     correlation_source VARCHAR(32),
     skb_mark BIGINT,
 
@@ -66,7 +67,6 @@ CREATE TABLE IF NOT EXISTS packets (
     src_mac MACADDR,
     dst_mac MACADDR,
     eth_type_raw INTEGER,
-    eth_type VARCHAR(64),
 
     -- VLAN
     vlan_id INTEGER,
@@ -75,7 +75,6 @@ CREATE TABLE IF NOT EXISTS packets (
     src_ip INET,
     dst_ip INET,
     ip_proto_raw INTEGER,
-    ip_proto VARCHAR(64),
 
     -- Transport layer
     src_port INTEGER,
@@ -83,12 +82,10 @@ CREATE TABLE IF NOT EXISTS packets (
 
     -- Packet metadata
     length INTEGER,
-    raw_present BOOLEAN DEFAULT FALSE,
     capture_sources TEXT[],
 
     -- DPI / flow enrichment metadata
     app_protocol VARCHAR(128),
-    app_master_protocol VARCHAR(128),
     app_category VARCHAR(128),
     app_confidence VARCHAR(64),
     app_hostname VARCHAR(255),
@@ -101,84 +98,9 @@ CREATE TABLE IF NOT EXISTS packets (
     -- Full packet dump
     raw BYTEA
 );
-
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW();
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS correlation_key TEXT;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS packet_id TEXT;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS packet_uid TEXT;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS correlation_source VARCHAR(32);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS skb_mark BIGINT;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS capture_iface VARCHAR(64);
-ALTER TABLE packets DROP COLUMN IF EXISTS iface;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_protocol VARCHAR(128);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_master_protocol VARCHAR(128);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_category VARCHAR(128);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_confidence VARCHAR(64);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_hostname VARCHAR(255);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_is_encrypted BOOLEAN;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_risk_score INTEGER;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS dpi_metadata JSONB;
-ALTER TABLE packets ALTER COLUMN dpi_metadata TYPE JSONB USING
-    CASE
-        WHEN dpi_metadata IS NULL THEN NULL
-        WHEN pg_typeof(dpi_metadata)::text = 'jsonb' THEN dpi_metadata
-        ELSE dpi_metadata::jsonb
-    END;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS capture_metadata JSONB;
-ALTER TABLE packets ALTER COLUMN capture_metadata TYPE JSONB USING
-    CASE
-        WHEN capture_metadata IS NULL THEN NULL
-        WHEN pg_typeof(capture_metadata)::text = 'jsonb' THEN capture_metadata
-        ELSE capture_metadata::jsonb
-    END;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS eth_type_raw INTEGER;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS ip_proto_raw INTEGER;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS raw_present BOOLEAN DEFAULT FALSE;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS capture_sources TEXT[];
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS ingress_if VARCHAR(64);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS egress_if VARCHAR(64);
-ALTER TABLE packets DROP COLUMN IF EXISTS observed_ifaces;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict VARCHAR(32);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_reason VARCHAR(128);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_confidence VARCHAR(32);
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS ingress_seen_at TIMESTAMPTZ;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS egress_seen_at TIMESTAMPTZ;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS verdict_seen_at TIMESTAMPTZ;
-ALTER TABLE packets ADD COLUMN IF NOT EXISTS telemetry_metadata JSONB;
-ALTER TABLE packets ALTER COLUMN telemetry_metadata TYPE JSONB USING
-    CASE
-        WHEN telemetry_metadata IS NULL THEN NULL
-        WHEN pg_typeof(telemetry_metadata)::text = 'jsonb' THEN telemetry_metadata
-        ELSE telemetry_metadata::jsonb
-    END;
-ALTER TABLE packets DROP COLUMN IF EXISTS direction;
-UPDATE packets
-SET raw_present = (raw IS NOT NULL)
-WHERE raw_present IS DISTINCT FROM (raw IS NOT NULL);
-UPDATE packets
-SET correlation_key = CASE
-    WHEN packet_id IS NOT NULL AND packet_id <> '' THEN 'pid:' || packet_id
-    WHEN packet_uid IS NOT NULL AND packet_uid <> '' THEN 'uid:' || packet_uid
-    ELSE 'row:' || id::text
-END
-WHERE correlation_key IS NULL OR correlation_key = '';
-UPDATE packets
-SET correlation_source = CASE
-    WHEN packet_id IS NOT NULL AND packet_id <> '' THEN 'kernel_mark'
-    ELSE 'legacy_hash'
-END
-WHERE correlation_source IS NULL OR correlation_source = '';
-UPDATE packets
-SET capture_sources = ARRAY_REMOVE(ARRAY[
-    CASE WHEN raw IS NOT NULL THEN 'af_packet' END,
-    CASE WHEN telemetry_metadata IS NOT NULL THEN 'telemetry' END
-], NULL)
-WHERE capture_sources IS NULL OR array_length(capture_sources, 1) IS NULL;
-UPDATE packets
-SET updated_at = COALESCE(verdict_seen_at, egress_seen_at, ingress_seen_at, timestamp, NOW())
-WHERE updated_at IS NULL;
 CREATE UNIQUE INDEX IF NOT EXISTS idx_packets_correlation_key ON packets(correlation_key);
 CREATE UNIQUE INDEX IF NOT EXISTS idx_packets_packet_uid ON packets(packet_uid);
+CREATE INDEX IF NOT EXISTS idx_packets_flow_id ON packets(flow_id);
 EOF