tc full packet test

setup_database.sh

@@ -44,9 +44,15 @@ sudo -u postgres psql -d "$DB_NAME" <<EOF
 CREATE TABLE IF NOT EXISTS packets (
     id BIGSERIAL PRIMARY KEY,
     timestamp TIMESTAMPTZ DEFAULT NOW(),
-    packet_uid TEXT UNIQUE,
+    updated_at TIMESTAMPTZ DEFAULT NOW(),
+    correlation_key TEXT UNIQUE,
+    packet_id TEXT,
+    packet_uid TEXT,
+    correlation_source VARCHAR(32),
+    skb_mark BIGINT,
 
     -- Interface metadata
+    capture_iface VARCHAR(64),
     ingress_if VARCHAR(64),
     egress_if VARCHAR(64),
     verdict VARCHAR(32),
@@ -89,13 +95,20 @@ CREATE TABLE IF NOT EXISTS packets (
     app_is_encrypted BOOLEAN,
     app_risk_score INTEGER,
     dpi_metadata JSONB,
+    capture_metadata JSONB,
     telemetry_metadata JSONB,
 
     -- Full packet dump
     raw BYTEA
 );
 
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ DEFAULT NOW();
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS correlation_key TEXT;
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS packet_id TEXT;
 ALTER TABLE packets ADD COLUMN IF NOT EXISTS packet_uid TEXT;
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS correlation_source VARCHAR(32);
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS skb_mark BIGINT;
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS capture_iface VARCHAR(64);
 ALTER TABLE packets DROP COLUMN IF EXISTS iface;
 ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_protocol VARCHAR(128);
 ALTER TABLE packets ADD COLUMN IF NOT EXISTS app_master_protocol VARCHAR(128);
@@ -111,6 +124,13 @@ ALTER TABLE packets ALTER COLUMN dpi_metadata TYPE JSONB USING
         WHEN pg_typeof(dpi_metadata)::text = 'jsonb' THEN dpi_metadata
         ELSE dpi_metadata::jsonb
     END;
+ALTER TABLE packets ADD COLUMN IF NOT EXISTS capture_metadata JSONB;
+ALTER TABLE packets ALTER COLUMN capture_metadata TYPE JSONB USING
+    CASE
+        WHEN capture_metadata IS NULL THEN NULL
+        WHEN pg_typeof(capture_metadata)::text = 'jsonb' THEN capture_metadata
+        ELSE capture_metadata::jsonb
+    END;
 ALTER TABLE packets ADD COLUMN IF NOT EXISTS eth_type_raw INTEGER;
 ALTER TABLE packets ADD COLUMN IF NOT EXISTS ip_proto_raw INTEGER;
 ALTER TABLE packets ADD COLUMN IF NOT EXISTS raw_present BOOLEAN DEFAULT FALSE;
@@ -136,11 +156,28 @@ UPDATE packets
 SET raw_present = (raw IS NOT NULL)
 WHERE raw_present IS DISTINCT FROM (raw IS NOT NULL);
 UPDATE packets
+SET correlation_key = CASE
+    WHEN packet_id IS NOT NULL AND packet_id <> '' THEN 'pid:' || packet_id
+    WHEN packet_uid IS NOT NULL AND packet_uid <> '' THEN 'uid:' || packet_uid
+    ELSE 'row:' || id::text
+END
+WHERE correlation_key IS NULL OR correlation_key = '';
+UPDATE packets
+SET correlation_source = CASE
+    WHEN packet_id IS NOT NULL AND packet_id <> '' THEN 'kernel_mark'
+    ELSE 'legacy_hash'
+END
+WHERE correlation_source IS NULL OR correlation_source = '';
+UPDATE packets
 SET capture_sources = ARRAY_REMOVE(ARRAY[
     CASE WHEN raw IS NOT NULL THEN 'af_packet' END,
     CASE WHEN telemetry_metadata IS NOT NULL THEN 'telemetry' END
 ], NULL)
 WHERE capture_sources IS NULL OR array_length(capture_sources, 1) IS NULL;
+UPDATE packets
+SET updated_at = COALESCE(verdict_seen_at, egress_seen_at, ingress_seen_at, timestamp, NOW())
+WHERE updated_at IS NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_packets_correlation_key ON packets(correlation_key);
 CREATE UNIQUE INDEX IF NOT EXISTS idx_packets_packet_uid ON packets(packet_uid);
 EOF
 
@@ -163,6 +200,8 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT, UPDATE ON SEQUENC
 
 -- Create indexes for faster queries
 CREATE INDEX IF NOT EXISTS idx_packets_timestamp ON packets(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_packets_updated_at ON packets(updated_at DESC);
+CREATE INDEX IF NOT EXISTS idx_packets_packet_id ON packets(packet_id);
 CREATE INDEX IF NOT EXISTS idx_packets_src_ip ON packets(src_ip);
 CREATE INDEX IF NOT EXISTS idx_packets_app_protocol ON packets(app_protocol);
 CREATE INDEX IF NOT EXISTS idx_packets_app_hostname ON packets(app_hostname);