@@ -46,13 +56,10 @@ HTML = """
diff --git a/src/fgai/anomaly.py b/src/fgai/anomaly.py index ce4b072..9024a19 100644 --- a/src/fgai/anomaly.py +++ b/src/fgai/anomaly.py @@ -75,7 +75,7 @@ def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]: def detect_source_anomalies( - events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, float | int]] | None = None + events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None ) -> list[AnomalyFinding]: baselines = baselines or {} by_src: dict[str, list[LogEvent]] = defaultdict(list) @@ -200,6 +200,19 @@ def detect_source_anomalies( score += min(15, distinct_dst_ports) reasons.append(f"many destination ports contacted ({distinct_dst_ports})") + baseline = baselines.get(src_ip) + if baseline: + known_destinations = set(baseline.get("known_destinations", [])) + known_ports = set(baseline.get("known_destination_ports", [])) + new_destinations = {event.dst_ip for event in src_events if event.dst_ip and event.dst_ip not in known_destinations} + new_ports = {event.fields.get("dstport") for event in src_events if event.fields.get("dstport") and event.fields.get("dstport") not in known_ports} + if len(known_destinations) >= 5 and len(new_destinations) >= 3: + score += min(15, 5 + len(new_destinations)) + reasons.append(f"new destinations relative to historical baseline ({len(new_destinations)})") + if len(known_ports) >= 3 and len(new_ports) >= 2: + score += min(12, 4 + len(new_ports)) + reasons.append(f"new destination ports relative to historical baseline ({len(new_ports)})") + if _is_public_ip(src_ip) and (utm_count or deny_count >= 10): score += 10 reasons.append("public source with repeated security-relevant events") @@ -232,6 +245,7 @@ def detect_source_anomalies( "distinct_dst_ports": distinct_dst_ports, "policy_count": len(policies), "implicit_deny_events": implicit_deny_count, + "baseline_ready": int(src_ip in baselines), }, ) ) diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 8278768..282237e 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -35,6 +35,10 @@ class BaselineStore: denies integer not null, utm integer not null, primary key (source_ip, bucket_start) ); + create table if not exists source_values ( + source_ip text not null, kind text not null, value text not null, + seen_count integer not null, primary key (source_ip, kind, value) + ); """ ) @@ -59,6 +63,13 @@ class BaselineStore: values[2] += _number(event.fields.get("hitcount")) values[3] += int(event.action in THREAT_ACTIONS) values[4] += int(is_utm_event(event)) + for kind, value in (("destination", event.dst_ip), ("destination_port", event.fields.get("dstport"))): + if value: + connection.execute( + """insert into source_values values (?, ?, ?, 1) + on conflict(source_ip, kind, value) do update set seen_count=seen_count+1""", + (event.src_ip, kind, value), + ) inserted += 1 for (source_ip, bucket), values in pending.items(): connection.execute( @@ -70,8 +81,8 @@ class BaselineStore: ) return inserted - def profiles(self, source_ips: set[str]) -> dict[str, dict[str, float | int]]: - profiles: dict[str, dict[str, float | int]] = {} + def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]: + profiles: dict[str, dict[str, object]] = {} with self._connect() as connection: for source_ip in source_ips: rows = connection.execute( @@ -83,9 +94,14 @@ class BaselineStore: continue rates = [row[0] * 60 / self.bucket_seconds for row in rows] hit_rates = [row[2] * 60 / self.bucket_seconds for row in rows] + known = connection.execute( + "select kind, value from source_values where source_ip=?", (source_ip,) + ).fetchall() profiles[source_ip] = { "samples": len(rows), "event_rate_mean": mean(rates), "event_rate_stddev": pstdev(rates) or 1.0, "hitcount_rate_mean": mean(hit_rates), "hitcount_rate_stddev": pstdev(hit_rates) or 1.0, + "known_destinations": [value for kind, value in known if kind == "destination"], + "known_destination_ports": [value for kind, value in known if kind == "destination_port"], } return profiles diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 3edf67b..6cabbda 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -14,14 +14,14 @@ HTML = """
-
@@ -46,13 +56,10 @@ HTML = """