diff --git a/src/fgai/anomaly.py b/src/fgai/anomaly.py index 3e1d60d..d6db5db 100644 --- a/src/fgai/anomaly.py +++ b/src/fgai/anomaly.py @@ -2,6 +2,7 @@ from __future__ import annotations import ipaddress from collections import Counter, defaultdict +from datetime import datetime from statistics import mean, pstdev from .logs import THREAT_ACTIONS, event_score, is_utm_event @@ -42,6 +43,37 @@ def _is_public_ip(value: str) -> bool: return False +def _event_timestamp(event: LogEvent) -> float | None: + """Return the FortiGate event time in seconds when present.""" + eventtime = event.fields.get("eventtime") + if eventtime: + try: + value = float(eventtime) + # Exports can use seconds, milliseconds, microseconds, or nanoseconds. + while value > 10_000_000_000: + value /= 1_000 + return value + except ValueError: + pass + + date = event.fields.get("date") + clock = event.fields.get("time") + if date and clock: + try: + return datetime.fromisoformat(f"{date}T{clock}").timestamp() + except ValueError: + return None + return None + + +def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]: + timestamps = [timestamp for event in events if (timestamp := _event_timestamp(event)) is not None] + if len(timestamps) < 2: + return None, 0.0 + duration_seconds = max(timestamps) - min(timestamps) + return len(timestamps) * 60 / max(1.0, duration_seconds), duration_seconds + + def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[AnomalyFinding]: by_src: dict[str, list[LogEvent]] = defaultdict(list) for event in events: @@ -60,19 +92,30 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[ sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events) for src_events in by_src.values() ] + hitcount_totals = [sum(_as_int(event.fields.get("hitcount")) for event in src_events) for src_events in by_src.values()] + source_rates = [rate for src_events in by_src.values() if (rate := _rate_per_minute(src_events)[0]) is not None] avg_events = mean(event_counts) std_events = pstdev(event_counts) or 1.0 avg_dst = mean(distinct_dst_counts) std_dst = pstdev(distinct_dst_counts) or 1.0 avg_bytes = mean(byte_totals) std_bytes = pstdev(byte_totals) or 1.0 + avg_hitcount = mean(hitcount_totals) + std_hitcount = pstdev(hitcount_totals) or 1.0 + avg_rate = mean(source_rates) if source_rates else 0.0 + std_rate = (pstdev(source_rates) or 1.0) if source_rates else 1.0 findings: list[AnomalyFinding] = [] for src_ip, src_events in by_src.items(): event_count = len(src_events) distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")}) distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")}) + distinct_src_ports = len({event.fields.get("srcport") for event in src_events if event.fields.get("srcport")}) + distinct_dst_ports = len({event.fields.get("dstport") for event in src_events if event.fields.get("dstport")}) total_bytes = sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events) + total_hitcount = sum(_as_int(event.fields.get("hitcount")) for event in src_events) + max_hitcount = max((_as_int(event.fields.get("hitcount")) for event in src_events), default=0) + event_rate, observed_duration = _rate_per_minute(src_events) deny_count = sum(1 for event in src_events if event.action in THREAT_ACTIONS) utm_count = sum(1 for event in src_events if is_utm_event(event)) high_severity_count = sum(1 for event in src_events if event.severity in {"critical", "high", "alert", "emergency"}) @@ -88,6 +131,13 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[ score += points reasons.append(f"unusually high event volume for source ({event_count} events, z={event_z:.1f})") + if event_rate is not None: + rate_z = (event_rate - avg_rate) / std_rate + if event_rate >= 20 and rate_z >= 2: + points = min(25, 10 + int(rate_z * 5)) + score += points + reasons.append(f"unusually high log rate ({event_rate:.1f} events/min, z={rate_z:.1f})") + dst_z = (distinct_dst - avg_dst) / std_dst if distinct_dst >= 10 and dst_z >= 2: points = min(25, 10 + int(dst_z * 5)) @@ -100,6 +150,12 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[ score += points reasons.append(f"unusually high byte volume ({total_bytes} bytes, z={byte_z:.1f})") + hitcount_z = (total_hitcount - avg_hitcount) / std_hitcount + if total_hitcount >= 1_000 and hitcount_z >= 2: + points = min(15, 5 + int(hitcount_z * 3)) + score += points + reasons.append(f"unusually high policy hitcount ({total_hitcount}, max event value {max_hitcount})") + if event_count >= 5: deny_rate = deny_count / event_count if deny_count >= 10 and deny_rate >= 0.5: @@ -124,6 +180,10 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[ score += min(15, distinct_services) reasons.append(f"many distinct services used ({distinct_services})") + if distinct_dst_ports >= 10 and event_count >= 10: + score += min(15, distinct_dst_ports) + reasons.append(f"many destination ports contacted ({distinct_dst_ports})") + if _is_public_ip(src_ip) and (utm_count or deny_count >= 10): score += 10 reasons.append("public source with repeated security-relevant events") @@ -147,6 +207,13 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[ "utm_events": utm_count, "high_severity_events": high_severity_count, "total_bytes": total_bytes, + "events_per_minute": round(event_rate, 2) if event_rate is not None else 0.0, + "observed_duration_seconds": round(observed_duration, 2), + "timed_events": sum(1 for event in src_events if _event_timestamp(event) is not None), + "hitcount_total": total_hitcount, + "hitcount_max": max_hitcount, + "distinct_src_ports": distinct_src_ports, + "distinct_dst_ports": distinct_dst_ports, "policy_count": len(policies), "implicit_deny_events": implicit_deny_count, }, diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 2f60bda..6a42659 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -93,6 +93,11 @@ async function refresh() { {label:'Score', key:'score'}, {label:'Severity', render:r => `${esc(r.severity)}`}, {label:'Confidence', key:'confidence'}, + {label:'Rate / ports / hits', render:r => { + const e = r.evidence || {}; + const rate = e.timed_events > 1 ? `${e.events_per_minute} events/min` : 'no timestamps'; + return esc(`${rate}; dst ports: ${e.distinct_dst_ports || 0}; src ports: ${e.distinct_src_ports || 0}; hitcount: ${e.hitcount_total || 0}`); + }}, {label:'Reasons', render:r => esc((r.reasons || []).join('; '))} ]); document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [ @@ -127,6 +132,8 @@ async function refresh() { const d = data.diagnostics || {}; document.getElementById('diagnostics').innerHTML = '

Top Sources

' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + + '

Top Destination Ports

' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + + '

Top Source Ports

' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '

Top Services

' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '

Local-in Failures

' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip'}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]); } diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index 5961882..2474234 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -56,6 +56,8 @@ def build_status( "anomaly_summary": anomaly_summary(anomalies), "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), + "top_destination_ports": top_field_values(events, "dstport", limit=10), + "top_source_ports": top_field_values(events, "srcport", limit=10), "top_services": top_field_values(events, "service", limit=10), "top_actions": top_field_values(events, "action", limit=10), "top_subtypes": top_field_values(events, "subtype", limit=10), diff --git a/tests/test_anomaly.py b/tests/test_anomaly.py index 52a2dec..6e860d4 100644 --- a/tests/test_anomaly.py +++ b/tests/test_anomaly.py @@ -37,6 +37,30 @@ class AnomalyTests(unittest.TestCase): self.assertEqual(summary["total"], 1) + def test_includes_rate_ports_and_hitcount_in_evidence(self): + events = [ + parse_log_line( + f"eventtime={1_700_000_000_000_000_000 + index * 100_000_000} type=traffic " + f"srcip=203.0.113.10 dstip=10.0.0.{index} srcport={40000 + index} dstport={1000 + index} " + "service=tcp/unknown action=deny hitcount=200" + ) + for index in range(12) + ] + events.extend( + parse_log_line( + f"eventtime={1_700_000_000_000_000_000 + index * 60_000_000_000} type=traffic " + f"srcip=10.0.0.{index} dstip=1.1.1.1 service=https action=accept hitcount=1" + ) + for index in range(1, 8) + ) + + finding = next(item for item in detect_source_anomalies(events) if item.subject == "203.0.113.10") + + self.assertGreater(finding.evidence["events_per_minute"], 20) + self.assertEqual(finding.evidence["distinct_dst_ports"], 12) + self.assertEqual(finding.evidence["distinct_src_ports"], 12) + self.assertEqual(finding.evidence["hitcount_total"], 2400) + if __name__ == "__main__": unittest.main()