add more fields

src/fgai/anomaly.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import ipaddress
 from collections import Counter, defaultdict
+from datetime import datetime
 from statistics import mean, pstdev
 
 from .logs import THREAT_ACTIONS, event_score, is_utm_event
@@ -42,6 +43,37 @@ def _is_public_ip(value: str) -> bool:
         return False
 
 
+def _event_timestamp(event: LogEvent) -> float | None:
+    """Return the FortiGate event time in seconds when present."""
+    eventtime = event.fields.get("eventtime")
+    if eventtime:
+        try:
+            value = float(eventtime)
+            # Exports can use seconds, milliseconds, microseconds, or nanoseconds.
+            while value > 10_000_000_000:
+                value /= 1_000
+            return value
+        except ValueError:
+            pass
+
+    date = event.fields.get("date")
+    clock = event.fields.get("time")
+    if date and clock:
+        try:
+            return datetime.fromisoformat(f"{date}T{clock}").timestamp()
+        except ValueError:
+            return None
+    return None
+
+
+def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]:
+    timestamps = [timestamp for event in events if (timestamp := _event_timestamp(event)) is not None]
+    if len(timestamps) < 2:
+        return None, 0.0
+    duration_seconds = max(timestamps) - min(timestamps)
+    return len(timestamps) * 60 / max(1.0, duration_seconds), duration_seconds
+
+
 def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[AnomalyFinding]:
     by_src: dict[str, list[LogEvent]] = defaultdict(list)
     for event in events:
@@ -60,19 +92,30 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
         sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events)
         for src_events in by_src.values()
     ]
+    hitcount_totals = [sum(_as_int(event.fields.get("hitcount")) for event in src_events) for src_events in by_src.values()]
+    source_rates = [rate for src_events in by_src.values() if (rate := _rate_per_minute(src_events)[0]) is not None]
     avg_events = mean(event_counts)
     std_events = pstdev(event_counts) or 1.0
     avg_dst = mean(distinct_dst_counts)
     std_dst = pstdev(distinct_dst_counts) or 1.0
     avg_bytes = mean(byte_totals)
     std_bytes = pstdev(byte_totals) or 1.0
+    avg_hitcount = mean(hitcount_totals)
+    std_hitcount = pstdev(hitcount_totals) or 1.0
+    avg_rate = mean(source_rates) if source_rates else 0.0
+    std_rate = (pstdev(source_rates) or 1.0) if source_rates else 1.0
 
     findings: list[AnomalyFinding] = []
     for src_ip, src_events in by_src.items():
         event_count = len(src_events)
         distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
         distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")})
+        distinct_src_ports = len({event.fields.get("srcport") for event in src_events if event.fields.get("srcport")})
+        distinct_dst_ports = len({event.fields.get("dstport") for event in src_events if event.fields.get("dstport")})
         total_bytes = sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events)
+        total_hitcount = sum(_as_int(event.fields.get("hitcount")) for event in src_events)
+        max_hitcount = max((_as_int(event.fields.get("hitcount")) for event in src_events), default=0)
+        event_rate, observed_duration = _rate_per_minute(src_events)
         deny_count = sum(1 for event in src_events if event.action in THREAT_ACTIONS)
         utm_count = sum(1 for event in src_events if is_utm_event(event))
         high_severity_count = sum(1 for event in src_events if event.severity in {"critical", "high", "alert", "emergency"})
@@ -88,6 +131,13 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
             score += points
             reasons.append(f"unusually high event volume for source ({event_count} events, z={event_z:.1f})")
 
+        if event_rate is not None:
+            rate_z = (event_rate - avg_rate) / std_rate
+            if event_rate >= 20 and rate_z >= 2:
+                points = min(25, 10 + int(rate_z * 5))
+                score += points
+                reasons.append(f"unusually high log rate ({event_rate:.1f} events/min, z={rate_z:.1f})")
+
         dst_z = (distinct_dst - avg_dst) / std_dst
         if distinct_dst >= 10 and dst_z >= 2:
             points = min(25, 10 + int(dst_z * 5))
@@ -100,6 +150,12 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
             score += points
             reasons.append(f"unusually high byte volume ({total_bytes} bytes, z={byte_z:.1f})")
 
+        hitcount_z = (total_hitcount - avg_hitcount) / std_hitcount
+        if total_hitcount >= 1_000 and hitcount_z >= 2:
+            points = min(15, 5 + int(hitcount_z * 3))
+            score += points
+            reasons.append(f"unusually high policy hitcount ({total_hitcount}, max event value {max_hitcount})")
+
         if event_count >= 5:
             deny_rate = deny_count / event_count
             if deny_count >= 10 and deny_rate >= 0.5:
@@ -124,6 +180,10 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
             score += min(15, distinct_services)
             reasons.append(f"many distinct services used ({distinct_services})")
 
+        if distinct_dst_ports >= 10 and event_count >= 10:
+            score += min(15, distinct_dst_ports)
+            reasons.append(f"many destination ports contacted ({distinct_dst_ports})")
+
         if _is_public_ip(src_ip) and (utm_count or deny_count >= 10):
             score += 10
             reasons.append("public source with repeated security-relevant events")
@@ -147,6 +207,13 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
                     "utm_events": utm_count,
                     "high_severity_events": high_severity_count,
                     "total_bytes": total_bytes,
+                    "events_per_minute": round(event_rate, 2) if event_rate is not None else 0.0,
+                    "observed_duration_seconds": round(observed_duration, 2),
+                    "timed_events": sum(1 for event in src_events if _event_timestamp(event) is not None),
+                    "hitcount_total": total_hitcount,
+                    "hitcount_max": max_hitcount,
+                    "distinct_src_ports": distinct_src_ports,
+                    "distinct_dst_ports": distinct_dst_ports,
                     "policy_count": len(policies),
                     "implicit_deny_events": implicit_deny_count,
                 },