add more

src/fgai/anomaly.py

@@ -75,9 +75,10 @@ def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]:
 
 
 def detect_source_anomalies(
-    events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None
+    events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None
 ) -> list[AnomalyFinding]:
     baselines = baselines or {}
+    field_deviations = field_deviations or {}
     by_src: dict[str, list[LogEvent]] = defaultdict(list)
     for event in events:
         if event.src_ip:
@@ -217,6 +218,10 @@ def detect_source_anomalies(
             score += 10
             reasons.append("public source with repeated security-relevant events")
 
+        for deviation in field_deviations.get(src_ip, []):
+            score += int(deviation.get("score", 0))
+            reasons.append(str(deviation.get("reason", "stream field baseline deviation")))
+
         if not reasons:
             continue
 

src/fgai/baseline.py

@@ -113,6 +113,34 @@ class BaselineStore:
                   on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
         return len(pending)
 
+    def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
+        current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
+        for event in events:
+            profile = profiles.get(event.fields.get("fgai_stream_id", ""))
+            if not profile:
+                continue
+            entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
+            if not entity:
+                continue
+            numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
+            for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
+                key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
+                current[key][0] += 1
+                if key[2] in numeric:
+                    current[key][1] += _number(event.fields.get(key[2]))
+        output: dict[str, list[dict[str, object]]] = defaultdict(list)
+        with self._connect() as connection:
+            for (stream, entity, field), values in current.items():
+                rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
+                if len(rows) < 12:
+                    continue
+                averages = [row[1] / row[0] if row[0] else 0 for row in rows]
+                baseline = mean(averages)
+                deviation = abs((values[1] / values[0] if values[0] else 0) - baseline)
+                if deviation > (pstdev(averages) or 1.0) * 3:
+                    output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": f"{field} deviates from its stream baseline"})
+        return output
+
     def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]:
         profiles: dict[str, dict[str, object]] = {}
         with self._connect() as connection:

src/fgai/monitor.py

@@ -62,7 +62,8 @@ def build_status(
                 events = []
     baseline = BaselineStore(baseline_path) if baseline_path else None
     profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
-    anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles)
+    field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {}
+    anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
     baseline_events = baseline.ingest(events) if baseline else 0
     profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
     intel_ips = sorted(
@@ -114,6 +115,7 @@ def build_status(
             "local_in_failures": local_in_failures(events, limit=10),
         },
         "event_context": build_event_context(events),
+        "field_deviations": field_deviations,
         "cross_source_correlations": correlate_source_ips(events),
         "anomalies": [
             {