add more

src/fgai/baseline.py

@@ -113,6 +113,34 @@ class BaselineStore:
                   on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
         return len(pending)
 
+    def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
+        current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
+        for event in events:
+            profile = profiles.get(event.fields.get("fgai_stream_id", ""))
+            if not profile:
+                continue
+            entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
+            if not entity:
+                continue
+            numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
+            for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
+                key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
+                current[key][0] += 1
+                if key[2] in numeric:
+                    current[key][1] += _number(event.fields.get(key[2]))
+        output: dict[str, list[dict[str, object]]] = defaultdict(list)
+        with self._connect() as connection:
+            for (stream, entity, field), values in current.items():
+                rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
+                if len(rows) < 12:
+                    continue
+                averages = [row[1] / row[0] if row[0] else 0 for row in rows]
+                baseline = mean(averages)
+                deviation = abs((values[1] / values[0] if values[0] else 0) - baseline)
+                if deviation > (pstdev(averages) or 1.0) * 3:
+                    output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": f"{field} deviates from its stream baseline"})
+        return output
+
     def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]:
         profiles: dict[str, dict[str, object]] = {}
         with self._connect() as connection: