Added categorical field-value baselines.

src/fgai/baseline.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import hashlib
 import sqlite3
 import time
-from collections import defaultdict
+from collections import Counter, defaultdict
 from pathlib import Path
 from statistics import mean, pstdev
 
@@ -44,6 +44,10 @@ class BaselineStore:
                   events integer not null, numeric_sum real not null, numeric_sum_squares real not null,
                   primary key (stream_id, entity, field, bucket_start)
                 );
+                create table if not exists profile_values (
+                  stream_id text not null, entity text not null, field text not null, value text not null,
+                  seen_count integer not null, primary key (stream_id, entity, field, value)
+                );
                 """
             )
 
@@ -90,6 +94,7 @@ class BaselineStore:
         observed_at = observed_at or int(time.time())
         bucket = observed_at - (observed_at % self.bucket_seconds)
         pending: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
+        pending_values: Counter[tuple[str, str, str, str]] = Counter()
         for event in events:
             stream_id = event.fields.get("fgai_stream_id", "")
             profile = profiles.get(stream_id)
@@ -107,10 +112,17 @@ class BaselineStore:
                 pending[key][0] += 1
                 pending[key][1] += value
                 pending[key][2] += value * value
+                if key[2] not in numeric:
+                    raw_value = event.fields.get(key[2])
+                    if raw_value:
+                        pending_values[(*key, raw_value)] += 1
         with self._connect() as connection:
             for (stream_id, entity, field), values in pending.items():
                 connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
                   on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
+            for key, count in pending_values.items():
+                connection.execute("""insert into profile_values values (?, ?, ?, ?, ?)
+                  on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count""", (*key, count))
         return len(pending)
 
     def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
@@ -145,6 +157,26 @@ class BaselineStore:
                 deviation = abs(current_value - mean(history))
                 if deviation > (pstdev(history) or 1.0) * 3:
                     output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": reason})
+            # Detect selected categorical values that have not appeared for this entity in prior data.
+            for event in events:
+                profile = profiles.get(event.fields.get("fgai_stream_id", ""))
+                if not profile:
+                    continue
+                entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
+                if not entity:
+                    continue
+                stream = event.fields.get("fgai_stream_id", "")
+                for field in getattr(profile, "categorical_fields", ()):
+                    field = str(field).lower()
+                    value = event.fields.get(field)
+                    if not value:
+                        continue
+                    known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
+                    known_total = connection.execute("select count(*) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
+                    if known is None and known_total >= 10:
+                        evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value}
+                        if evidence not in output[entity]:
+                            output[entity].append(evidence)
         return output
 
     def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]: