Implemented the baseline/noise and SQLite growth improvements.

tests/test_baseline.py

@@ -1,4 +1,5 @@
 import tempfile
+import time
 import unittest
 from pathlib import Path
 
@@ -89,3 +90,43 @@ class BaselineTests(unittest.TestCase):
             auth = next(item for item in deviations if item["detector"] == "auth_failure_burst")
             self.assertEqual(auth["weight"], 2.0)
             self.assertGreater(auth["score"], auth["base_score"])
+
+    def test_maintenance_prunes_old_baseline_rows(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+            profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}])
+            old = int(time.time()) - 20 * 86400
+            recent = int(time.time())
+
+            store.ingest([parse_log_line("srcip=10.0.0.1 dstport=443 old=1")], observed_at=old)
+            store.ingest_profile_fields([parse_log_line("fgai_stream_id=windows username=alice action=login old=1")], profiles, observed_at=old)
+            store.ingest([parse_log_line("srcip=10.0.0.2 dstport=443 recent=1")], observed_at=recent)
+            store.ingest_profile_fields([parse_log_line("fgai_stream_id=windows username=bob action=login recent=1")], profiles, observed_at=recent)
+
+            result = store.maintenance(retention_days=7, value_retention_days=7, max_values_per_field=2000)
+
+            self.assertGreaterEqual(result["deleted"]["source_buckets"], 1)
+            self.assertGreaterEqual(result["deleted"]["profile_buckets"], 1)
+            self.assertGreaterEqual(result["deleted"]["seen_events"], 1)
+            self.assertGreaterEqual(result["deleted"]["profile_seen_events"], 1)
+
+    def test_rare_values_are_limited_per_entity(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+            profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}])
+            now = int(time.time()) - 86400
+            baseline = [
+                parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=known-{index}.example baseline={index}")
+                for index in range(30)
+            ]
+            store.ingest_profile_fields(baseline, profiles, observed_at=now)
+
+            burst = [
+                parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=new-{index}.example burst={index}")
+                for index in range(12)
+            ]
+            deviations = store.profile_deviations(burst, profiles)["10.0.0.5"]
+            rare = [item for item in deviations if item["detector"] == "rare_value"]
+
+            self.assertLessEqual(len(rare), 5)
+            self.assertTrue(all(item["score"] >= 15 for item in rare))