fix retention

README.md

@@ -458,16 +458,19 @@ five-minute windows. Historical rate and hitcount-rate deviations then contribut
 to its anomaly score. Set `FGAI_BASELINE_DB` to use another location.
 
 SignalScope prunes old baseline buckets during each monitor cycle. The defaults
-keep 14 days of buckets and dedupe history, prune stale one-off categorical
+keep 7 days of buckets and dedupe history, prune stale one-off categorical
-values after 7 days, and cap high-cardinality values per stream/entity/field.
+values after 3 days, and cap high-cardinality values per stream/entity/field.
+Very noisy fields such as raw messages, URLs, payloads, request/response bodies,
+tokens, sessions, hashes and long values are still counted in bucket baselines
+but are not stored as distinct rare-value candidates.
 Tune these in the dashboard or in `state/fgai-config.json`:
 
 ```json
 {
   "baseline_training_days": 7,
-  "baseline_retention_days": 14,
+  "baseline_retention_days": 7,
-  "baseline_value_retention_days": 7,
+  "baseline_value_retention_days": 3,
-  "baseline_max_values_per_field": 2000
+  "baseline_max_values_per_field": 500
 }
 ```
 
@@ -480,11 +483,12 @@ If an existing baseline database has already grown large, stop the monitor and
 run a manual prune plus SQLite compaction:
 
 ```bash
-signalscope baseline-maintenance --baseline-db state/fgai-baseline.sqlite3 --retention-days 14 --value-retention-days 7 --max-values-per-field 2000 --vacuum
+signalscope baseline-maintenance --baseline-db state/fgai-baseline.sqlite3 --retention-days 7 --value-retention-days 3 --max-values-per-field 500 --vacuum
 ```
 
-`VACUUM` can take time on a large database and should not be run while the
+`VACUUM` can take time on a large database, needs free disk space roughly equal
-monitor is actively writing.
+to the database size, and should not be run while the monitor is actively
+writing. Without `--vacuum`, SQLite may delete rows but keep the file size.
 
 Analyze local logs:
 

src/fgai/baseline.py

@@ -15,12 +15,37 @@ from .detectors import DETECTOR_MINIMUMS, event_detector_categories
 from .normalization import canonical_value
 from .query_details import event_query_details
 
-DEFAULT_RETENTION_DAYS = 14
+DEFAULT_RETENTION_DAYS = 7
-DEFAULT_VALUE_RETENTION_DAYS = 7
+DEFAULT_VALUE_RETENTION_DAYS = 3
-DEFAULT_MAX_VALUES_PER_FIELD = 2000
+DEFAULT_MAX_VALUES_PER_FIELD = 500
 DEFAULT_TRAINING_DAYS = 7
 MIN_REPORTED_DEVIATION_SCORE = 15
 MAX_RARE_VALUES_PER_ENTITY = 5
+MAX_PROFILE_VALUE_LENGTH = 160
+NOISY_VALUE_FIELD_PARTS = (
+    "answer",
+    "body",
+    "commandline",
+    "command_line",
+    "context",
+    "cookie",
+    "ephemeral",
+    "fingerprint",
+    "full_message",
+    "hash",
+    "message",
+    "payload",
+    "queryparameter",
+    "raw",
+    "request",
+    "response",
+    "session",
+    "stack",
+    "token",
+    "uri",
+    "url",
+    "user_agent",
+)
 
 
 def _number(value: str | None) -> int:
@@ -97,6 +122,17 @@ def _relationship_key(left: str, right: str) -> str:
     return f"relationship:{left.lower()}->{right.lower()}"
 
 
+def _should_store_profile_value(field: str, value: str) -> bool:
+    field = field.lower()
+    if field.startswith("relationship:"):
+        return True
+    if len(value) > MAX_PROFILE_VALUE_LENGTH:
+        return False
+    if any(part in field for part in NOISY_VALUE_FIELD_PARTS):
+        return False
+    return True
+
+
 class BaselineStore:
     """Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
 
@@ -206,7 +242,14 @@ class BaselineStore:
                 )
         return inserted
 
-    def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int:
+    def ingest_profile_fields(
+        self,
+        events: list[LogEvent],
+        profiles: dict[str, object],
+        *,
+        observed_at: int | None = None,
+        max_values_per_field: int = DEFAULT_MAX_VALUES_PER_FIELD,
+    ) -> int:
         observed_at = observed_at or int(time.time())
         pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
         temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
@@ -253,7 +296,7 @@ class BaselineStore:
                         temporal_pending[temporal][2] += value * value
                         if key[2] not in numeric:
                             raw_value = event.fields.get(key[2])
-                            if raw_value:
+                            if raw_value and _should_store_profile_value(key[2], raw_value):
                                 pending_values[(*key[:3], raw_value)] += 1
             for (stream_id, entity, field, bucket), values in pending.items():
                 connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
@@ -261,7 +304,23 @@ class BaselineStore:
             for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items():
                 connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?)
                   on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values))
+            value_counts: dict[tuple[str, str, str], int] = {}
             for key, count in pending_values.items():
+                if max_values_per_field > 0:
+                    group_key = key[:3]
+                    known = connection.execute(
+                        "select 1 from profile_values where stream_id=? and entity=? and field=? and value=?",
+                        key,
+                    ).fetchone()
+                    if known is None:
+                        if group_key not in value_counts:
+                            value_counts[group_key] = int(connection.execute(
+                                "select count(*) from profile_values where stream_id=? and entity=? and field=?",
+                                group_key,
+                            ).fetchone()[0])
+                        if value_counts[group_key] >= max_values_per_field:
+                            continue
+                        value_counts[group_key] += 1
                 connection.execute("""insert into profile_values values (?, ?, ?, ?, ?, ?)
                   on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count,last_seen=excluded.last_seen""", (*key, count, observed_at))
             for (stream_id, entity, detector, bucket), count in detector_pending.items():
@@ -538,7 +597,35 @@ class BaselineStore:
         if vacuum:
             with self._connect() as connection:
                 connection.execute("vacuum")
-        return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0}
+        stats = self.stats()
+        return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, **stats}
+
+    def stats(self) -> dict[str, object]:
+        with self._connect() as connection:
+            tables = (
+                "seen_events",
+                "source_buckets",
+                "source_values",
+                "profile_seen_events",
+                "profile_buckets",
+                "profile_temporal_buckets",
+                "profile_detector_buckets",
+                "profile_detector_temporal_buckets",
+                "profile_values",
+            )
+            rows = {table: int(connection.execute(f"select count(*) from {table}").fetchone()[0]) for table in tables}
+            page_size = int(connection.execute("pragma page_size").fetchone()[0])
+            page_count = int(connection.execute("pragma page_count").fetchone()[0])
+            freelist_count = int(connection.execute("pragma freelist_count").fetchone()[0])
+        size_bytes = self.path.stat().st_size if self.path.exists() else 0
+        return {
+            "size_bytes": size_bytes,
+            "page_size": page_size,
+            "page_count": page_count,
+            "freelist_count": freelist_count,
+            "reclaimable_bytes": freelist_count * page_size,
+            "rows": rows,
+        }
 
     def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
         rows: list[dict[str, object]] = []

src/fgai/config.py

@@ -19,9 +19,9 @@ DEFAULT_CONFIG: dict[str, object] = {
     "graylog_raw_sample_events": 5000,
     "graylog_mcp_call_timeout_seconds": 8,
     "graylog_mcp_poll_timeout_seconds": 120,
-    "baseline_retention_days": 14,
+    "baseline_retention_days": 7,
-    "baseline_value_retention_days": 7,
+    "baseline_value_retention_days": 3,
-    "baseline_max_values_per_field": 2000,
+    "baseline_max_values_per_field": 500,
     "baseline_training_days": 7,
     "graylog_field_mapping": "",
     "llm_enabled": False,

src/fgai/monitor.py

@@ -392,12 +392,19 @@ def build_status(
                     deviation["score"] = 0
     anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
     baseline_events = baseline.ingest(events) if baseline else 0
-    profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
+    profile_baseline_fields = (
+        baseline.ingest_profile_fields(
+            events,
+            stream_profiles,
+            max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500),
+        )
+        if baseline else 0
+    )
     baseline_maintenance = (
         baseline.maintenance(
-            retention_days=int(runtime_values.get("baseline_retention_days", 14) or 14),
+            retention_days=int(runtime_values.get("baseline_retention_days", 7) or 7),
-            value_retention_days=int(runtime_values.get("baseline_value_retention_days", 7) or 7),
+            value_retention_days=int(runtime_values.get("baseline_value_retention_days", 3) or 3),
-            max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 2000) or 2000),
+            max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500),
         )
         if baseline
         else {}

tests/test_baseline.py

@@ -109,6 +109,41 @@ class BaselineTests(unittest.TestCase):
             self.assertGreaterEqual(result["deleted"]["profile_buckets"], 1)
             self.assertGreaterEqual(result["deleted"]["seen_events"], 1)
             self.assertGreaterEqual(result["deleted"]["profile_seen_events"], 1)
+            self.assertIn("rows", result)
+            self.assertIn("reclaimable_bytes", result)
+
+    def test_profile_value_storage_skips_noisy_high_cardinality_fields(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+            profiles = parse_profiles([{"stream_id": "web", "entity_field": "srcip", "categorical_fields": ["url", "action"]}])
+            events = [
+                parse_log_line(f"fgai_stream_id=web srcip=10.0.0.5 action=allow url=/download/{index}/very/noisy/path event={index}")
+                for index in range(10)
+            ]
+
+            store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000)
+            stats = store.stats()
+
+            self.assertEqual(stats["rows"]["profile_buckets"], 2)
+            with store._connect() as connection:
+                fields = {row[0] for row in connection.execute("select distinct field from profile_values").fetchall()}
+            self.assertIn("action", fields)
+            self.assertNotIn("url", fields)
+
+    def test_profile_value_storage_caps_new_values_during_ingest(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+            profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}])
+            events = [
+                parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=value-{index}.example event={index}")
+                for index in range(5)
+            ]
+
+            store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000, max_values_per_field=2)
+
+            with store._connect() as connection:
+                stored = connection.execute("select count(*) from profile_values").fetchone()[0]
+            self.assertEqual(stored, 2)
 
     def test_rare_values_are_limited_per_entity(self):
         with tempfile.TemporaryDirectory() as directory: