Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Surface baseline maintenance status in the dashboard

Browse Source 
Retention and per-field value capping already ran automatically every poll
cycle, but the dashboard showed only a single "DB size" line — no
visibility into what got deleted, which fields are hitting the
max-values-per-field cap, or whether disk space is reclaimable, without
SSHing in. baseline.py's maintenance() now also reports the specific
(stream, entity, field) rows it trimmed for exceeding the cap, surfaced in
a new Baseline Maintenance dashboard panel alongside size/reclaimable bytes
and rows deleted last cycle. Deliberately no live-vacuum button: VACUUM
needs the service stopped first, so the dashboard only shows when it's
worth doing and points at the existing offline CLI command.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-24 16:35:09 +02:00
parent 36516faf9a
commit e61403d7f6
6 changed files with 79 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
tests/test_baseline.py
View File

@@ -112,6 +112,35 @@ class BaselineTests(unittest.TestCase):
self.assertIn("rows", result)
self.assertIn("reclaimable_bytes", result)
def test_maintenance_reports_high_cardinality_fields_it_trims(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}])
events = [
parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=value-{index}.example event={index}")
for index in range(5)
]
store.ingest_profile_fields(events, profiles, observed_at=int(time.time()), max_values_per_field=0)
result = store.maintenance(retention_days=7, value_retention_days=7, max_values_per_field=2)
self.assertEqual(result["deleted"]["profile_values_capped"], 3)
row = next(item for item in result["high_cardinality_fields"] if item["field"] == "query_domain")
self.assertEqual(row["stream_id"], "dns")
self.assertEqual(row["entity"], "10.0.0.5")
self.assertEqual(row["distinct_values"], 5)
self.assertEqual(row["trimmed_to"], 2)
def test_maintenance_reports_no_high_cardinality_fields_when_under_cap(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}])
store.ingest_profile_fields([parse_log_line("fgai_stream_id=dns srcip=10.0.0.5 query_domain=only.example")], profiles, observed_at=int(time.time()))
result = store.maintenance(retention_days=7, value_retention_days=7, max_values_per_field=500)
self.assertEqual(result["high_cardinality_fields"], [])
def test_profile_value_storage_skips_noisy_high_cardinality_fields(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))

6
tests/test_dashboard.py
View File

@@ -109,6 +109,12 @@ class DashboardTests(unittest.TestCase):
self.assertIn("Detector Health", HTML)
self.assertIn("detector_health", HTML)
def test_dashboard_shows_baseline_maintenance(self):
self.assertIn("baseline-maintenance", HTML)
self.assertIn("Baseline Maintenance", HTML)
self.assertIn("high_cardinality_fields", HTML)
self.assertIn("baseline-maintenance --vacuum", HTML)
def test_review_actions_carry_detector_for_precision_tracking(self):
self.assertIn('data-detector="${esc(r.detector', HTML)
self.assertIn("detector: button.dataset.detector", HTML)