Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add real detector health tooling: learning state and historical precision

Browse Source 
Replaces the old Prometheus-only "detector precision" metric, which
recomputed from whatever field_deviations happened to be active in the
current poll cycle cross-referenced against live FeedbackStore rules — a
revoked or expired suppression rule made the signal vanish, and nothing was
tracked between polls. Adds DetectorPrecisionStore, a permanent append-only
log of review outcomes (kept deliberately separate from FeedbackStore's
mutable suppression rules) recorded the moment a human reviews a finding,
plus BaselineStore.detector_readiness() mirroring the existing per-field
profile_readiness() for per-detector learning state, and a plain-language
detector reference. Surfaced in a new Detector Health dashboard panel.
Observability only — nothing here changes scoring or suppression.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-24 16:22:02 +02:00
parent b0cc9dbdfe
commit 36516faf9a
11 changed files with 315 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
tests/test_baseline.py
View File

@@ -254,3 +254,62 @@ class BaselineTests(unittest.TestCase):
self.assertEqual(relation["field"], "relationship:username->srcip")
self.assertEqual(relation["value"], "10.0.0.99")
self.assertIn("new srcip value for username=peter", relation["reason"])
def test_detector_readiness_reports_learning_before_twelve_buckets(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"detectors": {"auth_failure": {"enabled": True}},
}])
for index in range(5):
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
readiness = store.detector_readiness(profiles)
row = next(item for item in readiness if item["detector"] == "auth_failure")
self.assertEqual(row["stream_id"], "windows")
self.assertEqual(row["buckets"], 5)
self.assertFalse(row["ready"])
def test_detector_readiness_reports_ready_after_twelve_buckets(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"detectors": {"auth_failure": {"enabled": True}},
}])
for index in range(12):
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
readiness = store.detector_readiness(profiles)
row = next(item for item in readiness if item["detector"] == "auth_failure")
self.assertEqual(row["buckets"], 12)
self.assertTrue(row["ready"])
def test_detector_readiness_respects_min_training_days(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"detectors": {"auth_failure": {"enabled": True}},
}])
now = int(time.time())
for index in range(12):
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=now - 86400 + index * 300)
readiness = store.detector_readiness(profiles, min_training_days=7)
row = next(item for item in readiness if item["detector"] == "auth_failure")
self.assertTrue(row["buckets"] >= 12)
self.assertFalse(row["ready"])