Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add real detector health tooling: learning state and historical precision

Browse Source 
Replaces the old Prometheus-only "detector precision" metric, which
recomputed from whatever field_deviations happened to be active in the
current poll cycle cross-referenced against live FeedbackStore rules — a
revoked or expired suppression rule made the signal vanish, and nothing was
tracked between polls. Adds DetectorPrecisionStore, a permanent append-only
log of review outcomes (kept deliberately separate from FeedbackStore's
mutable suppression rules) recorded the moment a human reviews a finding,
plus BaselineStore.detector_readiness() mirroring the existing per-field
profile_readiness() for per-detector learning state, and a plain-language
detector reference. Surfaced in a new Detector Health dashboard panel.
Observability only — nothing here changes scoring or suppression.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-24 16:22:02 +02:00
parent b0cc9dbdfe
commit 36516faf9a
11 changed files with 315 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
tests/test_monitor.py
View File

@@ -258,6 +258,43 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(pool["group"], "srcip")
self.assertEqual(set(pool["streams"]), {"firewall", "iis"})
def test_detector_health_lists_known_detectors_with_no_data_by_default(self):
with tempfile.TemporaryDirectory() as tmp:
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json"))
health_by_detector = {item["detector"]: item for item in status["detector_health"]}
self.assertIn("auth_failure_burst", health_by_detector)
self.assertIn("new_relationship", health_by_detector)
self.assertEqual(health_by_detector["auth_failure_burst"]["learning_state"], "no_data")
self.assertIsNone(health_by_detector["auth_failure_burst"]["precision"])
def test_detector_health_reflects_readiness_and_precision(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"
config_path.write_text(
json.dumps(
{
"graylog_streams": [{"id": "windows", "title": "Windows", "enabled": True}],
"graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "categorical_fields": ["action"], "detectors": {"auth_failure": {"enabled": True}}}],
}
),
encoding="utf-8",
)
readiness_rows = [{"stream_id": "windows", "detector": "auth_failure", "buckets": 12, "age_days": 10, "training_days": 0, "ready": True}]
precision_rows = {"auth_failure_burst": {"confirmed": 3, "false_positive": 1, "expected": 0, "precision": 0.75, "reviewed_total": 4}}
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]), \
patch("fgai.monitor.BaselineStore.detector_readiness", return_value=readiness_rows), \
patch("fgai.monitor.DetectorPrecisionStore.precision", return_value=precision_rows):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
row = next(item for item in status["detector_health"] if item["detector"] == "auth_failure_burst")
self.assertEqual(row["learning_state"], "ready")
self.assertEqual(row["streams_ready"], 1)
self.assertEqual(row["streams_configured"], 1)
self.assertEqual(row["confirmed"], 3)
self.assertEqual(row["precision"], 0.75)
def test_shared_entity_baselines_empty_for_single_stream(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"