Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add real detector health tooling: learning state and historical precision

Browse Source 
Replaces the old Prometheus-only "detector precision" metric, which
recomputed from whatever field_deviations happened to be active in the
current poll cycle cross-referenced against live FeedbackStore rules — a
revoked or expired suppression rule made the signal vanish, and nothing was
tracked between polls. Adds DetectorPrecisionStore, a permanent append-only
log of review outcomes (kept deliberately separate from FeedbackStore's
mutable suppression rules) recorded the moment a human reviews a finding,
plus BaselineStore.detector_readiness() mirroring the existing per-field
profile_readiness() for per-detector learning state, and a plain-language
detector reference. Surfaced in a new Detector Health dashboard panel.
Observability only — nothing here changes scoring or suppression.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-24 16:22:02 +02:00
parent b0cc9dbdfe
commit 36516faf9a
11 changed files with 315 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
tests/test_feedback.py
View File

@@ -3,7 +3,7 @@ import time
import unittest
from pathlib import Path
from fgai.feedback import FeedbackStore, feedback_matches, select_match, suggest_generalizations
from fgai.feedback import DetectorPrecisionStore, FeedbackStore, feedback_matches, select_match, suggest_generalizations
class FeedbackTests(unittest.TestCase):
@@ -157,6 +157,55 @@ class FeedbackTests(unittest.TestCase):
{"stream_id": "s", "entity": "alice", "field": "f", "status": "false_positive", "value_count": 2, "sample_values": ["1", "2"], "entry_ids": ["1", "2"]}
])
def test_detector_precision_store_computes_ratio_from_recorded_outcomes(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
store.record("auth_failure_burst", "false_positive", stream_id="windows", field="auth_failure")
store.record("new_relationship", "expected", stream_id="firewall", field="relationship:srcip->dstip")
precision = store.precision(days=30)
auth = precision["auth_failure_burst"]
self.assertEqual(auth["confirmed"], 2)
self.assertEqual(auth["false_positive"], 1)
self.assertAlmostEqual(auth["precision"], 2 / 3, places=3)
self.assertEqual(auth["reviewed_total"], 3)
self.assertIsNone(precision["new_relationship"]["precision"])
self.assertEqual(precision["new_relationship"]["reviewed_total"], 1)
def test_detector_precision_store_excludes_outcomes_outside_the_window(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
old_timestamp = int(time.time()) - 40 * 86400
store.record("auth_failure_burst", "confirmed", recorded_at=old_timestamp)
store.record("auth_failure_burst", "false_positive")
precision = store.precision(days=30)
self.assertEqual(precision["auth_failure_burst"]["confirmed"], 0)
self.assertEqual(precision["auth_failure_burst"]["false_positive"], 1)
def test_detector_precision_store_ignores_invalid_or_empty_input(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
store.record("", "confirmed")
store.record("auth_failure_burst", "not_a_real_status")
self.assertEqual(store.precision(days=30), {})
def test_detector_precision_store_maintenance_prunes_old_rows(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
store.record("auth_failure_burst", "confirmed", recorded_at=int(time.time()) - 200 * 86400)
store.record("auth_failure_burst", "confirmed")
removed = store.maintenance(retention_days=180)
self.assertEqual(removed, 1)
self.assertEqual(store.precision(days=3650)["auth_failure_burst"]["confirmed"], 1)
if __name__ == "__main__":
unittest.main()