Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add real detector health tooling: learning state and historical precision

Browse Source 
Replaces the old Prometheus-only "detector precision" metric, which
recomputed from whatever field_deviations happened to be active in the
current poll cycle cross-referenced against live FeedbackStore rules — a
revoked or expired suppression rule made the signal vanish, and nothing was
tracked between polls. Adds DetectorPrecisionStore, a permanent append-only
log of review outcomes (kept deliberately separate from FeedbackStore's
mutable suppression rules) recorded the moment a human reviews a finding,
plus BaselineStore.detector_readiness() mirroring the existing per-field
profile_readiness() for per-detector learning state, and a plain-language
detector reference. Surfaced in a new Detector Health dashboard panel.
Observability only — nothing here changes scoring or suppression.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-24 16:22:02 +02:00
parent b0cc9dbdfe
commit 36516faf9a
11 changed files with 315 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
tests/test_baseline.py
View File

@@ -254,3 +254,62 @@ class BaselineTests(unittest.TestCase):
self.assertEqual(relation["field"], "relationship:username->srcip")
self.assertEqual(relation["value"], "10.0.0.99")
self.assertIn("new srcip value for username=peter", relation["reason"])
def test_detector_readiness_reports_learning_before_twelve_buckets(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"detectors": {"auth_failure": {"enabled": True}},
}])
for index in range(5):
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
readiness = store.detector_readiness(profiles)
row = next(item for item in readiness if item["detector"] == "auth_failure")
self.assertEqual(row["stream_id"], "windows")
self.assertEqual(row["buckets"], 5)
self.assertFalse(row["ready"])
def test_detector_readiness_reports_ready_after_twelve_buckets(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"detectors": {"auth_failure": {"enabled": True}},
}])
for index in range(12):
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
readiness = store.detector_readiness(profiles)
row = next(item for item in readiness if item["detector"] == "auth_failure")
self.assertEqual(row["buckets"], 12)
self.assertTrue(row["ready"])
def test_detector_readiness_respects_min_training_days(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"detectors": {"auth_failure": {"enabled": True}},
}])
now = int(time.time())
for index in range(12):
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=now - 86400 + index * 300)
readiness = store.detector_readiness(profiles, min_training_days=7)
row = next(item for item in readiness if item["detector"] == "auth_failure")
self.assertTrue(row["buckets"] >= 12)
self.assertFalse(row["ready"])

11
tests/test_dashboard.py
View File

@@ -104,6 +104,17 @@ class DashboardTests(unittest.TestCase):
self.assertIn("Shared Entity Baselines", HTML)
self.assertIn("shared_entity_baselines", HTML)
def test_dashboard_shows_detector_health(self):
self.assertIn("detector-health", HTML)
self.assertIn("Detector Health", HTML)
self.assertIn("detector_health", HTML)
def test_review_actions_carry_detector_for_precision_tracking(self):
self.assertIn('data-detector="${esc(r.detector', HTML)
self.assertIn("detector: button.dataset.detector", HTML)
self.assertIn("detector: pendingReview.detector", HTML)
self.assertIn("detector: row.detector", HTML)
def test_dashboard_shows_shape_mismatch_warnings(self):
self.assertIn('id="profileShapeWarnings"', HTML)
self.assertIn("profile_shape_warnings", HTML)

51
tests/test_feedback.py
View File

@@ -3,7 +3,7 @@ import time
import unittest
from pathlib import Path
from fgai.feedback import FeedbackStore, feedback_matches, select_match, suggest_generalizations
from fgai.feedback import DetectorPrecisionStore, FeedbackStore, feedback_matches, select_match, suggest_generalizations
class FeedbackTests(unittest.TestCase):
@@ -157,6 +157,55 @@ class FeedbackTests(unittest.TestCase):
{"stream_id": "s", "entity": "alice", "field": "f", "status": "false_positive", "value_count": 2, "sample_values": ["1", "2"], "entry_ids": ["1", "2"]}
])
def test_detector_precision_store_computes_ratio_from_recorded_outcomes(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
store.record("auth_failure_burst", "false_positive", stream_id="windows", field="auth_failure")
store.record("new_relationship", "expected", stream_id="firewall", field="relationship:srcip->dstip")
precision = store.precision(days=30)
auth = precision["auth_failure_burst"]
self.assertEqual(auth["confirmed"], 2)
self.assertEqual(auth["false_positive"], 1)
self.assertAlmostEqual(auth["precision"], 2 / 3, places=3)
self.assertEqual(auth["reviewed_total"], 3)
self.assertIsNone(precision["new_relationship"]["precision"])
self.assertEqual(precision["new_relationship"]["reviewed_total"], 1)
def test_detector_precision_store_excludes_outcomes_outside_the_window(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
old_timestamp = int(time.time()) - 40 * 86400
store.record("auth_failure_burst", "confirmed", recorded_at=old_timestamp)
store.record("auth_failure_burst", "false_positive")
precision = store.precision(days=30)
self.assertEqual(precision["auth_failure_burst"]["confirmed"], 0)
self.assertEqual(precision["auth_failure_burst"]["false_positive"], 1)
def test_detector_precision_store_ignores_invalid_or_empty_input(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
store.record("", "confirmed")
store.record("auth_failure_burst", "not_a_real_status")
self.assertEqual(store.precision(days=30), {})
def test_detector_precision_store_maintenance_prunes_old_rows(self):
with tempfile.TemporaryDirectory() as directory:
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
store.record("auth_failure_burst", "confirmed", recorded_at=int(time.time()) - 200 * 86400)
store.record("auth_failure_burst", "confirmed")
removed = store.maintenance(retention_days=180)
self.assertEqual(removed, 1)
self.assertEqual(store.precision(days=3650)["auth_failure_burst"]["confirmed"], 1)
if __name__ == "__main__":
unittest.main()

37
tests/test_monitor.py
View File

@@ -258,6 +258,43 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(pool["group"], "srcip")
self.assertEqual(set(pool["streams"]), {"firewall", "iis"})
def test_detector_health_lists_known_detectors_with_no_data_by_default(self):
with tempfile.TemporaryDirectory() as tmp:
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json"))
health_by_detector = {item["detector"]: item for item in status["detector_health"]}
self.assertIn("auth_failure_burst", health_by_detector)
self.assertIn("new_relationship", health_by_detector)
self.assertEqual(health_by_detector["auth_failure_burst"]["learning_state"], "no_data")
self.assertIsNone(health_by_detector["auth_failure_burst"]["precision"])
def test_detector_health_reflects_readiness_and_precision(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"
config_path.write_text(
json.dumps(
{
"graylog_streams": [{"id": "windows", "title": "Windows", "enabled": True}],
"graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "categorical_fields": ["action"], "detectors": {"auth_failure": {"enabled": True}}}],
}
),
encoding="utf-8",
)
readiness_rows = [{"stream_id": "windows", "detector": "auth_failure", "buckets": 12, "age_days": 10, "training_days": 0, "ready": True}]
precision_rows = {"auth_failure_burst": {"confirmed": 3, "false_positive": 1, "expected": 0, "precision": 0.75, "reviewed_total": 4}}
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]), \
patch("fgai.monitor.BaselineStore.detector_readiness", return_value=readiness_rows), \
patch("fgai.monitor.DetectorPrecisionStore.precision", return_value=precision_rows):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
row = next(item for item in status["detector_health"] if item["detector"] == "auth_failure_burst")
self.assertEqual(row["learning_state"], "ready")
self.assertEqual(row["streams_ready"], 1)
self.assertEqual(row["streams_configured"], 1)
self.assertEqual(row["confirmed"], 3)
self.assertEqual(row["precision"], 0.75)
def test_shared_entity_baselines_empty_for_single_stream(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"