Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add feedback scopes, revoke and precision metrics

Browse Source 
Feedback rules now support value/field/entity/stream scope, require
expiry for broad suppressions, and can be revoked individually.
Adds an active-rules panel, detector precision metrics, and diagnostics.
This commit is contained in:
larssand
2026-09-07 19:51:54 +02:00
parent e9e0ab8a46
commit 7b94512abc
11 changed files with 393 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/test_dashboard.py
View File

@@ -76,6 +76,22 @@ class DashboardTests(unittest.TestCase):
self.assertIn("/api/feedback/clear", source)
self.assertIn("/api/review-state/clear", source)
def test_dashboard_exposes_feedback_revoke_endpoint(self):
source = inspect.getsource(dashboard)
self.assertIn("/api/feedback/revoke", source)
self.assertIn(".revoke(", source)
def test_dashboard_has_active_feedback_rules_panel(self):
self.assertIn('id="feedbackRules"', HTML)
self.assertIn("revoke-feedback-rule", HTML)
self.assertIn('id="feedbackRuleSummary"', HTML)
def test_dashboard_has_scoped_review_form(self):
self.assertIn('id="reviewScope"', HTML)
self.assertIn('value="entity"', HTML)
self.assertIn('value="stream"', HTML)
self.assertIn("Broad-scope rules must expire", HTML)
def test_dashboard_does_not_clear_streams_when_picker_is_unloaded(self):
self.assertIn("const streamValues = Object.values(window.streamSelection || {})", HTML)
self.assertIn("if (streamValues.length)", HTML)

71
tests/test_feedback.py
View File

@@ -1,8 +1,9 @@
import tempfile
import time
import unittest
from pathlib import Path
from fgai.feedback import FeedbackStore
from fgai.feedback import FeedbackStore, feedback_matches, select_match
class FeedbackTests(unittest.TestCase):
@@ -20,6 +21,74 @@ class FeedbackTests(unittest.TestCase):
self.assertEqual(all_items, {"removed": 1, "remaining": 0, "status": "all"})
self.assertEqual(store.entries(), [])
def test_feedback_scope_defaults_and_ids_are_stable(self):
with tempfile.TemporaryDirectory() as directory:
store = FeedbackStore(str(Path(directory) / "feedback.json"))
no_value = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username"})
self.assertEqual(no_value["scope"], "field")
with_value = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username", "value": "bob"})
self.assertEqual(with_value["scope"], "value")
self.assertEqual(len(store.entries()), 2)
again = store.add({"status": "confirmed", "entity": "alice", "stream_id": "windows", "field": "username"})
self.assertEqual(again["id"], no_value["id"])
self.assertEqual(len(store.entries()), 2)
self.assertEqual(next(item for item in store.entries() if item["id"] == no_value["id"]).get("status"), "confirmed")
def test_feedback_broad_scope_requires_expiry(self):
with tempfile.TemporaryDirectory() as directory:
store = FeedbackStore(str(Path(directory) / "feedback.json"))
with self.assertRaises(ValueError):
store.add({"status": "expected", "scope": "entity", "entity": "alice"})
entry = store.add({"status": "expected", "scope": "entity", "entity": "alice", "expires_at": int(time.time()) + 3600})
self.assertEqual(entry["scope"], "entity")
self.assertEqual(entry["stream_id"], "")
def test_feedback_stream_scope_requires_stream_id(self):
with tempfile.TemporaryDirectory() as directory:
store = FeedbackStore(str(Path(directory) / "feedback.json"))
with self.assertRaises(ValueError):
store.add({"status": "expected", "scope": "stream", "expires_at": int(time.time()) + 3600})
entry = store.add({"status": "expected", "scope": "stream", "stream_id": "vpn", "expires_at": int(time.time()) + 3600})
self.assertEqual(entry["entity"], "")
def test_feedback_entity_scope_requires_entity(self):
with tempfile.TemporaryDirectory() as directory:
store = FeedbackStore(str(Path(directory) / "feedback.json"))
with self.assertRaises(ValueError):
store.add({"status": "expected", "scope": "entity", "expires_at": int(time.time()) + 3600})
def test_feedback_revoke_removes_single_rule_by_id(self):
with tempfile.TemporaryDirectory() as directory:
store = FeedbackStore(str(Path(directory) / "feedback.json"))
one = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username"})
store.add({"status": "confirmed", "entity": "bob", "stream_id": "vpn", "field": "username"})
result = store.revoke(one["id"])
self.assertEqual(result, {"removed": 1, "remaining": 1, "id": one["id"]})
self.assertEqual(len(store.entries()), 1)
missing = store.revoke("does-not-exist")
self.assertEqual(missing, {"removed": 0, "remaining": 1, "id": "does-not-exist"})
def test_feedback_matches_respects_scope_and_precedence(self):
value_rule = {"scope": "value", "entity": "alice", "stream_id": "windows", "field": "username", "value": "10.0.0.1", "status": "confirmed"}
field_rule = {"scope": "field", "entity": "alice", "stream_id": "windows", "field": "username", "status": "expected"}
entity_rule = {"scope": "entity", "entity": "alice", "status": "expected"}
stream_rule = {"scope": "stream", "stream_id": "windows", "status": "expected"}
self.assertTrue(feedback_matches(stream_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.1"))
self.assertFalse(feedback_matches(stream_rule, entity="alice", stream_id="vpn", field="username", value="10.0.0.1"))
self.assertTrue(feedback_matches(entity_rule, entity="alice", stream_id="vpn", field="hostname", value="x"))
self.assertFalse(feedback_matches(field_rule, entity="alice", stream_id="windows", field="hostname", value="x"))
self.assertTrue(feedback_matches(value_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.1"))
self.assertFalse(feedback_matches(value_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.2"))
candidates = [stream_rule, entity_rule, field_rule, value_rule]
self.assertEqual(select_match(candidates), value_rule)
self.assertEqual(select_match([stream_rule, entity_rule]), entity_rule)
if __name__ == "__main__":
unittest.main()

42
tests/test_metrics.py
View File

@@ -8,3 +8,45 @@ class MetricsTests(unittest.TestCase):
output = prometheus_metrics({"summary": {"total": 10}, "anomaly_summary": {"high": 2}, "baseline": {"sources_ready": 3}, "capabilities": {"graylog_mcp": {"status": "connected", "events_fetched": 9}}})
self.assertIn('fgai_events_total{kind="total"} 10', output)
self.assertIn("fgai_graylog_mcp_connected 1", output)
def test_feedback_rules_counted_by_scope(self):
status = {
"feedback": [
{"scope": "entity", "status": "expected"},
{"scope": "value", "status": "confirmed"},
],
"field_deviations": {},
}
text = prometheus_metrics(status)
self.assertIn('fgai_feedback_rules{scope="entity"} 1', text)
self.assertIn('fgai_feedback_rules{scope="value"} 1', text)
self.assertIn('fgai_feedback_rules{scope="field"} 0', text)
self.assertIn('fgai_feedback_rules{scope="stream"} 0', text)
def test_suppressed_findings_and_detector_precision(self):
status = {
"feedback": [],
"field_deviations": {
"alice": [
{"detector": "rare_value", "feedback": "confirmed"},
{"detector": "rare_value", "feedback": "false_positive"},
{"detector": "rare_value", "feedback": "expected"},
{"detector": "auth_failure_burst", "feedback": "unreviewed"},
]
},
}
text = prometheus_metrics(status)
self.assertIn("fgai_feedback_suppressed_findings 2", text)
self.assertIn('fgai_detector_feedback_total{detector="rare_value",verdict="confirmed"} 1', text)
self.assertIn('fgai_detector_feedback_total{detector="rare_value",verdict="false_positive"} 1', text)
self.assertIn('fgai_detector_precision{detector="rare_value"} 0.5000', text)
self.assertNotIn('detector="auth_failure_burst"', text)
def test_metrics_handle_empty_status(self):
text = prometheus_metrics({})
self.assertIn('fgai_feedback_rules{scope="value"} 0', text)
self.assertIn("fgai_feedback_suppressed_findings 0", text)
if __name__ == "__main__":
unittest.main()