add feedback scopes, revoke and precision metrics
Feedback rules now support value/field/entity/stream scope, require expiry for broad suppressions, and can be revoked individually. Adds an active-rules panel, detector precision metrics, and diagnostics.
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from fgai.feedback import FeedbackStore
|
||||
from fgai.feedback import FeedbackStore, feedback_matches, select_match
|
||||
|
||||
|
||||
class FeedbackTests(unittest.TestCase):
|
||||
@@ -20,6 +21,74 @@ class FeedbackTests(unittest.TestCase):
|
||||
self.assertEqual(all_items, {"removed": 1, "remaining": 0, "status": "all"})
|
||||
self.assertEqual(store.entries(), [])
|
||||
|
||||
def test_feedback_scope_defaults_and_ids_are_stable(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = FeedbackStore(str(Path(directory) / "feedback.json"))
|
||||
no_value = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username"})
|
||||
self.assertEqual(no_value["scope"], "field")
|
||||
|
||||
with_value = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username", "value": "bob"})
|
||||
self.assertEqual(with_value["scope"], "value")
|
||||
self.assertEqual(len(store.entries()), 2)
|
||||
|
||||
again = store.add({"status": "confirmed", "entity": "alice", "stream_id": "windows", "field": "username"})
|
||||
self.assertEqual(again["id"], no_value["id"])
|
||||
self.assertEqual(len(store.entries()), 2)
|
||||
self.assertEqual(next(item for item in store.entries() if item["id"] == no_value["id"]).get("status"), "confirmed")
|
||||
|
||||
def test_feedback_broad_scope_requires_expiry(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = FeedbackStore(str(Path(directory) / "feedback.json"))
|
||||
with self.assertRaises(ValueError):
|
||||
store.add({"status": "expected", "scope": "entity", "entity": "alice"})
|
||||
entry = store.add({"status": "expected", "scope": "entity", "entity": "alice", "expires_at": int(time.time()) + 3600})
|
||||
self.assertEqual(entry["scope"], "entity")
|
||||
self.assertEqual(entry["stream_id"], "")
|
||||
|
||||
def test_feedback_stream_scope_requires_stream_id(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = FeedbackStore(str(Path(directory) / "feedback.json"))
|
||||
with self.assertRaises(ValueError):
|
||||
store.add({"status": "expected", "scope": "stream", "expires_at": int(time.time()) + 3600})
|
||||
entry = store.add({"status": "expected", "scope": "stream", "stream_id": "vpn", "expires_at": int(time.time()) + 3600})
|
||||
self.assertEqual(entry["entity"], "")
|
||||
|
||||
def test_feedback_entity_scope_requires_entity(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = FeedbackStore(str(Path(directory) / "feedback.json"))
|
||||
with self.assertRaises(ValueError):
|
||||
store.add({"status": "expected", "scope": "entity", "expires_at": int(time.time()) + 3600})
|
||||
|
||||
def test_feedback_revoke_removes_single_rule_by_id(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = FeedbackStore(str(Path(directory) / "feedback.json"))
|
||||
one = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username"})
|
||||
store.add({"status": "confirmed", "entity": "bob", "stream_id": "vpn", "field": "username"})
|
||||
|
||||
result = store.revoke(one["id"])
|
||||
self.assertEqual(result, {"removed": 1, "remaining": 1, "id": one["id"]})
|
||||
self.assertEqual(len(store.entries()), 1)
|
||||
|
||||
missing = store.revoke("does-not-exist")
|
||||
self.assertEqual(missing, {"removed": 0, "remaining": 1, "id": "does-not-exist"})
|
||||
|
||||
def test_feedback_matches_respects_scope_and_precedence(self):
|
||||
value_rule = {"scope": "value", "entity": "alice", "stream_id": "windows", "field": "username", "value": "10.0.0.1", "status": "confirmed"}
|
||||
field_rule = {"scope": "field", "entity": "alice", "stream_id": "windows", "field": "username", "status": "expected"}
|
||||
entity_rule = {"scope": "entity", "entity": "alice", "status": "expected"}
|
||||
stream_rule = {"scope": "stream", "stream_id": "windows", "status": "expected"}
|
||||
|
||||
self.assertTrue(feedback_matches(stream_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.1"))
|
||||
self.assertFalse(feedback_matches(stream_rule, entity="alice", stream_id="vpn", field="username", value="10.0.0.1"))
|
||||
self.assertTrue(feedback_matches(entity_rule, entity="alice", stream_id="vpn", field="hostname", value="x"))
|
||||
self.assertFalse(feedback_matches(field_rule, entity="alice", stream_id="windows", field="hostname", value="x"))
|
||||
self.assertTrue(feedback_matches(value_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.1"))
|
||||
self.assertFalse(feedback_matches(value_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.2"))
|
||||
|
||||
candidates = [stream_rule, entity_rule, field_rule, value_rule]
|
||||
self.assertEqual(select_match(candidates), value_rule)
|
||||
self.assertEqual(select_match([stream_rule, entity_rule]), entity_rule)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user