add feedback scopes, revoke and precision metrics

Feedback rules now support value/field/entity/stream scope, require
expiry for broad suppressions, and can be revoked individually.
Adds an active-rules panel, detector precision metrics, and diagnostics.

ROADMAP.md

@@ -58,11 +58,11 @@ Acceptance: an analyst can open an incident, see an ordered multi-stream timelin
 
 Goal: reduce repeat noise without hiding genuinely new behavior.
 
-- [ ] Show active feedback rules, expiry, scope, author, and matching findings in the UI.
-- [ ] Support feedback scope choices: exact value, entity and field, entire entity, or stream-wide rule.
-- [ ] Require an expiry for broad suppressions and warn before a broad rule is saved.
-- [ ] Track detector precision using confirmed versus false-positive feedback.
-- [ ] Include feedback-derived precision and suppression counts in metrics and diagnostics.
+- [x] Show active feedback rules, expiry, scope, author, and matching findings in the UI.
+- [x] Support feedback scope choices: exact value, entity and field, entire entity, or stream-wide rule.
+- [x] Require an expiry for broad suppressions and warn before a broad rule is saved.
+- [x] Track detector precision using confirmed versus false-positive feedback.
+- [x] Include feedback-derived precision and suppression counts in metrics and diagnostics.
 
 Acceptance: a false-positive decision suppresses only the intended pattern and all suppressions are auditable and reversible.
 

src/fgai/anomaly.py

@@ -5,6 +5,7 @@ from collections import Counter, defaultdict
 from datetime import datetime
 from statistics import mean, pstdev
 
+from .feedback import SUPPRESSING_STATUSES
 from .logs import THREAT_ACTIONS, event_score, is_utm_event
 from .models import AnomalyFinding, LogEvent
 
@@ -220,7 +221,8 @@ def detect_source_anomalies(
 
         for deviation in field_deviations.get(src_ip, []):
             score += int(deviation.get("score", 0))
-            reasons.append(str(deviation.get("reason", "stream field baseline deviation")))
+            if deviation.get("feedback") not in SUPPRESSING_STATUSES:
+                reasons.append(str(deviation.get("reason", "stream field baseline deviation")))
 
         if not reasons:
             continue

src/fgai/feedback.py

@@ -1,9 +1,77 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import time
 from pathlib import Path
 
+VALID_STATUSES = {"false_positive", "expected", "confirmed"}
+VALID_SCOPES = {"value", "field", "entity", "stream"}
+SCOPE_PRIORITY = {"value": 0, "field": 1, "entity": 2, "stream": 3}
+SUPPRESSING_STATUSES = {"false_positive", "expected"}
+
+_SCOPE_ID_FIELDS = {
+    "value": ("stream_id", "entity", "field", "value"),
+    "field": ("stream_id", "entity", "field"),
+    "entity": ("entity",),
+    "stream": ("stream_id",),
+}
+
+
+def _resolve_scope(item: dict[str, object]) -> str:
+    scope = str(item.get("scope") or "").lower()
+    if scope:
+        return scope
+    return "value" if str(item.get("value", "")) else "field"
+
+
+def _entry_id(scope: str, entry: dict[str, object]) -> str:
+    fields = _SCOPE_ID_FIELDS[scope]
+    payload = {"scope": scope, **{key: entry.get(key, "") for key in fields}}
+    return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()[:16]
+
+
+def _normalize(item: dict[str, object]) -> dict[str, object]:
+    scope = _resolve_scope(item)
+    if scope not in VALID_SCOPES:
+        raise ValueError("invalid feedback scope")
+    status = str(item.get("status", "")).lower()
+    if status not in VALID_STATUSES:
+        raise ValueError("invalid feedback status")
+    stream_id = str(item.get("stream_id", ""))
+    entity = str(item.get("entity", ""))
+    field = str(item.get("field", ""))
+    value = str(item.get("value", ""))
+    if scope == "stream" and not stream_id:
+        raise ValueError("stream-scoped feedback requires a stream_id")
+    if scope == "entity" and not entity:
+        raise ValueError("entity-scoped feedback requires an entity")
+    if scope in {"value", "field"} and not (entity and stream_id and field):
+        raise ValueError("field-scoped feedback requires entity, stream_id, and field")
+    expires_at = int(item.get("expires_at", 0) or 0)
+    if scope in {"entity", "stream"} and expires_at <= 0:
+        raise ValueError("broad-scope feedback rules require an expiry")
+    if scope == "stream":
+        entity = field = value = ""
+    elif scope == "entity":
+        stream_id = field = value = ""
+    elif scope == "field":
+        value = ""
+    entry = {
+        "stream_id": stream_id,
+        "entity": entity,
+        "field": field,
+        "value": value,
+        "scope": scope,
+        "status": status,
+        "note": str(item.get("note", "")),
+        "author": str(item.get("author", "")),
+        "created_at": int(time.time()),
+        "expires_at": expires_at,
+    }
+    entry["id"] = _entry_id(scope, entry)
+    return entry
+
 
 class FeedbackStore:
     def __init__(self, path: str = "state/signalscope-feedback.json") -> None:
@@ -15,17 +83,37 @@ class FeedbackStore:
         except (FileNotFoundError, json.JSONDecodeError):
             items = []
         now = int(time.time())
-        return [item for item in items if not item.get("expires_at") or int(item["expires_at"]) > now]
+        result = []
+        for item in items:
+            if not isinstance(item, dict):
+                continue
+            if item.get("expires_at") and int(item["expires_at"]) <= now:
+                continue
+            scope = _resolve_scope(item)
+            item = dict(item)
+            item["scope"] = scope
+            item.setdefault("author", "")
+            item.setdefault("id", _entry_id(scope, item))
+            result.append(item)
+        return result
 
     def add(self, item: dict[str, object]) -> dict[str, object]:
-        status = str(item.get("status", "")).lower()
-        if status not in {"false_positive", "expected", "confirmed"}:
-            raise ValueError("invalid feedback status")
-        entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "value": str(item.get("value", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)}
-        entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field"), value.get("value", "")) != (entry["stream_id"], entry["entity"], entry["field"], entry["value"])]
-        entries.append(entry); self.path.parent.mkdir(parents=True, exist_ok=True); self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8")
+        entry = _normalize(item)
+        entries = [value for value in self.entries() if value.get("id") != entry["id"]]
+        entries.append(entry)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8")
         return entry
 
+    def revoke(self, entry_id: str) -> dict[str, object]:
+        entries = self.entries()
+        before = len(entries)
+        entries = [item for item in entries if item.get("id") != entry_id]
+        removed = before - len(entries)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8")
+        return {"removed": removed, "remaining": len(entries), "id": entry_id}
+
     def clear(self, *, status: str | None = None) -> dict[str, object]:
         entries = self.entries()
         if status is None:
@@ -33,7 +121,7 @@ class FeedbackStore:
             entries = []
         else:
             status = status.lower()
-            if status not in {"false_positive", "expected", "confirmed"}:
+            if status not in VALID_STATUSES:
                 raise ValueError("invalid feedback status")
             before = len(entries)
             entries = [item for item in entries if str(item.get("status", "")).lower() != status]
@@ -41,3 +129,27 @@ class FeedbackStore:
         self.path.parent.mkdir(parents=True, exist_ok=True)
         self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8")
         return {"removed": removed, "remaining": len(entries), "status": status or "all"}
+
+
+def feedback_matches(entry: dict[str, object], *, entity: str, stream_id: str, field: str, value: str) -> bool:
+    scope = str(entry.get("scope") or _resolve_scope(entry))
+    if scope == "stream":
+        return str(entry.get("stream_id", "")) == stream_id
+    if scope == "entity":
+        return str(entry.get("entity", "")) == entity
+    if scope == "field":
+        return (
+            str(entry.get("entity", "")) == entity
+            and str(entry.get("stream_id", "")) == stream_id
+            and str(entry.get("field", "")) == field
+        )
+    return (
+        str(entry.get("entity", "")) == entity
+        and str(entry.get("stream_id", "")) == stream_id
+        and str(entry.get("field", "")) == field
+        and str(entry.get("value", "")) == value
+    )
+
+
+def select_match(candidates: list[dict[str, object]]) -> dict[str, object]:
+    return min(candidates, key=lambda item: SCOPE_PRIORITY.get(str(item.get("scope") or _resolve_scope(item)), 1))

src/fgai/incidents.py

@@ -7,6 +7,7 @@ from collections import defaultdict
 from pathlib import Path
 
 from .entities import entity_type
+from .feedback import SUPPRESSING_STATUSES
 from .models import AnomalyFinding
 
 
@@ -29,7 +30,7 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str,
     for entity, deviations in field_deviations.items():
         group = groups[entity]
         group["fields"].extend(deviations)
-        active = [item for item in deviations if item.get("feedback") not in {"expected", "false_positive"}]
+        active = [item for item in deviations if item.get("feedback") not in SUPPRESSING_STATUSES]
         group["score"] = min(100, int(group["score"]) + min(30, sum(int(item.get("score", 0)) for item in active)))
         group["evidence"].extend(str(item.get("reason", "")) for item in active)
         for item in active:

src/fgai/metrics.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from .feedback import SUPPRESSING_STATUSES, VALID_SCOPES
+
 
 def prometheus_metrics(status: dict[str, object]) -> str:
     summary = status.get("summary", {}) if isinstance(status.get("summary"), dict) else {}
@@ -8,6 +10,9 @@ def prometheus_metrics(status: dict[str, object]) -> str:
     capabilities = status.get("capabilities", {}) if isinstance(status.get("capabilities"), dict) else {}
     mcp = capabilities.get("graylog_mcp", {}) if isinstance(capabilities.get("graylog_mcp"), dict) else {}
     incidents = status.get("incidents", []) if isinstance(status.get("incidents"), list) else []
+    feedback = status.get("feedback", []) if isinstance(status.get("feedback"), list) else []
+    deviations = status.get("field_deviations", {}) if isinstance(status.get("field_deviations"), dict) else {}
+    all_deviations = [item for items in deviations.values() if isinstance(items, list) for item in items if isinstance(item, dict)]
     lines = ["# HELP fgai_events_total Events in the latest analysis window.", "# TYPE fgai_events_total gauge"]
     for key in ("total", "utm", "threat_actions", "critical_or_high"):
         lines.append(f'fgai_events_total{{kind="{key}"}} {int(summary.get(key, 0) or 0)}')
@@ -20,4 +25,26 @@ def prometheus_metrics(status: dict[str, object]) -> str:
     lines += ["# HELP signalscope_incidents Open incident candidates by severity.", "# TYPE signalscope_incidents gauge"]
     for severity in ("critical", "high", "medium", "low"):
         lines.append(f'signalscope_incidents{{severity="{severity}"}} {sum(1 for item in incidents if isinstance(item, dict) and item.get("severity") == severity)}')
+    lines += ["# HELP fgai_feedback_rules Active feedback rules by scope.", "# TYPE fgai_feedback_rules gauge"]
+    for scope in sorted(VALID_SCOPES):
+        lines.append(f'fgai_feedback_rules{{scope="{scope}"}} {sum(1 for item in feedback if isinstance(item, dict) and item.get("scope", "value") == scope)}')
+    suppressed = sum(1 for item in all_deviations if item.get("feedback") in SUPPRESSING_STATUSES)
+    lines += ["# HELP fgai_feedback_suppressed_findings Findings suppressed by feedback in the latest poll.", "# TYPE fgai_feedback_suppressed_findings gauge", f'fgai_feedback_suppressed_findings {suppressed}']
+    precision: dict[str, dict[str, int]] = {}
+    for item in all_deviations:
+        verdict = item.get("feedback")
+        if verdict not in {"confirmed", "false_positive"}:
+            continue
+        detector = str(item.get("detector", "unknown"))
+        counts = precision.setdefault(detector, {"confirmed": 0, "false_positive": 0})
+        counts[verdict] += 1
+    lines += ["# HELP fgai_detector_feedback_total Confirmed vs false-positive feedback by detector.", "# TYPE fgai_detector_feedback_total gauge"]
+    for detector, counts in sorted(precision.items()):
+        for verdict in ("confirmed", "false_positive"):
+            lines.append(f'fgai_detector_feedback_total{{detector="{detector}",verdict="{verdict}"}} {counts[verdict]}')
+    lines += ["# HELP fgai_detector_precision Confirmed / (confirmed + false_positive) by detector.", "# TYPE fgai_detector_precision gauge"]
+    for detector, counts in sorted(precision.items()):
+        denominator = counts["confirmed"] + counts["false_positive"]
+        if denominator:
+            lines.append(f'fgai_detector_precision{{detector="{detector}"}} {counts["confirmed"] / denominator:.4f}')
     return "\n".join(lines) + "\n"

src/fgai/monitor.py

@@ -12,7 +12,7 @@ from .baseline import BaselineStore
 from .config import ConfigStore
 from .correlation import correlate_source_ips
 from .event_context import build_event_context
-from .feedback import FeedbackStore
+from .feedback import FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match
 from .graylog_aggregate import GraylogAggregateSource
 from .graylog_mcp import GraylogMcpClient
 from .graylog_source import GraylogStreamSource
@@ -383,16 +383,18 @@ def build_status(
     feedback = FeedbackStore().entries()
     for entity, deviations in field_deviations.items():
         for deviation in deviations:
-            match = next((
+            stream_id = str(deviation.get("stream_id", ""))
+            field = str(deviation.get("field", ""))
+            value = str(deviation.get("value", ""))
+            candidates = [
                 item for item in feedback
-                if item.get("entity") == entity
-                and item.get("stream_id") == deviation.get("stream_id")
-                and item.get("field") == deviation.get("field")
-                and (not item.get("value") or item.get("value") == deviation.get("value", ""))
-            ), None)
-            if match:
+                if feedback_matches(item, entity=entity, stream_id=stream_id, field=field, value=value)
+            ]
+            if candidates:
+                match = select_match(candidates)
                 deviation["feedback"] = match["status"]
-                if match["status"] in {"false_positive", "expected"}:
+                deviation["feedback_rule_id"] = match.get("id", "")
+                if match["status"] in SUPPRESSING_STATUSES:
                     deviation["score"] = 0
     anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
     baseline_events = baseline.ingest(events) if baseline else 0

src/fgai/triage.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 from collections import Counter, defaultdict
 
+from .feedback import SUPPRESSING_STATUSES
+
 
 def _severity(score: int) -> str:
     return "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
@@ -56,7 +58,7 @@ def build_triage_queue(
             group["deviations"] = int(group["deviations"]) + 1
             group["streams"].add(str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")))
             group["detectors"][str(item.get("detector", "unknown"))] += 1
-            if status in {"expected", "false_positive"}:
+            if status in SUPPRESSING_STATUSES:
                 continue
             score = int(item.get("score", 0) or 0)
             active_scores.append(score)

tests/test_dashboard.py

@@ -76,6 +76,22 @@ class DashboardTests(unittest.TestCase):
         self.assertIn("/api/feedback/clear", source)
         self.assertIn("/api/review-state/clear", source)
 
+    def test_dashboard_exposes_feedback_revoke_endpoint(self):
+        source = inspect.getsource(dashboard)
+        self.assertIn("/api/feedback/revoke", source)
+        self.assertIn(".revoke(", source)
+
+    def test_dashboard_has_active_feedback_rules_panel(self):
+        self.assertIn('id="feedbackRules"', HTML)
+        self.assertIn("revoke-feedback-rule", HTML)
+        self.assertIn('id="feedbackRuleSummary"', HTML)
+
+    def test_dashboard_has_scoped_review_form(self):
+        self.assertIn('id="reviewScope"', HTML)
+        self.assertIn('value="entity"', HTML)
+        self.assertIn('value="stream"', HTML)
+        self.assertIn("Broad-scope rules must expire", HTML)
+
     def test_dashboard_does_not_clear_streams_when_picker_is_unloaded(self):
         self.assertIn("const streamValues = Object.values(window.streamSelection || {})", HTML)
         self.assertIn("if (streamValues.length)", HTML)

tests/test_feedback.py

@@ -1,8 +1,9 @@
 import tempfile
+import time
 import unittest
 from pathlib import Path
 
-from fgai.feedback import FeedbackStore
+from fgai.feedback import FeedbackStore, feedback_matches, select_match
 
 
 class FeedbackTests(unittest.TestCase):
@@ -20,6 +21,74 @@ class FeedbackTests(unittest.TestCase):
             self.assertEqual(all_items, {"removed": 1, "remaining": 0, "status": "all"})
             self.assertEqual(store.entries(), [])
 
+    def test_feedback_scope_defaults_and_ids_are_stable(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = FeedbackStore(str(Path(directory) / "feedback.json"))
+            no_value = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username"})
+            self.assertEqual(no_value["scope"], "field")
+
+            with_value = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username", "value": "bob"})
+            self.assertEqual(with_value["scope"], "value")
+            self.assertEqual(len(store.entries()), 2)
+
+            again = store.add({"status": "confirmed", "entity": "alice", "stream_id": "windows", "field": "username"})
+            self.assertEqual(again["id"], no_value["id"])
+            self.assertEqual(len(store.entries()), 2)
+            self.assertEqual(next(item for item in store.entries() if item["id"] == no_value["id"]).get("status"), "confirmed")
+
+    def test_feedback_broad_scope_requires_expiry(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = FeedbackStore(str(Path(directory) / "feedback.json"))
+            with self.assertRaises(ValueError):
+                store.add({"status": "expected", "scope": "entity", "entity": "alice"})
+            entry = store.add({"status": "expected", "scope": "entity", "entity": "alice", "expires_at": int(time.time()) + 3600})
+            self.assertEqual(entry["scope"], "entity")
+            self.assertEqual(entry["stream_id"], "")
+
+    def test_feedback_stream_scope_requires_stream_id(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = FeedbackStore(str(Path(directory) / "feedback.json"))
+            with self.assertRaises(ValueError):
+                store.add({"status": "expected", "scope": "stream", "expires_at": int(time.time()) + 3600})
+            entry = store.add({"status": "expected", "scope": "stream", "stream_id": "vpn", "expires_at": int(time.time()) + 3600})
+            self.assertEqual(entry["entity"], "")
+
+    def test_feedback_entity_scope_requires_entity(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = FeedbackStore(str(Path(directory) / "feedback.json"))
+            with self.assertRaises(ValueError):
+                store.add({"status": "expected", "scope": "entity", "expires_at": int(time.time()) + 3600})
+
+    def test_feedback_revoke_removes_single_rule_by_id(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = FeedbackStore(str(Path(directory) / "feedback.json"))
+            one = store.add({"status": "expected", "entity": "alice", "stream_id": "windows", "field": "username"})
+            store.add({"status": "confirmed", "entity": "bob", "stream_id": "vpn", "field": "username"})
+
+            result = store.revoke(one["id"])
+            self.assertEqual(result, {"removed": 1, "remaining": 1, "id": one["id"]})
+            self.assertEqual(len(store.entries()), 1)
+
+            missing = store.revoke("does-not-exist")
+            self.assertEqual(missing, {"removed": 0, "remaining": 1, "id": "does-not-exist"})
+
+    def test_feedback_matches_respects_scope_and_precedence(self):
+        value_rule = {"scope": "value", "entity": "alice", "stream_id": "windows", "field": "username", "value": "10.0.0.1", "status": "confirmed"}
+        field_rule = {"scope": "field", "entity": "alice", "stream_id": "windows", "field": "username", "status": "expected"}
+        entity_rule = {"scope": "entity", "entity": "alice", "status": "expected"}
+        stream_rule = {"scope": "stream", "stream_id": "windows", "status": "expected"}
+
+        self.assertTrue(feedback_matches(stream_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.1"))
+        self.assertFalse(feedback_matches(stream_rule, entity="alice", stream_id="vpn", field="username", value="10.0.0.1"))
+        self.assertTrue(feedback_matches(entity_rule, entity="alice", stream_id="vpn", field="hostname", value="x"))
+        self.assertFalse(feedback_matches(field_rule, entity="alice", stream_id="windows", field="hostname", value="x"))
+        self.assertTrue(feedback_matches(value_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.1"))
+        self.assertFalse(feedback_matches(value_rule, entity="alice", stream_id="windows", field="username", value="10.0.0.2"))
+
+        candidates = [stream_rule, entity_rule, field_rule, value_rule]
+        self.assertEqual(select_match(candidates), value_rule)
+        self.assertEqual(select_match([stream_rule, entity_rule]), entity_rule)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_metrics.py

@@ -8,3 +8,45 @@ class MetricsTests(unittest.TestCase):
         output = prometheus_metrics({"summary": {"total": 10}, "anomaly_summary": {"high": 2}, "baseline": {"sources_ready": 3}, "capabilities": {"graylog_mcp": {"status": "connected", "events_fetched": 9}}})
         self.assertIn('fgai_events_total{kind="total"} 10', output)
         self.assertIn("fgai_graylog_mcp_connected 1", output)
+
+    def test_feedback_rules_counted_by_scope(self):
+        status = {
+            "feedback": [
+                {"scope": "entity", "status": "expected"},
+                {"scope": "value", "status": "confirmed"},
+            ],
+            "field_deviations": {},
+        }
+        text = prometheus_metrics(status)
+        self.assertIn('fgai_feedback_rules{scope="entity"} 1', text)
+        self.assertIn('fgai_feedback_rules{scope="value"} 1', text)
+        self.assertIn('fgai_feedback_rules{scope="field"} 0', text)
+        self.assertIn('fgai_feedback_rules{scope="stream"} 0', text)
+
+    def test_suppressed_findings_and_detector_precision(self):
+        status = {
+            "feedback": [],
+            "field_deviations": {
+                "alice": [
+                    {"detector": "rare_value", "feedback": "confirmed"},
+                    {"detector": "rare_value", "feedback": "false_positive"},
+                    {"detector": "rare_value", "feedback": "expected"},
+                    {"detector": "auth_failure_burst", "feedback": "unreviewed"},
+                ]
+            },
+        }
+        text = prometheus_metrics(status)
+        self.assertIn("fgai_feedback_suppressed_findings 2", text)
+        self.assertIn('fgai_detector_feedback_total{detector="rare_value",verdict="confirmed"} 1', text)
+        self.assertIn('fgai_detector_feedback_total{detector="rare_value",verdict="false_positive"} 1', text)
+        self.assertIn('fgai_detector_precision{detector="rare_value"} 0.5000', text)
+        self.assertNotIn('detector="auth_failure_burst"', text)
+
+    def test_metrics_handle_empty_status(self):
+        text = prometheus_metrics({})
+        self.assertIn('fgai_feedback_rules{scope="value"} 0', text)
+        self.assertIn("fgai_feedback_suppressed_findings 0", text)
+
+
+if __name__ == "__main__":
+    unittest.main()