diff --git a/README.md b/README.md
index 7a1a1de..ec6c797 100644
--- a/README.md
+++ b/README.md
@@ -182,6 +182,16 @@ The review buttons are part of the detection loop:
- Expiry should be used for temporary expected changes, such as maintenance or a
migration window.
+Every review click also permanently records which detector produced the
+finding and the verdict, independent of the suppression rule it creates —
+so revoking or letting a rule expire later never erases that history. Check
+`Diagnostics -> Detector Health` to see, per detector: whether it has
+learned enough baseline history to be trusted yet, its confirmed-vs-false-positive
+precision over the last 30 days, and a one-line description of what
+actually triggers it. A detector with no reviews yet shows `n/a` for
+precision rather than a misleading 0% or 100% — the number only becomes
+meaningful once there's real history behind it.
+
## Primary Workflow: Graylog MCP
Graylog 7.1 MCP is the primary log-source integration. In the dashboard, open
diff --git a/ROADMAP.md b/ROADMAP.md
index b56a0fc..cb5d5a1 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -94,7 +94,17 @@ Goal: make findings more accurate before adding more integrations.
- [x] Add a dry-run replay command for historic JSONL or Graylog exports using temporary baselines.
- [x] Add direct Graylog MCP time-range replay and result comparison against saved detector configurations.
- [ ] Add dashboard controls for launching safe replay jobs and viewing detector deltas.
-- [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold.
+- [x] Add baseline confidence tooling: per detector learning state, expected
+ false-positive rate, and why a deviation crossed threshold. The prior
+ "detector precision" metric only existed in the Prometheus endpoint and
+ recomputed from whatever was active in the current poll cycle — a
+ revoked or expired suppression rule made the signal vanish. Replaced with
+ a permanent, append-only review-outcome log (`DetectorPrecisionStore`,
+ separate from the mutable `FeedbackStore`) recorded the moment a human
+ clicks Expected/False positive/Confirmed, plus per-detector learning
+ state (`BaselineStore.detector_readiness()`, mirroring the existing
+ per-field `profile_readiness()`) and a plain-language "what triggers
+ this" reference — all surfaced in a new Detector Health dashboard panel.
- [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status.
- [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings.
- [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping.
diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py
index 624ac94..a8c2049 100644
--- a/src/fgai/baseline.py
+++ b/src/fgai/baseline.py
@@ -668,6 +668,17 @@ class BaselineStore:
rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days})
return rows
+ def detector_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
+ rows: list[dict[str, object]] = []
+ with self._connect() as connection:
+ for stream_id, profile in profiles.items():
+ for detector in getattr(profile, "detectors", {}):
+ count = connection.execute("select count(distinct bucket_start) from profile_detector_buckets where stream_id=? and detector=?", (stream_id, str(detector).lower())).fetchone()[0]
+ oldest = connection.execute("select min(bucket_start) from profile_detector_buckets where stream_id=? and detector=?", (stream_id, str(detector).lower())).fetchone()[0]
+ age_days = _age_days(oldest, int(time.time()))
+ rows.append({"stream_id": stream_id, "detector": str(detector), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days})
+ return rows
+
def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]:
profiles: dict[str, dict[str, object]] = {}
with self._connect() as connection:
diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py
index 1151e42..422314e 100644
--- a/src/fgai/dashboard.py
+++ b/src/fgai/dashboard.py
@@ -10,7 +10,7 @@ from .config import ConfigStore
from .exports import investigation_report, investigation_report_markdown
from .graylog_mcp import GraylogMcpClient
from .metrics import prometheus_metrics
-from .feedback import FeedbackStore
+from .feedback import DetectorPrecisionStore, FeedbackStore
from .incidents import IncidentStore
@@ -896,7 +896,7 @@ async function refresh() {
if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? 'No current field deviations in this poll; showing cached findings from the last non-empty poll.
' : ''}` + table(fieldRows, [
{label:'', render:r => { const key = deviationKey(r); return `${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('${summary} ${events}
Expected False positive Confirm
`}
+ {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Suggested', render:suggestReview}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const clusterNote = r.cluster_size >= 5 ? `[part of a ${r.cluster_size}-entity pattern${r.cluster_dampened ? '; score capped' : ''}] ` : ''; const summary=esc(`${clusterNote}${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('${summary} ${events}
Expected False positive Confirm
`}
], 'field-deviations');
const bulkReviewBar = document.getElementById('bulkReviewBar');
const updateBulkReviewBar = () => {
@@ -930,7 +930,7 @@ async function refresh() {
const author = document.getElementById('bulkReviewAuthor').value || '';
const days = Number(document.getElementById('bulkReviewExpiryDays').value || '0');
const expiresAt = days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0;
- const items = selectedRows.map(row => ({status, scope, entity: row.entity, stream_id: row.stream_id, field: row.field, value: row.value, note, author, expires_at: expiresAt}));
+ const items = selectedRows.map(row => ({status, scope, entity: row.entity, stream_id: row.stream_id, field: row.field, value: row.value, detector: row.detector, note, author, expires_at: expiresAt}));
const response = await fetch('/api/feedback/bulk', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({items})});
const result = await response.json().catch(() => ({}));
if (response.ok) {
@@ -950,7 +950,7 @@ async function refresh() {
reviewScopeWarning.hidden = !broad;
};
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', () => {
- pendingReview = {status: button.dataset.status, entity: button.dataset.entity, stream_id: button.dataset.stream, field: button.dataset.field, value: button.dataset.value};
+ pendingReview = {status: button.dataset.status, entity: button.dataset.entity, stream_id: button.dataset.stream, field: button.dataset.field, value: button.dataset.value, detector: button.dataset.detector};
reviewScopeSelect.value = pendingReview.value ? 'value' : 'field';
document.getElementById('reviewNote').value = '';
document.getElementById('reviewExpiryDays').value = '0';
@@ -980,7 +980,7 @@ async function refresh() {
if (!confirm(`This will suppress ALL findings for ${target} across every field until it expires. Continue?`)) return;
}
lastReviewAuthor = author;
- const payload = {status: pendingReview.status, scope, entity: pendingReview.entity, stream_id: pendingReview.stream_id, field: pendingReview.field, value: pendingReview.value, note, author, expires_at: days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0};
+ const payload = {status: pendingReview.status, scope, entity: pendingReview.entity, stream_id: pendingReview.stream_id, field: pendingReview.field, value: pendingReview.value, detector: pendingReview.detector, note, author, expires_at: days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0};
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload)});
if (response.ok) {
document.getElementById('feedbackNotice').textContent = 'Review saved. The matching pattern will be labeled on the next refresh.';
@@ -1070,6 +1070,7 @@ async function refresh() {
collapsibleSection('correlations', 'Cross-Source Correlations', table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations')) +
collapsibleSection('entities', 'Entities', table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities')) +
collapsibleSection('profile-readiness', 'Profile Baseline Readiness', table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `Remove ` : '-'}], 'profile-readiness')) +
+ collapsibleSection('detector-health', 'Detector Health', table(data.detector_health || [], [{label:'Detector', key:'detector'}, {label:'Description', key:'description'}, {label:'Learning state', render:r => r.learning_state === 'no_data' ? 'no data yet' : r.learning_state === 'ready' ? `ready (${r.streams_ready}/${r.streams_configured} streams)` : `learning (${r.streams_ready}/${r.streams_configured} streams)`}, {label:'Precision (30d)', render:r => r.precision === null || r.precision === undefined ? 'n/a' : `${Math.round(r.precision * 100)}%`}, {label:'Confirmed', key:'confirmed'}, {label:'False positive', key:'false_positive'}, {label:'Expected', key:'expected'}], 'detector-health')) +
collapsibleSection('data-quality', 'Data Quality', table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}], 'data-quality')) +
collapsibleSection('feedback-rules', 'Feedback Rules', table(
['value', 'field', 'entity', 'stream'].map(scope => ({scope, count: feedbackRules.filter(item => (item.scope || 'value') === scope).length})),
@@ -1552,7 +1553,9 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
if self.path == "/api/feedback" and self._is_loopback_client():
try:
payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8"))
- self._send(200, "application/json", json.dumps(FeedbackStore().add(payload)).encode("utf-8"))
+ entry = FeedbackStore().add(payload)
+ DetectorPrecisionStore().record(str(payload.get("detector", "")), str(payload.get("status", "")), stream_id=str(payload.get("stream_id", "")), field=str(payload.get("field", "")))
+ self._send(200, "application/json", json.dumps(entry).encode("utf-8"))
except (ValueError, json.JSONDecodeError) as exc:
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
return
@@ -1560,7 +1563,13 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
try:
payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 65_536)).decode("utf-8"))
items = payload.get("items", []) if isinstance(payload, dict) else []
- self._send(200, "application/json", json.dumps(FeedbackStore().add_many(items if isinstance(items, list) else [])).encode("utf-8"))
+ items = items if isinstance(items, list) else []
+ result = FeedbackStore().add_many(items)
+ precision_store = DetectorPrecisionStore()
+ for item in items:
+ if isinstance(item, dict):
+ precision_store.record(str(item.get("detector", "")), str(item.get("status", "")), stream_id=str(item.get("stream_id", "")), field=str(item.get("field", "")))
+ self._send(200, "application/json", json.dumps(result).encode("utf-8"))
except (ValueError, json.JSONDecodeError) as exc:
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
return
diff --git a/src/fgai/detectors.py b/src/fgai/detectors.py
index 714517a..f0de7d4 100644
--- a/src/fgai/detectors.py
+++ b/src/fgai/detectors.py
@@ -11,6 +11,21 @@ DETECTOR_MINIMUMS = {
"deny_action": 10,
}
+# Display-only reference for the dashboard's Detector Health panel — never
+# consulted by detection/scoring logic itself. Keyed by the name reported on
+# a deviation (the *_burst suffix baseline.py appends when it flags one of
+# these), not the bare profile.detectors config key.
+DETECTOR_DESCRIPTIONS = {
+ "auth_failure_burst": "Authentication failures for an entity above its historical baseline rate.",
+ "dns_query_burst": "DNS query volume for an entity above its historical baseline rate.",
+ "deny_action_burst": "Denied/blocked security events for an entity above its historical baseline rate.",
+ "event_rate_burst": "Overall event volume for an entity above its historical baseline rate.",
+ "numeric_baseline": "A profiled numeric field's value deviates more than 3 standard deviations from its historical baseline.",
+ "new_relationship": "A profiled field-pair combination (e.g. user -> source IP) not seen before for this entity.",
+ "rare_value": "A profiled categorical field value not seen before for this entity.",
+ "dns_network_auth_sequence": "An ordered cross-stream sequence (e.g. DNS lookup -> connection -> auth) matched for this entity.",
+}
+
def event_detector_categories(event: LogEvent) -> tuple[str, ...]:
fields = event.fields
diff --git a/src/fgai/feedback.py b/src/fgai/feedback.py
index 2bcb8bb..94108bb 100644
--- a/src/fgai/feedback.py
+++ b/src/fgai/feedback.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import hashlib
import json
+import sqlite3
import time
from pathlib import Path
@@ -153,6 +154,61 @@ class FeedbackStore:
return {"removed": removed, "remaining": len(entries), "status": status or "all"}
+class DetectorPrecisionStore:
+ """Permanent, append-only log of review outcomes per detector.
+
+ Deliberately separate from FeedbackStore, which holds mutable suppression
+ rules that can be revoked or expire — reusing that as a historical signal
+ would let a revoked rule erase the record of whether a detector's findings
+ tend to be right. Here, once a human reviews a finding, the verdict is kept
+ forever regardless of what later happens to the resulting suppression rule.
+ """
+
+ def __init__(self, path: str = "state/signalscope-detector-precision.sqlite3") -> None:
+ self.path = Path(path)
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ with sqlite3.connect(self.path) as connection:
+ connection.execute(
+ "create table if not exists outcomes ("
+ "id integer primary key autoincrement, detector text not null, status text not null, "
+ "stream_id text not null default '', field text not null default '', recorded_at integer not null)"
+ )
+ connection.execute("create index if not exists idx_outcomes_detector on outcomes(detector, recorded_at)")
+
+ def record(self, detector: str, status: str, *, stream_id: str = "", field: str = "", recorded_at: int | None = None) -> None:
+ detector = str(detector or "").strip()
+ status = str(status or "").lower()
+ if not detector or status not in VALID_STATUSES:
+ return
+ with sqlite3.connect(self.path) as connection:
+ connection.execute(
+ "insert into outcomes (detector, status, stream_id, field, recorded_at) values (?, ?, ?, ?, ?)",
+ (detector, status, stream_id, field, recorded_at if recorded_at is not None else int(time.time())),
+ )
+
+ def precision(self, *, days: int = 30) -> dict[str, dict[str, object]]:
+ cutoff = int(time.time()) - max(1, days) * 86400
+ result: dict[str, dict[str, object]] = {}
+ with sqlite3.connect(self.path) as connection:
+ rows = connection.execute(
+ "select detector, status, count(*) from outcomes where recorded_at >= ? group by detector, status",
+ (cutoff,),
+ ).fetchall()
+ for detector, status, count in rows:
+ counts = result.setdefault(detector, {"confirmed": 0, "false_positive": 0, "expected": 0})
+ counts[status] = count
+ for counts in result.values():
+ denominator = counts["confirmed"] + counts["false_positive"]
+ counts["precision"] = round(counts["confirmed"] / denominator, 4) if denominator else None
+ counts["reviewed_total"] = denominator + counts["expected"]
+ return result
+
+ def maintenance(self, *, retention_days: int = 180) -> int:
+ cutoff = int(time.time()) - max(1, retention_days) * 86400
+ with sqlite3.connect(self.path) as connection:
+ return connection.execute("delete from outcomes where recorded_at < ?", (cutoff,)).rowcount
+
+
def feedback_matches(entry: dict[str, object], *, entity: str, stream_id: str, field: str, value: str) -> bool:
scope = str(entry.get("scope") or _resolve_scope(entry))
if scope == "stream":
diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py
index e25341c..f605a44 100644
--- a/src/fgai/monitor.py
+++ b/src/fgai/monitor.py
@@ -13,7 +13,8 @@ from .baseline import BaselineStore
from .config import ConfigStore
from .correlation import correlate_source_ips
from .event_context import build_event_context
-from .feedback import FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations
+from .detectors import DETECTOR_DESCRIPTIONS
+from .feedback import DetectorPrecisionStore, FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations
from .graylog_aggregate import GraylogAggregateSource
from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource
@@ -286,6 +287,39 @@ def _cluster_field_deviations(field_deviations: dict[str, list[dict[str, object]
deviation["cluster_dampened"] = True
+# baseline.py reports a burst deviation's detector as f"{key}_burst" (see
+# profile_deviations), but profile.detectors/detector_readiness() use the
+# bare config key — this is the one place that reconciles the two names.
+_DETECTOR_READINESS_DISPLAY_NAME = {"auth_failure": "auth_failure_burst", "dns_query": "dns_query_burst", "deny_action": "deny_action_burst"}
+
+
+def _build_detector_health(detector_readiness: list[dict[str, object]], detector_precision: dict[str, dict[str, object]]) -> list[dict[str, object]]:
+ readiness_by_detector: dict[str, list[dict[str, object]]] = defaultdict(list)
+ for row in detector_readiness:
+ display_name = _DETECTOR_READINESS_DISPLAY_NAME.get(str(row.get("detector", "")), str(row.get("detector", "")))
+ readiness_by_detector[display_name].append(row)
+
+ names = sorted({*DETECTOR_DESCRIPTIONS, *readiness_by_detector, *detector_precision})
+ health = []
+ for name in names:
+ streams = readiness_by_detector.get(name, [])
+ ready_streams = sum(1 for item in streams if item.get("ready"))
+ precision = detector_precision.get(name, {})
+ health.append({
+ "detector": name,
+ "description": DETECTOR_DESCRIPTIONS.get(name, ""),
+ "streams_configured": len(streams),
+ "streams_ready": ready_streams,
+ "learning_state": "no_data" if not streams else "ready" if ready_streams == len(streams) else "learning",
+ "confirmed": precision.get("confirmed", 0),
+ "false_positive": precision.get("false_positive", 0),
+ "expected": precision.get("expected", 0),
+ "precision": precision.get("precision"),
+ "reviewed_total": precision.get("reviewed_total", 0),
+ })
+ return health
+
+
def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
@@ -592,6 +626,9 @@ def build_status(
}
for item in profile_readiness
]
+ detector_readiness_rows = baseline.detector_readiness(stream_profiles, min_training_days=baseline_training_days) if baseline else []
+ detector_precision = DetectorPrecisionStore().precision(days=30)
+ detector_health = _build_detector_health(detector_readiness_rows, detector_precision)
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
discovery_cache_events = []
discovered_profile_fields = 0
@@ -734,6 +771,7 @@ def build_status(
"profile_shape_warnings": profile_shape_warnings,
"shared_entity_baselines": shared_entity_baselines,
"profile_readiness": profile_readiness,
+ "detector_health": detector_health,
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),
"top_destination_ips": top_field_values(events, "dstip", limit=10),
diff --git a/tests/test_baseline.py b/tests/test_baseline.py
index aa4c88f..a9f18e8 100644
--- a/tests/test_baseline.py
+++ b/tests/test_baseline.py
@@ -254,3 +254,62 @@ class BaselineTests(unittest.TestCase):
self.assertEqual(relation["field"], "relationship:username->srcip")
self.assertEqual(relation["value"], "10.0.0.99")
self.assertIn("new srcip value for username=peter", relation["reason"])
+
+ def test_detector_readiness_reports_learning_before_twelve_buckets(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+ profiles = parse_profiles([{
+ "stream_id": "windows",
+ "entity_field": "username",
+ "categorical_fields": ["action"],
+ "detectors": {"auth_failure": {"enabled": True}},
+ }])
+ for index in range(5):
+ event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
+ store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
+
+ readiness = store.detector_readiness(profiles)
+
+ row = next(item for item in readiness if item["detector"] == "auth_failure")
+ self.assertEqual(row["stream_id"], "windows")
+ self.assertEqual(row["buckets"], 5)
+ self.assertFalse(row["ready"])
+
+ def test_detector_readiness_reports_ready_after_twelve_buckets(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+ profiles = parse_profiles([{
+ "stream_id": "windows",
+ "entity_field": "username",
+ "categorical_fields": ["action"],
+ "detectors": {"auth_failure": {"enabled": True}},
+ }])
+ for index in range(12):
+ event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
+ store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
+
+ readiness = store.detector_readiness(profiles)
+
+ row = next(item for item in readiness if item["detector"] == "auth_failure")
+ self.assertEqual(row["buckets"], 12)
+ self.assertTrue(row["ready"])
+
+ def test_detector_readiness_respects_min_training_days(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+ profiles = parse_profiles([{
+ "stream_id": "windows",
+ "entity_field": "username",
+ "categorical_fields": ["action"],
+ "detectors": {"auth_failure": {"enabled": True}},
+ }])
+ now = int(time.time())
+ for index in range(12):
+ event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
+ store.ingest_profile_fields([event], profiles, observed_at=now - 86400 + index * 300)
+
+ readiness = store.detector_readiness(profiles, min_training_days=7)
+
+ row = next(item for item in readiness if item["detector"] == "auth_failure")
+ self.assertTrue(row["buckets"] >= 12)
+ self.assertFalse(row["ready"])
diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py
index 7f4e910..2e1eb73 100644
--- a/tests/test_dashboard.py
+++ b/tests/test_dashboard.py
@@ -104,6 +104,17 @@ class DashboardTests(unittest.TestCase):
self.assertIn("Shared Entity Baselines", HTML)
self.assertIn("shared_entity_baselines", HTML)
+ def test_dashboard_shows_detector_health(self):
+ self.assertIn("detector-health", HTML)
+ self.assertIn("Detector Health", HTML)
+ self.assertIn("detector_health", HTML)
+
+ def test_review_actions_carry_detector_for_precision_tracking(self):
+ self.assertIn('data-detector="${esc(r.detector', HTML)
+ self.assertIn("detector: button.dataset.detector", HTML)
+ self.assertIn("detector: pendingReview.detector", HTML)
+ self.assertIn("detector: row.detector", HTML)
+
def test_dashboard_shows_shape_mismatch_warnings(self):
self.assertIn('id="profileShapeWarnings"', HTML)
self.assertIn("profile_shape_warnings", HTML)
diff --git a/tests/test_feedback.py b/tests/test_feedback.py
index 1121354..ca149ad 100644
--- a/tests/test_feedback.py
+++ b/tests/test_feedback.py
@@ -3,7 +3,7 @@ import time
import unittest
from pathlib import Path
-from fgai.feedback import FeedbackStore, feedback_matches, select_match, suggest_generalizations
+from fgai.feedback import DetectorPrecisionStore, FeedbackStore, feedback_matches, select_match, suggest_generalizations
class FeedbackTests(unittest.TestCase):
@@ -157,6 +157,55 @@ class FeedbackTests(unittest.TestCase):
{"stream_id": "s", "entity": "alice", "field": "f", "status": "false_positive", "value_count": 2, "sample_values": ["1", "2"], "entry_ids": ["1", "2"]}
])
+ def test_detector_precision_store_computes_ratio_from_recorded_outcomes(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
+ store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
+ store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
+ store.record("auth_failure_burst", "false_positive", stream_id="windows", field="auth_failure")
+ store.record("new_relationship", "expected", stream_id="firewall", field="relationship:srcip->dstip")
+
+ precision = store.precision(days=30)
+
+ auth = precision["auth_failure_burst"]
+ self.assertEqual(auth["confirmed"], 2)
+ self.assertEqual(auth["false_positive"], 1)
+ self.assertAlmostEqual(auth["precision"], 2 / 3, places=3)
+ self.assertEqual(auth["reviewed_total"], 3)
+ self.assertIsNone(precision["new_relationship"]["precision"])
+ self.assertEqual(precision["new_relationship"]["reviewed_total"], 1)
+
+ def test_detector_precision_store_excludes_outcomes_outside_the_window(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
+ old_timestamp = int(time.time()) - 40 * 86400
+ store.record("auth_failure_burst", "confirmed", recorded_at=old_timestamp)
+ store.record("auth_failure_burst", "false_positive")
+
+ precision = store.precision(days=30)
+
+ self.assertEqual(precision["auth_failure_burst"]["confirmed"], 0)
+ self.assertEqual(precision["auth_failure_burst"]["false_positive"], 1)
+
+ def test_detector_precision_store_ignores_invalid_or_empty_input(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
+ store.record("", "confirmed")
+ store.record("auth_failure_burst", "not_a_real_status")
+
+ self.assertEqual(store.precision(days=30), {})
+
+ def test_detector_precision_store_maintenance_prunes_old_rows(self):
+ with tempfile.TemporaryDirectory() as directory:
+ store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
+ store.record("auth_failure_burst", "confirmed", recorded_at=int(time.time()) - 200 * 86400)
+ store.record("auth_failure_burst", "confirmed")
+
+ removed = store.maintenance(retention_days=180)
+
+ self.assertEqual(removed, 1)
+ self.assertEqual(store.precision(days=3650)["auth_failure_burst"]["confirmed"], 1)
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_monitor.py b/tests/test_monitor.py
index 403c099..e4192d0 100644
--- a/tests/test_monitor.py
+++ b/tests/test_monitor.py
@@ -258,6 +258,43 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(pool["group"], "srcip")
self.assertEqual(set(pool["streams"]), {"firewall", "iis"})
+ def test_detector_health_lists_known_detectors_with_no_data_by_default(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
+ status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json"))
+
+ health_by_detector = {item["detector"]: item for item in status["detector_health"]}
+ self.assertIn("auth_failure_burst", health_by_detector)
+ self.assertIn("new_relationship", health_by_detector)
+ self.assertEqual(health_by_detector["auth_failure_burst"]["learning_state"], "no_data")
+ self.assertIsNone(health_by_detector["auth_failure_burst"]["precision"])
+
+ def test_detector_health_reflects_readiness_and_precision(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ config_path = Path(tmp) / "config.json"
+ config_path.write_text(
+ json.dumps(
+ {
+ "graylog_streams": [{"id": "windows", "title": "Windows", "enabled": True}],
+ "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "categorical_fields": ["action"], "detectors": {"auth_failure": {"enabled": True}}}],
+ }
+ ),
+ encoding="utf-8",
+ )
+ readiness_rows = [{"stream_id": "windows", "detector": "auth_failure", "buckets": 12, "age_days": 10, "training_days": 0, "ready": True}]
+ precision_rows = {"auth_failure_burst": {"confirmed": 3, "false_positive": 1, "expected": 0, "precision": 0.75, "reviewed_total": 4}}
+ with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]), \
+ patch("fgai.monitor.BaselineStore.detector_readiness", return_value=readiness_rows), \
+ patch("fgai.monitor.DetectorPrecisionStore.precision", return_value=precision_rows):
+ status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+ row = next(item for item in status["detector_health"] if item["detector"] == "auth_failure_burst")
+ self.assertEqual(row["learning_state"], "ready")
+ self.assertEqual(row["streams_ready"], 1)
+ self.assertEqual(row["streams_configured"], 1)
+ self.assertEqual(row["confirmed"], 3)
+ self.assertEqual(row["precision"], 0.75)
+
def test_shared_entity_baselines_empty_for_single_stream(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"