From 36516faf9a778f0672f65ba8848091f7680ced9afdac2caf8897c2bf3c5d05c7 Mon Sep 17 00:00:00 2001 From: larssand Date: Thu, 24 Sep 2026 16:22:02 +0200 Subject: [PATCH] Add real detector health tooling: learning state and historical precision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the old Prometheus-only "detector precision" metric, which recomputed from whatever field_deviations happened to be active in the current poll cycle cross-referenced against live FeedbackStore rules — a revoked or expired suppression rule made the signal vanish, and nothing was tracked between polls. Adds DetectorPrecisionStore, a permanent append-only log of review outcomes (kept deliberately separate from FeedbackStore's mutable suppression rules) recorded the moment a human reviews a finding, plus BaselineStore.detector_readiness() mirroring the existing per-field profile_readiness() for per-detector learning state, and a plain-language detector reference. Surfaced in a new Detector Health dashboard panel. Observability only — nothing here changes scoring or suppression. Co-Authored-By: Claude Sonnet 5 --- README.md | 10 +++++++ ROADMAP.md | 12 ++++++++- src/fgai/baseline.py | 11 ++++++++ src/fgai/dashboard.py | 23 +++++++++++----- src/fgai/detectors.py | 15 +++++++++++ src/fgai/feedback.py | 56 ++++++++++++++++++++++++++++++++++++++ src/fgai/monitor.py | 40 +++++++++++++++++++++++++++- tests/test_baseline.py | 59 +++++++++++++++++++++++++++++++++++++++++ tests/test_dashboard.py | 11 ++++++++ tests/test_feedback.py | 51 ++++++++++++++++++++++++++++++++++- tests/test_monitor.py | 37 ++++++++++++++++++++++++++ 11 files changed, 315 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7a1a1de..ec6c797 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,16 @@ The review buttons are part of the detection loop: - Expiry should be used for temporary expected changes, such as maintenance or a migration window. +Every review click also permanently records which detector produced the +finding and the verdict, independent of the suppression rule it creates — +so revoking or letting a rule expire later never erases that history. Check +`Diagnostics -> Detector Health` to see, per detector: whether it has +learned enough baseline history to be trusted yet, its confirmed-vs-false-positive +precision over the last 30 days, and a one-line description of what +actually triggers it. A detector with no reviews yet shows `n/a` for +precision rather than a misleading 0% or 100% — the number only becomes +meaningful once there's real history behind it. + ## Primary Workflow: Graylog MCP Graylog 7.1 MCP is the primary log-source integration. In the dashboard, open diff --git a/ROADMAP.md b/ROADMAP.md index b56a0fc..cb5d5a1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -94,7 +94,17 @@ Goal: make findings more accurate before adding more integrations. - [x] Add a dry-run replay command for historic JSONL or Graylog exports using temporary baselines. - [x] Add direct Graylog MCP time-range replay and result comparison against saved detector configurations. - [ ] Add dashboard controls for launching safe replay jobs and viewing detector deltas. -- [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold. +- [x] Add baseline confidence tooling: per detector learning state, expected + false-positive rate, and why a deviation crossed threshold. The prior + "detector precision" metric only existed in the Prometheus endpoint and + recomputed from whatever was active in the current poll cycle — a + revoked or expired suppression rule made the signal vanish. Replaced with + a permanent, append-only review-outcome log (`DetectorPrecisionStore`, + separate from the mutable `FeedbackStore`) recorded the moment a human + clicks Expected/False positive/Confirmed, plus per-detector learning + state (`BaselineStore.detector_readiness()`, mirroring the existing + per-field `profile_readiness()`) and a plain-language "what triggers + this" reference — all surfaced in a new Detector Health dashboard panel. - [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status. - [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings. - [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping. diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 624ac94..a8c2049 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -668,6 +668,17 @@ class BaselineStore: rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days}) return rows + def detector_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + with self._connect() as connection: + for stream_id, profile in profiles.items(): + for detector in getattr(profile, "detectors", {}): + count = connection.execute("select count(distinct bucket_start) from profile_detector_buckets where stream_id=? and detector=?", (stream_id, str(detector).lower())).fetchone()[0] + oldest = connection.execute("select min(bucket_start) from profile_detector_buckets where stream_id=? and detector=?", (stream_id, str(detector).lower())).fetchone()[0] + age_days = _age_days(oldest, int(time.time())) + rows.append({"stream_id": stream_id, "detector": str(detector), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days}) + return rows + def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]: profiles: dict[str, dict[str, object]] = {} with self._connect() as connection: diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 1151e42..422314e 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -10,7 +10,7 @@ from .config import ConfigStore from .exports import investigation_report, investigation_report_markdown from .graylog_mcp import GraylogMcpClient from .metrics import prometheus_metrics -from .feedback import FeedbackStore +from .feedback import DetectorPrecisionStore, FeedbackStore from .incidents import IncidentStore @@ -896,7 +896,7 @@ async function refresh() { if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows; document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '

No current field deviations in this poll; showing cached findings from the last non-empty poll.

' : ''}` + table(fieldRows, [ {label:'', render:r => { const key = deviationKey(r); return ``; }}, - {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Suggested', render:suggestReview}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const clusterNote = r.cluster_size >= 5 ? `[part of a ${r.cluster_size}-entity pattern${r.cluster_dampened ? '; score capped' : ''}] ` : ''; const summary=esc(`${clusterNote}${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `
${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('
'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} + {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Suggested', render:suggestReview}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const clusterNote = r.cluster_size >= 5 ? `[part of a ${r.cluster_size}-entity pattern${r.cluster_dampened ? '; score capped' : ''}] ` : ''; const summary=esc(`${clusterNote}${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `
${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('
'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} ], 'field-deviations'); const bulkReviewBar = document.getElementById('bulkReviewBar'); const updateBulkReviewBar = () => { @@ -930,7 +930,7 @@ async function refresh() { const author = document.getElementById('bulkReviewAuthor').value || ''; const days = Number(document.getElementById('bulkReviewExpiryDays').value || '0'); const expiresAt = days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0; - const items = selectedRows.map(row => ({status, scope, entity: row.entity, stream_id: row.stream_id, field: row.field, value: row.value, note, author, expires_at: expiresAt})); + const items = selectedRows.map(row => ({status, scope, entity: row.entity, stream_id: row.stream_id, field: row.field, value: row.value, detector: row.detector, note, author, expires_at: expiresAt})); const response = await fetch('/api/feedback/bulk', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({items})}); const result = await response.json().catch(() => ({})); if (response.ok) { @@ -950,7 +950,7 @@ async function refresh() { reviewScopeWarning.hidden = !broad; }; document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', () => { - pendingReview = {status: button.dataset.status, entity: button.dataset.entity, stream_id: button.dataset.stream, field: button.dataset.field, value: button.dataset.value}; + pendingReview = {status: button.dataset.status, entity: button.dataset.entity, stream_id: button.dataset.stream, field: button.dataset.field, value: button.dataset.value, detector: button.dataset.detector}; reviewScopeSelect.value = pendingReview.value ? 'value' : 'field'; document.getElementById('reviewNote').value = ''; document.getElementById('reviewExpiryDays').value = '0'; @@ -980,7 +980,7 @@ async function refresh() { if (!confirm(`This will suppress ALL findings for ${target} across every field until it expires. Continue?`)) return; } lastReviewAuthor = author; - const payload = {status: pendingReview.status, scope, entity: pendingReview.entity, stream_id: pendingReview.stream_id, field: pendingReview.field, value: pendingReview.value, note, author, expires_at: days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0}; + const payload = {status: pendingReview.status, scope, entity: pendingReview.entity, stream_id: pendingReview.stream_id, field: pendingReview.field, value: pendingReview.value, detector: pendingReview.detector, note, author, expires_at: days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0}; const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload)}); if (response.ok) { document.getElementById('feedbackNotice').textContent = 'Review saved. The matching pattern will be labeled on the next refresh.'; @@ -1070,6 +1070,7 @@ async function refresh() { collapsibleSection('correlations', 'Cross-Source Correlations', table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations')) + collapsibleSection('entities', 'Entities', table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities')) + collapsibleSection('profile-readiness', 'Profile Baseline Readiness', table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `` : '-'}], 'profile-readiness')) + + collapsibleSection('detector-health', 'Detector Health', table(data.detector_health || [], [{label:'Detector', key:'detector'}, {label:'Description', key:'description'}, {label:'Learning state', render:r => r.learning_state === 'no_data' ? 'no data yet' : r.learning_state === 'ready' ? `ready (${r.streams_ready}/${r.streams_configured} streams)` : `learning (${r.streams_ready}/${r.streams_configured} streams)`}, {label:'Precision (30d)', render:r => r.precision === null || r.precision === undefined ? 'n/a' : `${Math.round(r.precision * 100)}%`}, {label:'Confirmed', key:'confirmed'}, {label:'False positive', key:'false_positive'}, {label:'Expected', key:'expected'}], 'detector-health')) + collapsibleSection('data-quality', 'Data Quality', table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}], 'data-quality')) + collapsibleSection('feedback-rules', 'Feedback Rules', table( ['value', 'field', 'entity', 'stream'].map(scope => ({scope, count: feedbackRules.filter(item => (item.scope || 'value') === scope).length})), @@ -1552,7 +1553,9 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | if self.path == "/api/feedback" and self._is_loopback_client(): try: payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8")) - self._send(200, "application/json", json.dumps(FeedbackStore().add(payload)).encode("utf-8")) + entry = FeedbackStore().add(payload) + DetectorPrecisionStore().record(str(payload.get("detector", "")), str(payload.get("status", "")), stream_id=str(payload.get("stream_id", "")), field=str(payload.get("field", ""))) + self._send(200, "application/json", json.dumps(entry).encode("utf-8")) except (ValueError, json.JSONDecodeError) as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return @@ -1560,7 +1563,13 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | try: payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 65_536)).decode("utf-8")) items = payload.get("items", []) if isinstance(payload, dict) else [] - self._send(200, "application/json", json.dumps(FeedbackStore().add_many(items if isinstance(items, list) else [])).encode("utf-8")) + items = items if isinstance(items, list) else [] + result = FeedbackStore().add_many(items) + precision_store = DetectorPrecisionStore() + for item in items: + if isinstance(item, dict): + precision_store.record(str(item.get("detector", "")), str(item.get("status", "")), stream_id=str(item.get("stream_id", "")), field=str(item.get("field", ""))) + self._send(200, "application/json", json.dumps(result).encode("utf-8")) except (ValueError, json.JSONDecodeError) as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return diff --git a/src/fgai/detectors.py b/src/fgai/detectors.py index 714517a..f0de7d4 100644 --- a/src/fgai/detectors.py +++ b/src/fgai/detectors.py @@ -11,6 +11,21 @@ DETECTOR_MINIMUMS = { "deny_action": 10, } +# Display-only reference for the dashboard's Detector Health panel — never +# consulted by detection/scoring logic itself. Keyed by the name reported on +# a deviation (the *_burst suffix baseline.py appends when it flags one of +# these), not the bare profile.detectors config key. +DETECTOR_DESCRIPTIONS = { + "auth_failure_burst": "Authentication failures for an entity above its historical baseline rate.", + "dns_query_burst": "DNS query volume for an entity above its historical baseline rate.", + "deny_action_burst": "Denied/blocked security events for an entity above its historical baseline rate.", + "event_rate_burst": "Overall event volume for an entity above its historical baseline rate.", + "numeric_baseline": "A profiled numeric field's value deviates more than 3 standard deviations from its historical baseline.", + "new_relationship": "A profiled field-pair combination (e.g. user -> source IP) not seen before for this entity.", + "rare_value": "A profiled categorical field value not seen before for this entity.", + "dns_network_auth_sequence": "An ordered cross-stream sequence (e.g. DNS lookup -> connection -> auth) matched for this entity.", +} + def event_detector_categories(event: LogEvent) -> tuple[str, ...]: fields = event.fields diff --git a/src/fgai/feedback.py b/src/fgai/feedback.py index 2bcb8bb..94108bb 100644 --- a/src/fgai/feedback.py +++ b/src/fgai/feedback.py @@ -2,6 +2,7 @@ from __future__ import annotations import hashlib import json +import sqlite3 import time from pathlib import Path @@ -153,6 +154,61 @@ class FeedbackStore: return {"removed": removed, "remaining": len(entries), "status": status or "all"} +class DetectorPrecisionStore: + """Permanent, append-only log of review outcomes per detector. + + Deliberately separate from FeedbackStore, which holds mutable suppression + rules that can be revoked or expire — reusing that as a historical signal + would let a revoked rule erase the record of whether a detector's findings + tend to be right. Here, once a human reviews a finding, the verdict is kept + forever regardless of what later happens to the resulting suppression rule. + """ + + def __init__(self, path: str = "state/signalscope-detector-precision.sqlite3") -> None: + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.path) as connection: + connection.execute( + "create table if not exists outcomes (" + "id integer primary key autoincrement, detector text not null, status text not null, " + "stream_id text not null default '', field text not null default '', recorded_at integer not null)" + ) + connection.execute("create index if not exists idx_outcomes_detector on outcomes(detector, recorded_at)") + + def record(self, detector: str, status: str, *, stream_id: str = "", field: str = "", recorded_at: int | None = None) -> None: + detector = str(detector or "").strip() + status = str(status or "").lower() + if not detector or status not in VALID_STATUSES: + return + with sqlite3.connect(self.path) as connection: + connection.execute( + "insert into outcomes (detector, status, stream_id, field, recorded_at) values (?, ?, ?, ?, ?)", + (detector, status, stream_id, field, recorded_at if recorded_at is not None else int(time.time())), + ) + + def precision(self, *, days: int = 30) -> dict[str, dict[str, object]]: + cutoff = int(time.time()) - max(1, days) * 86400 + result: dict[str, dict[str, object]] = {} + with sqlite3.connect(self.path) as connection: + rows = connection.execute( + "select detector, status, count(*) from outcomes where recorded_at >= ? group by detector, status", + (cutoff,), + ).fetchall() + for detector, status, count in rows: + counts = result.setdefault(detector, {"confirmed": 0, "false_positive": 0, "expected": 0}) + counts[status] = count + for counts in result.values(): + denominator = counts["confirmed"] + counts["false_positive"] + counts["precision"] = round(counts["confirmed"] / denominator, 4) if denominator else None + counts["reviewed_total"] = denominator + counts["expected"] + return result + + def maintenance(self, *, retention_days: int = 180) -> int: + cutoff = int(time.time()) - max(1, retention_days) * 86400 + with sqlite3.connect(self.path) as connection: + return connection.execute("delete from outcomes where recorded_at < ?", (cutoff,)).rowcount + + def feedback_matches(entry: dict[str, object], *, entity: str, stream_id: str, field: str, value: str) -> bool: scope = str(entry.get("scope") or _resolve_scope(entry)) if scope == "stream": diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index e25341c..f605a44 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -13,7 +13,8 @@ from .baseline import BaselineStore from .config import ConfigStore from .correlation import correlate_source_ips from .event_context import build_event_context -from .feedback import FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations +from .detectors import DETECTOR_DESCRIPTIONS +from .feedback import DetectorPrecisionStore, FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations from .graylog_aggregate import GraylogAggregateSource from .graylog_mcp import GraylogMcpClient from .graylog_source import GraylogStreamSource @@ -286,6 +287,39 @@ def _cluster_field_deviations(field_deviations: dict[str, list[dict[str, object] deviation["cluster_dampened"] = True +# baseline.py reports a burst deviation's detector as f"{key}_burst" (see +# profile_deviations), but profile.detectors/detector_readiness() use the +# bare config key — this is the one place that reconciles the two names. +_DETECTOR_READINESS_DISPLAY_NAME = {"auth_failure": "auth_failure_burst", "dns_query": "dns_query_burst", "deny_action": "deny_action_burst"} + + +def _build_detector_health(detector_readiness: list[dict[str, object]], detector_precision: dict[str, dict[str, object]]) -> list[dict[str, object]]: + readiness_by_detector: dict[str, list[dict[str, object]]] = defaultdict(list) + for row in detector_readiness: + display_name = _DETECTOR_READINESS_DISPLAY_NAME.get(str(row.get("detector", "")), str(row.get("detector", ""))) + readiness_by_detector[display_name].append(row) + + names = sorted({*DETECTOR_DESCRIPTIONS, *readiness_by_detector, *detector_precision}) + health = [] + for name in names: + streams = readiness_by_detector.get(name, []) + ready_streams = sum(1 for item in streams if item.get("ready")) + precision = detector_precision.get(name, {}) + health.append({ + "detector": name, + "description": DETECTOR_DESCRIPTIONS.get(name, ""), + "streams_configured": len(streams), + "streams_ready": ready_streams, + "learning_state": "no_data" if not streams else "ready" if ready_streams == len(streams) else "learning", + "confirmed": precision.get("confirmed", 0), + "false_positive": precision.get("false_positive", 0), + "expected": precision.get("expected", 0), + "precision": precision.get("precision"), + "reviewed_total": precision.get("reviewed_total", 0), + }) + return health + + def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str: return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id @@ -592,6 +626,9 @@ def build_status( } for item in profile_readiness ] + detector_readiness_rows = baseline.detector_readiness(stream_profiles, min_training_days=baseline_training_days) if baseline else [] + detector_precision = DetectorPrecisionStore().precision(days=30) + detector_health = _build_detector_health(detector_readiness_rows, detector_precision) stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles) discovery_cache_events = [] discovered_profile_fields = 0 @@ -734,6 +771,7 @@ def build_status( "profile_shape_warnings": profile_shape_warnings, "shared_entity_baselines": shared_entity_baselines, "profile_readiness": profile_readiness, + "detector_health": detector_health, "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), "top_destination_ips": top_field_values(events, "dstip", limit=10), diff --git a/tests/test_baseline.py b/tests/test_baseline.py index aa4c88f..a9f18e8 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -254,3 +254,62 @@ class BaselineTests(unittest.TestCase): self.assertEqual(relation["field"], "relationship:username->srcip") self.assertEqual(relation["value"], "10.0.0.99") self.assertIn("new srcip value for username=peter", relation["reason"]) + + def test_detector_readiness_reports_learning_before_twelve_buckets(self): + with tempfile.TemporaryDirectory() as directory: + store = BaselineStore(str(Path(directory) / "baseline.sqlite3")) + profiles = parse_profiles([{ + "stream_id": "windows", + "entity_field": "username", + "categorical_fields": ["action"], + "detectors": {"auth_failure": {"enabled": True}}, + }]) + for index in range(5): + event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}") + store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300) + + readiness = store.detector_readiness(profiles) + + row = next(item for item in readiness if item["detector"] == "auth_failure") + self.assertEqual(row["stream_id"], "windows") + self.assertEqual(row["buckets"], 5) + self.assertFalse(row["ready"]) + + def test_detector_readiness_reports_ready_after_twelve_buckets(self): + with tempfile.TemporaryDirectory() as directory: + store = BaselineStore(str(Path(directory) / "baseline.sqlite3")) + profiles = parse_profiles([{ + "stream_id": "windows", + "entity_field": "username", + "categorical_fields": ["action"], + "detectors": {"auth_failure": {"enabled": True}}, + }]) + for index in range(12): + event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}") + store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300) + + readiness = store.detector_readiness(profiles) + + row = next(item for item in readiness if item["detector"] == "auth_failure") + self.assertEqual(row["buckets"], 12) + self.assertTrue(row["ready"]) + + def test_detector_readiness_respects_min_training_days(self): + with tempfile.TemporaryDirectory() as directory: + store = BaselineStore(str(Path(directory) / "baseline.sqlite3")) + profiles = parse_profiles([{ + "stream_id": "windows", + "entity_field": "username", + "categorical_fields": ["action"], + "detectors": {"auth_failure": {"enabled": True}}, + }]) + now = int(time.time()) + for index in range(12): + event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}") + store.ingest_profile_fields([event], profiles, observed_at=now - 86400 + index * 300) + + readiness = store.detector_readiness(profiles, min_training_days=7) + + row = next(item for item in readiness if item["detector"] == "auth_failure") + self.assertTrue(row["buckets"] >= 12) + self.assertFalse(row["ready"]) diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 7f4e910..2e1eb73 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -104,6 +104,17 @@ class DashboardTests(unittest.TestCase): self.assertIn("Shared Entity Baselines", HTML) self.assertIn("shared_entity_baselines", HTML) + def test_dashboard_shows_detector_health(self): + self.assertIn("detector-health", HTML) + self.assertIn("Detector Health", HTML) + self.assertIn("detector_health", HTML) + + def test_review_actions_carry_detector_for_precision_tracking(self): + self.assertIn('data-detector="${esc(r.detector', HTML) + self.assertIn("detector: button.dataset.detector", HTML) + self.assertIn("detector: pendingReview.detector", HTML) + self.assertIn("detector: row.detector", HTML) + def test_dashboard_shows_shape_mismatch_warnings(self): self.assertIn('id="profileShapeWarnings"', HTML) self.assertIn("profile_shape_warnings", HTML) diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 1121354..ca149ad 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -3,7 +3,7 @@ import time import unittest from pathlib import Path -from fgai.feedback import FeedbackStore, feedback_matches, select_match, suggest_generalizations +from fgai.feedback import DetectorPrecisionStore, FeedbackStore, feedback_matches, select_match, suggest_generalizations class FeedbackTests(unittest.TestCase): @@ -157,6 +157,55 @@ class FeedbackTests(unittest.TestCase): {"stream_id": "s", "entity": "alice", "field": "f", "status": "false_positive", "value_count": 2, "sample_values": ["1", "2"], "entry_ids": ["1", "2"]} ]) + def test_detector_precision_store_computes_ratio_from_recorded_outcomes(self): + with tempfile.TemporaryDirectory() as directory: + store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3")) + store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure") + store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure") + store.record("auth_failure_burst", "false_positive", stream_id="windows", field="auth_failure") + store.record("new_relationship", "expected", stream_id="firewall", field="relationship:srcip->dstip") + + precision = store.precision(days=30) + + auth = precision["auth_failure_burst"] + self.assertEqual(auth["confirmed"], 2) + self.assertEqual(auth["false_positive"], 1) + self.assertAlmostEqual(auth["precision"], 2 / 3, places=3) + self.assertEqual(auth["reviewed_total"], 3) + self.assertIsNone(precision["new_relationship"]["precision"]) + self.assertEqual(precision["new_relationship"]["reviewed_total"], 1) + + def test_detector_precision_store_excludes_outcomes_outside_the_window(self): + with tempfile.TemporaryDirectory() as directory: + store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3")) + old_timestamp = int(time.time()) - 40 * 86400 + store.record("auth_failure_burst", "confirmed", recorded_at=old_timestamp) + store.record("auth_failure_burst", "false_positive") + + precision = store.precision(days=30) + + self.assertEqual(precision["auth_failure_burst"]["confirmed"], 0) + self.assertEqual(precision["auth_failure_burst"]["false_positive"], 1) + + def test_detector_precision_store_ignores_invalid_or_empty_input(self): + with tempfile.TemporaryDirectory() as directory: + store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3")) + store.record("", "confirmed") + store.record("auth_failure_burst", "not_a_real_status") + + self.assertEqual(store.precision(days=30), {}) + + def test_detector_precision_store_maintenance_prunes_old_rows(self): + with tempfile.TemporaryDirectory() as directory: + store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3")) + store.record("auth_failure_burst", "confirmed", recorded_at=int(time.time()) - 200 * 86400) + store.record("auth_failure_burst", "confirmed") + + removed = store.maintenance(retention_days=180) + + self.assertEqual(removed, 1) + self.assertEqual(store.precision(days=3650)["auth_failure_burst"]["confirmed"], 1) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 403c099..e4192d0 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -258,6 +258,43 @@ class MonitorTests(unittest.TestCase): self.assertEqual(pool["group"], "srcip") self.assertEqual(set(pool["streams"]), {"firewall", "iis"}) + def test_detector_health_lists_known_detectors_with_no_data_by_default(self): + with tempfile.TemporaryDirectory() as tmp: + with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]): + status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json")) + + health_by_detector = {item["detector"]: item for item in status["detector_health"]} + self.assertIn("auth_failure_burst", health_by_detector) + self.assertIn("new_relationship", health_by_detector) + self.assertEqual(health_by_detector["auth_failure_burst"]["learning_state"], "no_data") + self.assertIsNone(health_by_detector["auth_failure_burst"]["precision"]) + + def test_detector_health_reflects_readiness_and_precision(self): + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / "config.json" + config_path.write_text( + json.dumps( + { + "graylog_streams": [{"id": "windows", "title": "Windows", "enabled": True}], + "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "categorical_fields": ["action"], "detectors": {"auth_failure": {"enabled": True}}}], + } + ), + encoding="utf-8", + ) + readiness_rows = [{"stream_id": "windows", "detector": "auth_failure", "buckets": 12, "age_days": 10, "training_days": 0, "ready": True}] + precision_rows = {"auth_failure_burst": {"confirmed": 3, "false_positive": 1, "expected": 0, "precision": 0.75, "reviewed_total": 4}} + with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]), \ + patch("fgai.monitor.BaselineStore.detector_readiness", return_value=readiness_rows), \ + patch("fgai.monitor.DetectorPrecisionStore.precision", return_value=precision_rows): + status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) + + row = next(item for item in status["detector_health"] if item["detector"] == "auth_failure_burst") + self.assertEqual(row["learning_state"], "ready") + self.assertEqual(row["streams_ready"], 1) + self.assertEqual(row["streams_configured"], 1) + self.assertEqual(row["confirmed"], 3) + self.assertEqual(row["precision"], 0.75) + def test_shared_entity_baselines_empty_for_single_stream(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json"