Add real detector health tooling: learning state and historical precision
Replaces the old Prometheus-only "detector precision" metric, which recomputed from whatever field_deviations happened to be active in the current poll cycle cross-referenced against live FeedbackStore rules — a revoked or expired suppression rule made the signal vanish, and nothing was tracked between polls. Adds DetectorPrecisionStore, a permanent append-only log of review outcomes (kept deliberately separate from FeedbackStore's mutable suppression rules) recorded the moment a human reviews a finding, plus BaselineStore.detector_readiness() mirroring the existing per-field profile_readiness() for per-detector learning state, and a plain-language detector reference. Surfaced in a new Detector Health dashboard panel. Observability only — nothing here changes scoring or suppression. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
10
README.md
10
README.md
@@ -182,6 +182,16 @@ The review buttons are part of the detection loop:
|
||||
- Expiry should be used for temporary expected changes, such as maintenance or a
|
||||
migration window.
|
||||
|
||||
Every review click also permanently records which detector produced the
|
||||
finding and the verdict, independent of the suppression rule it creates —
|
||||
so revoking or letting a rule expire later never erases that history. Check
|
||||
`Diagnostics -> Detector Health` to see, per detector: whether it has
|
||||
learned enough baseline history to be trusted yet, its confirmed-vs-false-positive
|
||||
precision over the last 30 days, and a one-line description of what
|
||||
actually triggers it. A detector with no reviews yet shows `n/a` for
|
||||
precision rather than a misleading 0% or 100% — the number only becomes
|
||||
meaningful once there's real history behind it.
|
||||
|
||||
## Primary Workflow: Graylog MCP
|
||||
|
||||
Graylog 7.1 MCP is the primary log-source integration. In the dashboard, open
|
||||
|
||||
12
ROADMAP.md
12
ROADMAP.md
@@ -94,7 +94,17 @@ Goal: make findings more accurate before adding more integrations.
|
||||
- [x] Add a dry-run replay command for historic JSONL or Graylog exports using temporary baselines.
|
||||
- [x] Add direct Graylog MCP time-range replay and result comparison against saved detector configurations.
|
||||
- [ ] Add dashboard controls for launching safe replay jobs and viewing detector deltas.
|
||||
- [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold.
|
||||
- [x] Add baseline confidence tooling: per detector learning state, expected
|
||||
false-positive rate, and why a deviation crossed threshold. The prior
|
||||
"detector precision" metric only existed in the Prometheus endpoint and
|
||||
recomputed from whatever was active in the current poll cycle — a
|
||||
revoked or expired suppression rule made the signal vanish. Replaced with
|
||||
a permanent, append-only review-outcome log (`DetectorPrecisionStore`,
|
||||
separate from the mutable `FeedbackStore`) recorded the moment a human
|
||||
clicks Expected/False positive/Confirmed, plus per-detector learning
|
||||
state (`BaselineStore.detector_readiness()`, mirroring the existing
|
||||
per-field `profile_readiness()`) and a plain-language "what triggers
|
||||
this" reference — all surfaced in a new Detector Health dashboard panel.
|
||||
- [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status.
|
||||
- [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings.
|
||||
- [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping.
|
||||
|
||||
@@ -668,6 +668,17 @@ class BaselineStore:
|
||||
rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days})
|
||||
return rows
|
||||
|
||||
def detector_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
|
||||
rows: list[dict[str, object]] = []
|
||||
with self._connect() as connection:
|
||||
for stream_id, profile in profiles.items():
|
||||
for detector in getattr(profile, "detectors", {}):
|
||||
count = connection.execute("select count(distinct bucket_start) from profile_detector_buckets where stream_id=? and detector=?", (stream_id, str(detector).lower())).fetchone()[0]
|
||||
oldest = connection.execute("select min(bucket_start) from profile_detector_buckets where stream_id=? and detector=?", (stream_id, str(detector).lower())).fetchone()[0]
|
||||
age_days = _age_days(oldest, int(time.time()))
|
||||
rows.append({"stream_id": stream_id, "detector": str(detector), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days})
|
||||
return rows
|
||||
|
||||
def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]:
|
||||
profiles: dict[str, dict[str, object]] = {}
|
||||
with self._connect() as connection:
|
||||
|
||||
@@ -10,7 +10,7 @@ from .config import ConfigStore
|
||||
from .exports import investigation_report, investigation_report_markdown
|
||||
from .graylog_mcp import GraylogMcpClient
|
||||
from .metrics import prometheus_metrics
|
||||
from .feedback import FeedbackStore
|
||||
from .feedback import DetectorPrecisionStore, FeedbackStore
|
||||
from .incidents import IncidentStore
|
||||
|
||||
|
||||
@@ -896,7 +896,7 @@ async function refresh() {
|
||||
if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
|
||||
document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [
|
||||
{label:'', render:r => { const key = deviationKey(r); return `<input type="checkbox" class="deviation-select" data-key="${esc(key)}" ${selectedDeviationKeys.has(key) ? 'checked' : ''}>`; }},
|
||||
{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Suggested', render:suggestReview}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const clusterNote = r.cluster_size >= 5 ? `[part of a ${r.cluster_size}-entity pattern${r.cluster_dampened ? '; score capped' : ''}] ` : ''; const summary=esc(`${clusterNote}${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">False positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Confirm</button></div>`}
|
||||
{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Suggested', render:suggestReview}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const clusterNote = r.cluster_size >= 5 ? `[part of a ${r.cluster_size}-entity pattern${r.cluster_dampened ? '; score capped' : ''}] ` : ''; const summary=esc(`${clusterNote}${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}" data-detector="${esc(r.detector || '')}">Expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}" data-detector="${esc(r.detector || '')}">False positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}" data-detector="${esc(r.detector || '')}">Confirm</button></div>`}
|
||||
], 'field-deviations');
|
||||
const bulkReviewBar = document.getElementById('bulkReviewBar');
|
||||
const updateBulkReviewBar = () => {
|
||||
@@ -930,7 +930,7 @@ async function refresh() {
|
||||
const author = document.getElementById('bulkReviewAuthor').value || '';
|
||||
const days = Number(document.getElementById('bulkReviewExpiryDays').value || '0');
|
||||
const expiresAt = days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0;
|
||||
const items = selectedRows.map(row => ({status, scope, entity: row.entity, stream_id: row.stream_id, field: row.field, value: row.value, note, author, expires_at: expiresAt}));
|
||||
const items = selectedRows.map(row => ({status, scope, entity: row.entity, stream_id: row.stream_id, field: row.field, value: row.value, detector: row.detector, note, author, expires_at: expiresAt}));
|
||||
const response = await fetch('/api/feedback/bulk', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({items})});
|
||||
const result = await response.json().catch(() => ({}));
|
||||
if (response.ok) {
|
||||
@@ -950,7 +950,7 @@ async function refresh() {
|
||||
reviewScopeWarning.hidden = !broad;
|
||||
};
|
||||
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', () => {
|
||||
pendingReview = {status: button.dataset.status, entity: button.dataset.entity, stream_id: button.dataset.stream, field: button.dataset.field, value: button.dataset.value};
|
||||
pendingReview = {status: button.dataset.status, entity: button.dataset.entity, stream_id: button.dataset.stream, field: button.dataset.field, value: button.dataset.value, detector: button.dataset.detector};
|
||||
reviewScopeSelect.value = pendingReview.value ? 'value' : 'field';
|
||||
document.getElementById('reviewNote').value = '';
|
||||
document.getElementById('reviewExpiryDays').value = '0';
|
||||
@@ -980,7 +980,7 @@ async function refresh() {
|
||||
if (!confirm(`This will suppress ALL findings for ${target} across every field until it expires. Continue?`)) return;
|
||||
}
|
||||
lastReviewAuthor = author;
|
||||
const payload = {status: pendingReview.status, scope, entity: pendingReview.entity, stream_id: pendingReview.stream_id, field: pendingReview.field, value: pendingReview.value, note, author, expires_at: days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0};
|
||||
const payload = {status: pendingReview.status, scope, entity: pendingReview.entity, stream_id: pendingReview.stream_id, field: pendingReview.field, value: pendingReview.value, detector: pendingReview.detector, note, author, expires_at: days > 0 ? Math.floor(Date.now()/1000) + days * 86400 : 0};
|
||||
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload)});
|
||||
if (response.ok) {
|
||||
document.getElementById('feedbackNotice').textContent = 'Review saved. The matching pattern will be labeled on the next refresh.';
|
||||
@@ -1070,6 +1070,7 @@ async function refresh() {
|
||||
collapsibleSection('correlations', 'Cross-Source Correlations', table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations')) +
|
||||
collapsibleSection('entities', 'Entities', table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities')) +
|
||||
collapsibleSection('profile-readiness', 'Profile Baseline Readiness', table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `<button type="button" class="delete-stream-profile small-action danger-action" title="Delete profile" data-stream-id="${esc(r.stream_id)}" data-stream-name="${esc(r.stream_title || r.profile_name || r.stream_id)}">Remove</button>` : '-'}], 'profile-readiness')) +
|
||||
collapsibleSection('detector-health', 'Detector Health', table(data.detector_health || [], [{label:'Detector', key:'detector'}, {label:'Description', key:'description'}, {label:'Learning state', render:r => r.learning_state === 'no_data' ? 'no data yet' : r.learning_state === 'ready' ? `ready (${r.streams_ready}/${r.streams_configured} streams)` : `learning (${r.streams_ready}/${r.streams_configured} streams)`}, {label:'Precision (30d)', render:r => r.precision === null || r.precision === undefined ? 'n/a' : `${Math.round(r.precision * 100)}%`}, {label:'Confirmed', key:'confirmed'}, {label:'False positive', key:'false_positive'}, {label:'Expected', key:'expected'}], 'detector-health')) +
|
||||
collapsibleSection('data-quality', 'Data Quality', table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}], 'data-quality')) +
|
||||
collapsibleSection('feedback-rules', 'Feedback Rules', table(
|
||||
['value', 'field', 'entity', 'stream'].map(scope => ({scope, count: feedbackRules.filter(item => (item.scope || 'value') === scope).length})),
|
||||
@@ -1552,7 +1553,9 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
|
||||
if self.path == "/api/feedback" and self._is_loopback_client():
|
||||
try:
|
||||
payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8"))
|
||||
self._send(200, "application/json", json.dumps(FeedbackStore().add(payload)).encode("utf-8"))
|
||||
entry = FeedbackStore().add(payload)
|
||||
DetectorPrecisionStore().record(str(payload.get("detector", "")), str(payload.get("status", "")), stream_id=str(payload.get("stream_id", "")), field=str(payload.get("field", "")))
|
||||
self._send(200, "application/json", json.dumps(entry).encode("utf-8"))
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
|
||||
return
|
||||
@@ -1560,7 +1563,13 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
|
||||
try:
|
||||
payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 65_536)).decode("utf-8"))
|
||||
items = payload.get("items", []) if isinstance(payload, dict) else []
|
||||
self._send(200, "application/json", json.dumps(FeedbackStore().add_many(items if isinstance(items, list) else [])).encode("utf-8"))
|
||||
items = items if isinstance(items, list) else []
|
||||
result = FeedbackStore().add_many(items)
|
||||
precision_store = DetectorPrecisionStore()
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
precision_store.record(str(item.get("detector", "")), str(item.get("status", "")), stream_id=str(item.get("stream_id", "")), field=str(item.get("field", "")))
|
||||
self._send(200, "application/json", json.dumps(result).encode("utf-8"))
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
|
||||
return
|
||||
|
||||
@@ -11,6 +11,21 @@ DETECTOR_MINIMUMS = {
|
||||
"deny_action": 10,
|
||||
}
|
||||
|
||||
# Display-only reference for the dashboard's Detector Health panel — never
|
||||
# consulted by detection/scoring logic itself. Keyed by the name reported on
|
||||
# a deviation (the *_burst suffix baseline.py appends when it flags one of
|
||||
# these), not the bare profile.detectors config key.
|
||||
DETECTOR_DESCRIPTIONS = {
|
||||
"auth_failure_burst": "Authentication failures for an entity above its historical baseline rate.",
|
||||
"dns_query_burst": "DNS query volume for an entity above its historical baseline rate.",
|
||||
"deny_action_burst": "Denied/blocked security events for an entity above its historical baseline rate.",
|
||||
"event_rate_burst": "Overall event volume for an entity above its historical baseline rate.",
|
||||
"numeric_baseline": "A profiled numeric field's value deviates more than 3 standard deviations from its historical baseline.",
|
||||
"new_relationship": "A profiled field-pair combination (e.g. user -> source IP) not seen before for this entity.",
|
||||
"rare_value": "A profiled categorical field value not seen before for this entity.",
|
||||
"dns_network_auth_sequence": "An ordered cross-stream sequence (e.g. DNS lookup -> connection -> auth) matched for this entity.",
|
||||
}
|
||||
|
||||
|
||||
def event_detector_categories(event: LogEvent) -> tuple[str, ...]:
|
||||
fields = event.fields
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
@@ -153,6 +154,61 @@ class FeedbackStore:
|
||||
return {"removed": removed, "remaining": len(entries), "status": status or "all"}
|
||||
|
||||
|
||||
class DetectorPrecisionStore:
|
||||
"""Permanent, append-only log of review outcomes per detector.
|
||||
|
||||
Deliberately separate from FeedbackStore, which holds mutable suppression
|
||||
rules that can be revoked or expire — reusing that as a historical signal
|
||||
would let a revoked rule erase the record of whether a detector's findings
|
||||
tend to be right. Here, once a human reviews a finding, the verdict is kept
|
||||
forever regardless of what later happens to the resulting suppression rule.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str = "state/signalscope-detector-precision.sqlite3") -> None:
|
||||
self.path = Path(path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sqlite3.connect(self.path) as connection:
|
||||
connection.execute(
|
||||
"create table if not exists outcomes ("
|
||||
"id integer primary key autoincrement, detector text not null, status text not null, "
|
||||
"stream_id text not null default '', field text not null default '', recorded_at integer not null)"
|
||||
)
|
||||
connection.execute("create index if not exists idx_outcomes_detector on outcomes(detector, recorded_at)")
|
||||
|
||||
def record(self, detector: str, status: str, *, stream_id: str = "", field: str = "", recorded_at: int | None = None) -> None:
|
||||
detector = str(detector or "").strip()
|
||||
status = str(status or "").lower()
|
||||
if not detector or status not in VALID_STATUSES:
|
||||
return
|
||||
with sqlite3.connect(self.path) as connection:
|
||||
connection.execute(
|
||||
"insert into outcomes (detector, status, stream_id, field, recorded_at) values (?, ?, ?, ?, ?)",
|
||||
(detector, status, stream_id, field, recorded_at if recorded_at is not None else int(time.time())),
|
||||
)
|
||||
|
||||
def precision(self, *, days: int = 30) -> dict[str, dict[str, object]]:
|
||||
cutoff = int(time.time()) - max(1, days) * 86400
|
||||
result: dict[str, dict[str, object]] = {}
|
||||
with sqlite3.connect(self.path) as connection:
|
||||
rows = connection.execute(
|
||||
"select detector, status, count(*) from outcomes where recorded_at >= ? group by detector, status",
|
||||
(cutoff,),
|
||||
).fetchall()
|
||||
for detector, status, count in rows:
|
||||
counts = result.setdefault(detector, {"confirmed": 0, "false_positive": 0, "expected": 0})
|
||||
counts[status] = count
|
||||
for counts in result.values():
|
||||
denominator = counts["confirmed"] + counts["false_positive"]
|
||||
counts["precision"] = round(counts["confirmed"] / denominator, 4) if denominator else None
|
||||
counts["reviewed_total"] = denominator + counts["expected"]
|
||||
return result
|
||||
|
||||
def maintenance(self, *, retention_days: int = 180) -> int:
|
||||
cutoff = int(time.time()) - max(1, retention_days) * 86400
|
||||
with sqlite3.connect(self.path) as connection:
|
||||
return connection.execute("delete from outcomes where recorded_at < ?", (cutoff,)).rowcount
|
||||
|
||||
|
||||
def feedback_matches(entry: dict[str, object], *, entity: str, stream_id: str, field: str, value: str) -> bool:
|
||||
scope = str(entry.get("scope") or _resolve_scope(entry))
|
||||
if scope == "stream":
|
||||
|
||||
@@ -13,7 +13,8 @@ from .baseline import BaselineStore
|
||||
from .config import ConfigStore
|
||||
from .correlation import correlate_source_ips
|
||||
from .event_context import build_event_context
|
||||
from .feedback import FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations
|
||||
from .detectors import DETECTOR_DESCRIPTIONS
|
||||
from .feedback import DetectorPrecisionStore, FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations
|
||||
from .graylog_aggregate import GraylogAggregateSource
|
||||
from .graylog_mcp import GraylogMcpClient
|
||||
from .graylog_source import GraylogStreamSource
|
||||
@@ -286,6 +287,39 @@ def _cluster_field_deviations(field_deviations: dict[str, list[dict[str, object]
|
||||
deviation["cluster_dampened"] = True
|
||||
|
||||
|
||||
# baseline.py reports a burst deviation's detector as f"{key}_burst" (see
|
||||
# profile_deviations), but profile.detectors/detector_readiness() use the
|
||||
# bare config key — this is the one place that reconciles the two names.
|
||||
_DETECTOR_READINESS_DISPLAY_NAME = {"auth_failure": "auth_failure_burst", "dns_query": "dns_query_burst", "deny_action": "deny_action_burst"}
|
||||
|
||||
|
||||
def _build_detector_health(detector_readiness: list[dict[str, object]], detector_precision: dict[str, dict[str, object]]) -> list[dict[str, object]]:
|
||||
readiness_by_detector: dict[str, list[dict[str, object]]] = defaultdict(list)
|
||||
for row in detector_readiness:
|
||||
display_name = _DETECTOR_READINESS_DISPLAY_NAME.get(str(row.get("detector", "")), str(row.get("detector", "")))
|
||||
readiness_by_detector[display_name].append(row)
|
||||
|
||||
names = sorted({*DETECTOR_DESCRIPTIONS, *readiness_by_detector, *detector_precision})
|
||||
health = []
|
||||
for name in names:
|
||||
streams = readiness_by_detector.get(name, [])
|
||||
ready_streams = sum(1 for item in streams if item.get("ready"))
|
||||
precision = detector_precision.get(name, {})
|
||||
health.append({
|
||||
"detector": name,
|
||||
"description": DETECTOR_DESCRIPTIONS.get(name, ""),
|
||||
"streams_configured": len(streams),
|
||||
"streams_ready": ready_streams,
|
||||
"learning_state": "no_data" if not streams else "ready" if ready_streams == len(streams) else "learning",
|
||||
"confirmed": precision.get("confirmed", 0),
|
||||
"false_positive": precision.get("false_positive", 0),
|
||||
"expected": precision.get("expected", 0),
|
||||
"precision": precision.get("precision"),
|
||||
"reviewed_total": precision.get("reviewed_total", 0),
|
||||
})
|
||||
return health
|
||||
|
||||
|
||||
def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
|
||||
return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
|
||||
|
||||
@@ -592,6 +626,9 @@ def build_status(
|
||||
}
|
||||
for item in profile_readiness
|
||||
]
|
||||
detector_readiness_rows = baseline.detector_readiness(stream_profiles, min_training_days=baseline_training_days) if baseline else []
|
||||
detector_precision = DetectorPrecisionStore().precision(days=30)
|
||||
detector_health = _build_detector_health(detector_readiness_rows, detector_precision)
|
||||
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
|
||||
discovery_cache_events = []
|
||||
discovered_profile_fields = 0
|
||||
@@ -734,6 +771,7 @@ def build_status(
|
||||
"profile_shape_warnings": profile_shape_warnings,
|
||||
"shared_entity_baselines": shared_entity_baselines,
|
||||
"profile_readiness": profile_readiness,
|
||||
"detector_health": detector_health,
|
||||
"diagnostics": {
|
||||
"top_source_ips": top_field_values(events, "srcip", limit=10),
|
||||
"top_destination_ips": top_field_values(events, "dstip", limit=10),
|
||||
|
||||
@@ -254,3 +254,62 @@ class BaselineTests(unittest.TestCase):
|
||||
self.assertEqual(relation["field"], "relationship:username->srcip")
|
||||
self.assertEqual(relation["value"], "10.0.0.99")
|
||||
self.assertIn("new srcip value for username=peter", relation["reason"])
|
||||
|
||||
def test_detector_readiness_reports_learning_before_twelve_buckets(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
|
||||
profiles = parse_profiles([{
|
||||
"stream_id": "windows",
|
||||
"entity_field": "username",
|
||||
"categorical_fields": ["action"],
|
||||
"detectors": {"auth_failure": {"enabled": True}},
|
||||
}])
|
||||
for index in range(5):
|
||||
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
|
||||
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
|
||||
|
||||
readiness = store.detector_readiness(profiles)
|
||||
|
||||
row = next(item for item in readiness if item["detector"] == "auth_failure")
|
||||
self.assertEqual(row["stream_id"], "windows")
|
||||
self.assertEqual(row["buckets"], 5)
|
||||
self.assertFalse(row["ready"])
|
||||
|
||||
def test_detector_readiness_reports_ready_after_twelve_buckets(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
|
||||
profiles = parse_profiles([{
|
||||
"stream_id": "windows",
|
||||
"entity_field": "username",
|
||||
"categorical_fields": ["action"],
|
||||
"detectors": {"auth_failure": {"enabled": True}},
|
||||
}])
|
||||
for index in range(12):
|
||||
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
|
||||
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
|
||||
|
||||
readiness = store.detector_readiness(profiles)
|
||||
|
||||
row = next(item for item in readiness if item["detector"] == "auth_failure")
|
||||
self.assertEqual(row["buckets"], 12)
|
||||
self.assertTrue(row["ready"])
|
||||
|
||||
def test_detector_readiness_respects_min_training_days(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
|
||||
profiles = parse_profiles([{
|
||||
"stream_id": "windows",
|
||||
"entity_field": "username",
|
||||
"categorical_fields": ["action"],
|
||||
"detectors": {"auth_failure": {"enabled": True}},
|
||||
}])
|
||||
now = int(time.time())
|
||||
for index in range(12):
|
||||
event = parse_log_line(f"fgai_stream_id=windows username=alice action=failed baseline={index}")
|
||||
store.ingest_profile_fields([event], profiles, observed_at=now - 86400 + index * 300)
|
||||
|
||||
readiness = store.detector_readiness(profiles, min_training_days=7)
|
||||
|
||||
row = next(item for item in readiness if item["detector"] == "auth_failure")
|
||||
self.assertTrue(row["buckets"] >= 12)
|
||||
self.assertFalse(row["ready"])
|
||||
|
||||
@@ -104,6 +104,17 @@ class DashboardTests(unittest.TestCase):
|
||||
self.assertIn("Shared Entity Baselines", HTML)
|
||||
self.assertIn("shared_entity_baselines", HTML)
|
||||
|
||||
def test_dashboard_shows_detector_health(self):
|
||||
self.assertIn("detector-health", HTML)
|
||||
self.assertIn("Detector Health", HTML)
|
||||
self.assertIn("detector_health", HTML)
|
||||
|
||||
def test_review_actions_carry_detector_for_precision_tracking(self):
|
||||
self.assertIn('data-detector="${esc(r.detector', HTML)
|
||||
self.assertIn("detector: button.dataset.detector", HTML)
|
||||
self.assertIn("detector: pendingReview.detector", HTML)
|
||||
self.assertIn("detector: row.detector", HTML)
|
||||
|
||||
def test_dashboard_shows_shape_mismatch_warnings(self):
|
||||
self.assertIn('id="profileShapeWarnings"', HTML)
|
||||
self.assertIn("profile_shape_warnings", HTML)
|
||||
|
||||
@@ -3,7 +3,7 @@ import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from fgai.feedback import FeedbackStore, feedback_matches, select_match, suggest_generalizations
|
||||
from fgai.feedback import DetectorPrecisionStore, FeedbackStore, feedback_matches, select_match, suggest_generalizations
|
||||
|
||||
|
||||
class FeedbackTests(unittest.TestCase):
|
||||
@@ -157,6 +157,55 @@ class FeedbackTests(unittest.TestCase):
|
||||
{"stream_id": "s", "entity": "alice", "field": "f", "status": "false_positive", "value_count": 2, "sample_values": ["1", "2"], "entry_ids": ["1", "2"]}
|
||||
])
|
||||
|
||||
def test_detector_precision_store_computes_ratio_from_recorded_outcomes(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
|
||||
store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
|
||||
store.record("auth_failure_burst", "confirmed", stream_id="windows", field="auth_failure")
|
||||
store.record("auth_failure_burst", "false_positive", stream_id="windows", field="auth_failure")
|
||||
store.record("new_relationship", "expected", stream_id="firewall", field="relationship:srcip->dstip")
|
||||
|
||||
precision = store.precision(days=30)
|
||||
|
||||
auth = precision["auth_failure_burst"]
|
||||
self.assertEqual(auth["confirmed"], 2)
|
||||
self.assertEqual(auth["false_positive"], 1)
|
||||
self.assertAlmostEqual(auth["precision"], 2 / 3, places=3)
|
||||
self.assertEqual(auth["reviewed_total"], 3)
|
||||
self.assertIsNone(precision["new_relationship"]["precision"])
|
||||
self.assertEqual(precision["new_relationship"]["reviewed_total"], 1)
|
||||
|
||||
def test_detector_precision_store_excludes_outcomes_outside_the_window(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
|
||||
old_timestamp = int(time.time()) - 40 * 86400
|
||||
store.record("auth_failure_burst", "confirmed", recorded_at=old_timestamp)
|
||||
store.record("auth_failure_burst", "false_positive")
|
||||
|
||||
precision = store.precision(days=30)
|
||||
|
||||
self.assertEqual(precision["auth_failure_burst"]["confirmed"], 0)
|
||||
self.assertEqual(precision["auth_failure_burst"]["false_positive"], 1)
|
||||
|
||||
def test_detector_precision_store_ignores_invalid_or_empty_input(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
|
||||
store.record("", "confirmed")
|
||||
store.record("auth_failure_burst", "not_a_real_status")
|
||||
|
||||
self.assertEqual(store.precision(days=30), {})
|
||||
|
||||
def test_detector_precision_store_maintenance_prunes_old_rows(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = DetectorPrecisionStore(str(Path(directory) / "precision.sqlite3"))
|
||||
store.record("auth_failure_burst", "confirmed", recorded_at=int(time.time()) - 200 * 86400)
|
||||
store.record("auth_failure_burst", "confirmed")
|
||||
|
||||
removed = store.maintenance(retention_days=180)
|
||||
|
||||
self.assertEqual(removed, 1)
|
||||
self.assertEqual(store.precision(days=3650)["auth_failure_burst"]["confirmed"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -258,6 +258,43 @@ class MonitorTests(unittest.TestCase):
|
||||
self.assertEqual(pool["group"], "srcip")
|
||||
self.assertEqual(set(pool["streams"]), {"firewall", "iis"})
|
||||
|
||||
def test_detector_health_lists_known_detectors_with_no_data_by_default(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
|
||||
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json"))
|
||||
|
||||
health_by_detector = {item["detector"]: item for item in status["detector_health"]}
|
||||
self.assertIn("auth_failure_burst", health_by_detector)
|
||||
self.assertIn("new_relationship", health_by_detector)
|
||||
self.assertEqual(health_by_detector["auth_failure_burst"]["learning_state"], "no_data")
|
||||
self.assertIsNone(health_by_detector["auth_failure_burst"]["precision"])
|
||||
|
||||
def test_detector_health_reflects_readiness_and_precision(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
config_path = Path(tmp) / "config.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"graylog_streams": [{"id": "windows", "title": "Windows", "enabled": True}],
|
||||
"graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "categorical_fields": ["action"], "detectors": {"auth_failure": {"enabled": True}}}],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
readiness_rows = [{"stream_id": "windows", "detector": "auth_failure", "buckets": 12, "age_days": 10, "training_days": 0, "ready": True}]
|
||||
precision_rows = {"auth_failure_burst": {"confirmed": 3, "false_positive": 1, "expected": 0, "precision": 0.75, "reviewed_total": 4}}
|
||||
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]), \
|
||||
patch("fgai.monitor.BaselineStore.detector_readiness", return_value=readiness_rows), \
|
||||
patch("fgai.monitor.DetectorPrecisionStore.precision", return_value=precision_rows):
|
||||
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
||||
|
||||
row = next(item for item in status["detector_health"] if item["detector"] == "auth_failure_burst")
|
||||
self.assertEqual(row["learning_state"], "ready")
|
||||
self.assertEqual(row["streams_ready"], 1)
|
||||
self.assertEqual(row["streams_configured"], 1)
|
||||
self.assertEqual(row["confirmed"], 3)
|
||||
self.assertEqual(row["precision"], 0.75)
|
||||
|
||||
def test_shared_entity_baselines_empty_for_single_stream(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
config_path = Path(tmp) / "config.json"
|
||||
|
||||
Reference in New Issue
Block a user