Continued the Detection Quality roadmap milestone.
This commit is contained in:
larssand
@@ -22,9 +22,11 @@ operational value and dependency, not by UI appeal.
|
||||
|
||||
Goal: make findings more accurate before adding more integrations.
|
||||
|
||||
- [ ] Add baseline confidence based on sample count, profile age, and time-bucket coverage.
|
||||
- [ ] Add burst detection for event rate, authentication failures, DNS volume, and denied traffic.
|
||||
- [ ] Add rare-value detection with frequency thresholds and configurable field weights.
|
||||
- [x] Add baseline confidence based on sample count and time-bucket coverage.
|
||||
- [x] Add generic event-rate burst detection per stream/entity.
|
||||
- [x] Add rare-value detection with a minimum historical observation threshold.
|
||||
- [ ] Add detector-specific authentication failure, DNS volume, and denied-traffic burst thresholds.
|
||||
- [ ] Add configurable per-field detector weights.
|
||||
- [ ] Add sequence detection, for example DNS lookup -> outbound connection -> authentication event.
|
||||
- [ ] Add per-stream detector enablement and thresholds in the UI.
|
||||
- [ ] Add a dry-run replay command to evaluate detector changes against a selected historic Graylog time range.
|
||||
|
||||
@@ -33,6 +33,14 @@ def _event_epoch(event: LogEvent, fallback: int) -> int:
|
||||
return fallback
|
||||
|
||||
|
||||
def _baseline_confidence(samples: int, *, temporal: bool) -> str:
|
||||
if samples >= 24 and temporal:
|
||||
return "high"
|
||||
if samples >= 18:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
class BaselineStore:
|
||||
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
|
||||
|
||||
@@ -161,6 +169,7 @@ class BaselineStore:
|
||||
|
||||
def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
|
||||
current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
|
||||
entity_events: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
|
||||
for event in events:
|
||||
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
|
||||
if not profile:
|
||||
@@ -168,6 +177,7 @@ class BaselineStore:
|
||||
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
|
||||
if not entity:
|
||||
continue
|
||||
entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
|
||||
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
|
||||
for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
|
||||
key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
|
||||
@@ -181,25 +191,54 @@ class BaselineStore:
|
||||
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
|
||||
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
|
||||
rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
|
||||
temporal = True
|
||||
baseline_scope = "same weekday/hour"
|
||||
if len(rows) < 12:
|
||||
rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
|
||||
temporal = False
|
||||
baseline_scope = "all observed periods"
|
||||
if len(rows) < 12:
|
||||
continue
|
||||
if values[1] == 0:
|
||||
history = [row[0] for row in rows]
|
||||
current_value = values[0]
|
||||
reason = f"{field} event rate deviates from its {baseline_scope} baseline"
|
||||
else:
|
||||
history = [row[1] / row[0] if row[0] else 0 for row in rows]
|
||||
current_value = values[1] / values[0] if values[0] else 0
|
||||
reason = f"{field} value deviates from its {baseline_scope} baseline"
|
||||
continue
|
||||
history = [row[1] / row[0] if row[0] else 0 for row in rows]
|
||||
current_value = values[1] / values[0] if values[0] else 0
|
||||
reason = f"{field} value deviates from its {baseline_scope} baseline"
|
||||
deviation = abs(current_value - mean(history))
|
||||
if deviation > (pstdev(history) or 1.0) * 3:
|
||||
samples = sorted({event.fields.get(field, "") for event in matching})[:5]
|
||||
sample_values = sorted({event.fields.get(field, "") for event in matching})[:5]
|
||||
evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
|
||||
output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": samples, "sample_events": evidence_events})
|
||||
confidence = _baseline_confidence(len(rows), temporal=temporal)
|
||||
score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
|
||||
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
|
||||
|
||||
# Event-rate burst is calculated once per stream/entity, rather than once per selected field.
|
||||
for (stream, entity), matching in entity_events.items():
|
||||
profile = profiles.get(stream)
|
||||
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())] if profile else []
|
||||
if not fields:
|
||||
continue
|
||||
reference_field = str(fields[0]).lower()
|
||||
current_timestamp = _event_epoch(matching[-1], int(time.time()))
|
||||
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
|
||||
rows = connection.execute("select events from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, reference_field, moment.weekday(), moment.hour)).fetchall()
|
||||
temporal = True
|
||||
baseline_scope = "same weekday/hour"
|
||||
if len(rows) < 12:
|
||||
rows = connection.execute("select events from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, reference_field)).fetchall()
|
||||
temporal = False
|
||||
baseline_scope = "all observed periods"
|
||||
if len(rows) < 12:
|
||||
continue
|
||||
history = [row[0] for row in rows]
|
||||
current_value = len(matching)
|
||||
z_score = (current_value - mean(history)) / (pstdev(history) or 1.0)
|
||||
if z_score < 3:
|
||||
continue
|
||||
confidence = _baseline_confidence(len(rows), temporal=temporal)
|
||||
score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
|
||||
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": "", "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
|
||||
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
|
||||
# Detect selected categorical values that have not appeared for this entity in prior data.
|
||||
for event in events:
|
||||
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
|
||||
@@ -215,10 +254,10 @@ class BaselineStore:
|
||||
if not value:
|
||||
continue
|
||||
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
|
||||
known_total = connection.execute("select count(*) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
|
||||
if known is None and known_total >= 10:
|
||||
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
|
||||
if known is None and known_total >= 30:
|
||||
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
|
||||
evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
|
||||
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": 12, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
|
||||
if evidence not in output[entity]:
|
||||
output[entity].append(evidence)
|
||||
return output
|
||||
|
||||
@@ -179,7 +179,7 @@ async function refresh() {
|
||||
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
|
||||
const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item})));
|
||||
document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [
|
||||
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
|
||||
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Confidence', key:'confidence'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; samples ${r.baseline_samples ?? '-'}; scope ${r.baseline_scope ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
|
||||
], 'field-deviations');
|
||||
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
|
||||
const note = prompt('Review note (optional):') || '';
|
||||
@@ -210,12 +210,13 @@ async function refresh() {
|
||||
const d = data.diagnostics || {};
|
||||
const context = data.event_context || {};
|
||||
const quality = data.data_quality || {};
|
||||
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id}));
|
||||
const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id]));
|
||||
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: profileNames[item.stream_id] || item.stream_id, stream_title: streamTitles[item.stream_id] || item.stream_id}));
|
||||
const correlations = data.cross_source_correlations || [];
|
||||
document.getElementById('diagnostics').innerHTML =
|
||||
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
|
||||
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +
|
||||
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
|
||||
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
|
||||
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
|
||||
'<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
|
||||
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
|
||||
|
||||
@@ -27,3 +27,21 @@ class BaselineTests(unittest.TestCase):
|
||||
]
|
||||
self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000), 1)
|
||||
self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_300), 0)
|
||||
|
||||
def test_profile_rate_burst_is_one_confident_detector(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
|
||||
profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}])
|
||||
for index in range(12):
|
||||
event = parse_log_line(f"fgai_stream_id=windows username=alice action=login baseline={index}")
|
||||
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
|
||||
|
||||
burst = [
|
||||
parse_log_line(f"fgai_stream_id=windows username=alice action=login burst={index}")
|
||||
for index in range(20)
|
||||
]
|
||||
deviations = store.profile_deviations(burst, profiles)["alice"]
|
||||
|
||||
rate = next(item for item in deviations if item["detector"] == "event_rate_burst")
|
||||
self.assertEqual(rate["field"], "event_rate")
|
||||
self.assertEqual(rate["baseline_samples"], 12)
|
||||
|
||||
Reference in New Issue
Block a user