Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configurable per-field detector weights är klart.

Browse Source
This commit is contained in:
larssand
2026-06-25 19:36:08 +02:00
parent 899dbdfc84
commit 9da1aedf52
9 changed files with 106 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/fgai/baseline.py
View File

@@ -42,6 +42,26 @@ def _baseline_confidence(samples: int, *, temporal: bool) -> str:
return "low"
def _weighted_score(base: int, profile: object | None, field: str, detector: str) -> tuple[int, float]:
weights = getattr(profile, "field_weights", {}) if profile else {}
if not isinstance(weights, dict):
return base, 1.0
field_key = str(field).lower()
detector_key = str(detector).lower()
multiplier = 1.0
for key in (field_key, detector_key):
value = weights.get(key)
if isinstance(value, (int, float)):
multiplier *= float(value)
nested = weights.get(field_key)
if isinstance(nested, dict):
value = nested.get(detector_key)
if isinstance(value, (int, float)):
multiplier *= float(value)
multiplier = max(0.0, min(5.0, multiplier))
return min(100, max(0, int(round(base * multiplier)))), round(multiplier, 2)
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -211,6 +231,7 @@ class BaselineStore:
output: dict[str, list[dict[str, object]]] = defaultdict(list)
with self._connect() as connection:
for (stream, entity, field), values in current.items():
profile = profiles.get(stream)
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
@@ -233,8 +254,9 @@ class BaselineStore:
sample_values = sorted({event.fields.get(field, "") for event in matching})[:5]
evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
score, weight = _weighted_score(base_score, profile, field, "numeric_baseline")
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
# Event-rate burst is calculated once per stream/entity, rather than once per selected field.
for (stream, entity), matching in entity_events.items():
@@ -260,9 +282,10 @@ class BaselineStore:
if z_score < 3:
continue
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
base_score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst")
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": "", "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
for (stream, entity, detector), current_value in detector_current.items():
profile = profiles.get(stream)
@@ -290,9 +313,10 @@ class BaselineStore:
if z_score < z_threshold:
continue
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score))
base_score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score))
score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst")
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or event.fields.get("query_domain", ""), "action": event.action, "severity": event.severity, "service": event.fields.get("service", event.fields.get("query_type", "")), "value": detector, "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
# Detect selected categorical values that have not appeared for this entity in prior data.
for event in events:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
@@ -311,7 +335,8 @@ class BaselineStore:
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 30:
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": 12, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
score, weight = _weighted_score(12, profile, field, "rare_value")
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
return output