Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ny central normalisering

Browse Source
This commit is contained in:
larssand
2026-06-29 18:55:18 +02:00
parent 0b31e6c55e
commit 20b0e0a92e
13 changed files with 146 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/fgai/baseline.py
View File

@@ -12,6 +12,7 @@ from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
from .entities import profile_entity
from .detectors import DETECTOR_MINIMUMS, event_detector_categories
from .normalization import canonical_value
def _number(value: str | None) -> int:
@@ -62,6 +63,19 @@ def _weighted_score(base: int, profile: object | None, field: str, detector: str
return min(100, max(0, int(round(base * multiplier)))), round(multiplier, 2)
def _sample_event(event: LogEvent, value: str = "") -> dict[str, str]:
return {
"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
"source": event.src_ip or event.fields.get("source", ""),
"destination": event.dst_ip or canonical_value(event.fields, "context"),
"action": event.action,
"severity": event.severity,
"service": canonical_value(event.fields, "service"),
"value": value,
"message": canonical_value(event.fields, "context")[:240],
}
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -252,7 +266,7 @@ class BaselineStore:
deviation = abs(current_value - mean(history))
if deviation > (pstdev(history) or 1.0) * 3:
sample_values = sorted({event.fields.get(field, "") for event in matching})[:5]
evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
evidence_events = [_sample_event(event, event.fields.get(field, "")) for event in matching[:5]]
confidence = _baseline_confidence(len(rows), temporal=temporal)
base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
score, weight = _weighted_score(base_score, profile, field, "numeric_baseline")
@@ -284,7 +298,7 @@ class BaselineStore:
confidence = _baseline_confidence(len(rows), temporal=temporal)
base_score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst")
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": "", "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
samples = [_sample_event(event) for event in matching[:5]]
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
for (stream, entity, detector), current_value in detector_current.items():
@@ -315,7 +329,7 @@ class BaselineStore:
confidence = _baseline_confidence(len(rows), temporal=temporal)
base_score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score))
score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst")
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or event.fields.get("query_domain", ""), "action": event.action, "severity": event.severity, "service": event.fields.get("service", event.fields.get("query_type", "")), "value": detector, "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
samples = [_sample_event(event, detector) for event in matching[:5]]
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
# Detect selected categorical values that have not appeared for this entity in prior data.
for event in events:
@@ -336,7 +350,7 @@ class BaselineStore:
if known is None and known_total >= 30:
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
score, weight = _weighted_score(12, profile, field, "rare_value")
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
return output