Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Continued the Detection Quality roadmap.

Browse Source
This commit is contained in:
larssand
2026-06-24 21:32:18 +02:00
parent cdef3e1355
commit ea4aaf57ed
14 changed files with 258 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/fgai/baseline.py
View File

@@ -11,6 +11,7 @@ from statistics import mean, pstdev
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
from .entities import profile_entity
from .detectors import DETECTOR_MINIMUMS, event_detector_categories
def _number(value: str | None) -> int:
@@ -78,6 +79,15 @@ class BaselineStore:
events integer not null, numeric_sum real not null, numeric_sum_squares real not null,
primary key (stream_id, entity, field, weekday, hour, bucket_start)
);
create table if not exists profile_detector_buckets (
stream_id text not null, entity text not null, detector text not null, bucket_start integer not null,
events integer not null, primary key (stream_id, entity, detector, bucket_start)
);
create table if not exists profile_detector_temporal_buckets (
stream_id text not null, entity text not null, detector text not null,
weekday integer not null, hour integer not null, bucket_start integer not null,
events integer not null, primary key (stream_id, entity, detector, weekday, hour, bucket_start)
);
"""
)
@@ -124,6 +134,8 @@ class BaselineStore:
observed_at = observed_at or int(time.time())
pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
detector_pending: Counter[tuple[str, str, str, int]] = Counter()
detector_temporal_pending: Counter[tuple[str, str, str, int, int, int]] = Counter()
pending_values: Counter[tuple[str, str, str, str]] = Counter()
with self._connect() as connection:
for event in events:
@@ -140,6 +152,9 @@ class BaselineStore:
timestamp = _event_epoch(event, observed_at)
bucket = timestamp - (timestamp % self.bucket_seconds)
moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
for detector in event_detector_categories(event):
detector_pending[(stream_id, entity, detector, bucket)] += 1
detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in fields:
@@ -165,11 +180,18 @@ class BaselineStore:
for key, count in pending_values.items():
connection.execute("""insert into profile_values values (?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count""", (*key, count))
for (stream_id, entity, detector, bucket), count in detector_pending.items():
connection.execute("""insert into profile_detector_buckets values (?, ?, ?, ?, ?)
on conflict(stream_id, entity, detector, bucket_start) do update set events=events+excluded.events""", (stream_id, entity, detector, bucket, count))
for (stream_id, entity, detector, weekday, hour, bucket), count in detector_temporal_pending.items():
connection.execute("""insert into profile_detector_temporal_buckets values (?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, detector, weekday, hour, bucket_start) do update set events=events+excluded.events""", (stream_id, entity, detector, weekday, hour, bucket, count))
return len(pending)
def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
entity_events: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
detector_current: Counter[tuple[str, str, str]] = Counter()
for event in events:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
@@ -178,6 +200,8 @@ class BaselineStore:
if not entity:
continue
entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
for detector in event_detector_categories(event):
detector_current[(event.fields.get("fgai_stream_id", ""), entity, detector)] += 1
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
@@ -239,6 +263,36 @@ class BaselineStore:
score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": "", "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
for (stream, entity, detector), current_value in detector_current.items():
profile = profiles.get(stream)
settings = getattr(profile, "detectors", {}).get(detector, {}) if profile else {}
if not settings.get("enabled", True):
continue
minimum = int(settings.get("minimum", DETECTOR_MINIMUMS[detector]))
z_threshold = float(settings.get("z_threshold", 3.0))
if current_value < minimum:
continue
matching = [event for event in entity_events[(stream, entity)] if detector in event_detector_categories(event)]
current_timestamp = _event_epoch(matching[-1], int(time.time()))
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
rows = connection.execute("select events from profile_detector_temporal_buckets where stream_id=? and entity=? and detector=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, detector, moment.weekday(), moment.hour)).fetchall()
temporal = True
baseline_scope = "same weekday/hour"
if len(rows) < 12:
rows = connection.execute("select events from profile_detector_buckets where stream_id=? and entity=? and detector=? order by bucket_start desc limit 25", (stream, entity, detector)).fetchall()
temporal = False
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
history = [row[0] for row in rows]
z_score = (current_value - mean(history)) / (pstdev(history) or 1.0)
if z_score < z_threshold:
continue
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score))
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or event.fields.get("query_domain", ""), "action": event.action, "severity": event.severity, "service": event.fields.get("service", event.fields.get("query_type", "")), "value": detector, "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
# Detect selected categorical values that have not appeared for this entity in prior data.
for event in events:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))