Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the next multi-source detection layer in this repository.

Browse Source
This commit is contained in:
larssand
2026-06-24 19:16:16 +02:00
parent f6bee0438c
commit 868022008a
15 changed files with 281 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
src/fgai/baseline.py
View File

@@ -4,11 +4,13 @@ import hashlib
import sqlite3
import time
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from statistics import mean, pstdev
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
from .entities import profile_entity
def _number(value: str | None) -> int:
@@ -18,6 +20,19 @@ def _number(value: str | None) -> int:
return 0
def _event_epoch(event: LogEvent, fallback: int) -> int:
value = event.fields.get("eventtime", event.fields.get("timestamp", ""))
if not value:
return fallback
try:
return int(float(value))
except ValueError:
try:
return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
except ValueError:
return fallback
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -48,6 +63,13 @@ class BaselineStore:
stream_id text not null, entity text not null, field text not null, value text not null,
seen_count integer not null, primary key (stream_id, entity, field, value)
);
create table if not exists profile_seen_events (fingerprint text primary key);
create table if not exists profile_temporal_buckets (
stream_id text not null, entity text not null, field text not null,
weekday integer not null, hour integer not null, bucket_start integer not null,
events integer not null, numeric_sum real not null, numeric_sum_squares real not null,
primary key (stream_id, entity, field, weekday, hour, bucket_start)
);
"""
)
@@ -92,34 +114,46 @@ class BaselineStore:
def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int:
observed_at = observed_at or int(time.time())
bucket = observed_at - (observed_at % self.bucket_seconds)
pending: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
pending_values: Counter[tuple[str, str, str, str]] = Counter()
for event in events:
stream_id = event.fields.get("fgai_stream_id", "")
profile = profiles.get(stream_id)
if not profile:
continue
entity_field = str(getattr(profile, "entity_field", "")).lower()
entity = event.fields.get(entity_field)
if not entity:
continue
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in fields:
key = (stream_id, entity, str(field).lower())
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
pending[key][0] += 1
pending[key][1] += value
pending[key][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
pending_values[(*key, raw_value)] += 1
with self._connect() as connection:
for (stream_id, entity, field), values in pending.items():
for event in events:
stream_id = event.fields.get("fgai_stream_id", "")
profile = profiles.get(stream_id)
if not profile:
continue
fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1:
continue
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
continue
timestamp = _event_epoch(event, observed_at)
bucket = timestamp - (timestamp % self.bucket_seconds)
moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in fields:
key = (stream_id, entity, str(field).lower(), bucket)
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
pending[key][0] += 1
pending[key][1] += value
pending[key][2] += value * value
temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
temporal_pending[temporal][0] += 1
temporal_pending[temporal][1] += value
temporal_pending[temporal][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
pending_values[(*key[:3], raw_value)] += 1
for (stream_id, entity, field, bucket), values in pending.items():
connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items():
connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values))
for key, count in pending_values.items():
connection.execute("""insert into profile_values values (?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count""", (*key, count))
@@ -131,7 +165,7 @@ class BaselineStore:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
continue
entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
continue
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
@@ -143,20 +177,26 @@ class BaselineStore:
output: dict[str, list[dict[str, object]]] = defaultdict(list)
with self._connect() as connection:
for (stream, entity, field), values in current.items():
rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
baseline_scope = "same weekday/hour"
if len(rows) < 12:
rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
if values[1] == 0:
history = [row[0] for row in rows]
current_value = values[0]
reason = f"{field} event rate deviates from its stream baseline"
reason = f"{field} event rate deviates from its {baseline_scope} baseline"
else:
history = [row[1] / row[0] if row[0] else 0 for row in rows]
current_value = values[1] / values[0] if values[0] else 0
reason = f"{field} value deviates from its stream baseline"
reason = f"{field} value deviates from its {baseline_scope} baseline"
deviation = abs(current_value - mean(history))
if deviation > (pstdev(history) or 1.0) * 3:
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and event.fields.get(str(getattr(profiles.get(stream), "entity_field", "")).lower()) == entity and event.fields.get(field)]
samples = sorted({event.fields.get(field, "") for event in matching})[:5]
evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": samples, "sample_events": evidence_events})
@@ -165,7 +205,7 @@ class BaselineStore:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
continue
entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
continue
stream = event.fields.get("fgai_stream_id", "")
@@ -177,7 +217,8 @@ class BaselineStore:
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
known_total = connection.execute("select count(*) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 10:
evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value}
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
return output