Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix retention

Browse Source
This commit is contained in:
larssand
2026-07-06 12:41:34 +02:00
parent 6a0df3bc74
commit 841f94dc9f
6 changed files with 159 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/fgai/baseline.py
View File

@@ -15,12 +15,37 @@ from .detectors import DETECTOR_MINIMUMS, event_detector_categories
from .normalization import canonical_value
from .query_details import event_query_details
DEFAULT_RETENTION_DAYS = 14
DEFAULT_VALUE_RETENTION_DAYS = 7
DEFAULT_MAX_VALUES_PER_FIELD = 2000
DEFAULT_RETENTION_DAYS = 7
DEFAULT_VALUE_RETENTION_DAYS = 3
DEFAULT_MAX_VALUES_PER_FIELD = 500
DEFAULT_TRAINING_DAYS = 7
MIN_REPORTED_DEVIATION_SCORE = 15
MAX_RARE_VALUES_PER_ENTITY = 5
MAX_PROFILE_VALUE_LENGTH = 160
NOISY_VALUE_FIELD_PARTS = (
"answer",
"body",
"commandline",
"command_line",
"context",
"cookie",
"ephemeral",
"fingerprint",
"full_message",
"hash",
"message",
"payload",
"queryparameter",
"raw",
"request",
"response",
"session",
"stack",
"token",
"uri",
"url",
"user_agent",
)
def _number(value: str | None) -> int:
@@ -97,6 +122,17 @@ def _relationship_key(left: str, right: str) -> str:
return f"relationship:{left.lower()}->{right.lower()}"
def _should_store_profile_value(field: str, value: str) -> bool:
field = field.lower()
if field.startswith("relationship:"):
return True
if len(value) > MAX_PROFILE_VALUE_LENGTH:
return False
if any(part in field for part in NOISY_VALUE_FIELD_PARTS):
return False
return True
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -206,7 +242,14 @@ class BaselineStore:
)
return inserted
def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int:
def ingest_profile_fields(
self,
events: list[LogEvent],
profiles: dict[str, object],
*,
observed_at: int | None = None,
max_values_per_field: int = DEFAULT_MAX_VALUES_PER_FIELD,
) -> int:
observed_at = observed_at or int(time.time())
pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
@@ -253,7 +296,7 @@ class BaselineStore:
temporal_pending[temporal][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
if raw_value and _should_store_profile_value(key[2], raw_value):
pending_values[(*key[:3], raw_value)] += 1
for (stream_id, entity, field, bucket), values in pending.items():
connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
@@ -261,7 +304,23 @@ class BaselineStore:
for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items():
connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values))
value_counts: dict[tuple[str, str, str], int] = {}
for key, count in pending_values.items():
if max_values_per_field > 0:
group_key = key[:3]
known = connection.execute(
"select 1 from profile_values where stream_id=? and entity=? and field=? and value=?",
key,
).fetchone()
if known is None:
if group_key not in value_counts:
value_counts[group_key] = int(connection.execute(
"select count(*) from profile_values where stream_id=? and entity=? and field=?",
group_key,
).fetchone()[0])
if value_counts[group_key] >= max_values_per_field:
continue
value_counts[group_key] += 1
connection.execute("""insert into profile_values values (?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count,last_seen=excluded.last_seen""", (*key, count, observed_at))
for (stream_id, entity, detector, bucket), count in detector_pending.items():
@@ -538,7 +597,35 @@ class BaselineStore:
if vacuum:
with self._connect() as connection:
connection.execute("vacuum")
return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0}
stats = self.stats()
return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, **stats}
def stats(self) -> dict[str, object]:
with self._connect() as connection:
tables = (
"seen_events",
"source_buckets",
"source_values",
"profile_seen_events",
"profile_buckets",
"profile_temporal_buckets",
"profile_detector_buckets",
"profile_detector_temporal_buckets",
"profile_values",
)
rows = {table: int(connection.execute(f"select count(*) from {table}").fetchone()[0]) for table in tables}
page_size = int(connection.execute("pragma page_size").fetchone()[0])
page_count = int(connection.execute("pragma page_count").fetchone()[0])
freelist_count = int(connection.execute("pragma freelist_count").fetchone()[0])
size_bytes = self.path.stat().st_size if self.path.exists() else 0
return {
"size_bytes": size_bytes,
"page_size": page_size,
"page_count": page_count,
"freelist_count": freelist_count,
"reclaimable_bytes": freelist_count * page_size,
"rows": rows,
}
def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []