Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented cleanup/triage direction. ui, and baselinbe days

Browse Source
This commit is contained in:
larssand
2026-06-29 20:20:00 +02:00
parent d764c5a038
commit d540b8a77d
9 changed files with 298 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/fgai/baseline.py
View File

@@ -18,6 +18,7 @@ from .query_details import event_query_details
DEFAULT_RETENTION_DAYS = 14
DEFAULT_VALUE_RETENTION_DAYS = 7
DEFAULT_MAX_VALUES_PER_FIELD = 2000
DEFAULT_TRAINING_DAYS = 7
MIN_REPORTED_DEVIATION_SCORE = 15
MAX_RARE_VALUES_PER_ENTITY = 5
@@ -50,6 +51,12 @@ def _baseline_confidence(samples: int, *, temporal: bool) -> str:
return "low"
def _age_days(oldest_bucket: int | None, newest_timestamp: int) -> float:
if not oldest_bucket:
return 0.0
return max(0.0, (newest_timestamp - int(oldest_bucket)) / 86400)
def _weighted_score(base: int, profile: object | None, field: str, detector: str) -> tuple[int, float]:
weights = getattr(profile, "field_weights", {}) if profile else {}
if not isinstance(weights, dict):
@@ -254,7 +261,7 @@ class BaselineStore:
on conflict(stream_id, entity, detector, weekday, hour, bucket_start) do update set events=events+excluded.events""", (stream_id, entity, detector, weekday, hour, bucket, count))
return len(pending)
def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object], *, min_training_days: int = 0) -> dict[str, list[dict[str, object]]]:
current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
entity_events: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
detector_current: Counter[tuple[str, str, str]] = Counter()
@@ -291,6 +298,10 @@ class BaselineStore:
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
baseline_age_days = _age_days(oldest, current_timestamp)
if baseline_age_days < min_training_days:
continue
if values[1] == 0:
continue
history = [row[1] / row[0] if row[0] else 0 for row in rows]
@@ -304,7 +315,7 @@ class BaselineStore:
base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
score, weight = _weighted_score(base_score, profile, field, "numeric_baseline")
if score >= MIN_REPORTED_DEVIATION_SCORE:
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
# Event-rate burst is calculated once per stream/entity, rather than once per selected field.
for (stream, entity), matching in entity_events.items():
@@ -324,6 +335,10 @@ class BaselineStore:
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, reference_field)).fetchone()[0]
baseline_age_days = _age_days(oldest, current_timestamp)
if baseline_age_days < min_training_days:
continue
history = [row[0] for row in rows]
current_value = len(matching)
z_score = (current_value - mean(history)) / (pstdev(history) or 1.0)
@@ -334,7 +349,7 @@ class BaselineStore:
score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst")
samples = [_sample_event(event) for event in matching[:5]]
if score >= MIN_REPORTED_DEVIATION_SCORE:
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
for (stream, entity, detector), current_value in detector_current.items():
profile = profiles.get(stream)
@@ -357,6 +372,10 @@ class BaselineStore:
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
oldest = connection.execute("select min(bucket_start) from profile_detector_buckets where stream_id=? and entity=? and detector=?", (stream, entity, detector)).fetchone()[0]
baseline_age_days = _age_days(oldest, current_timestamp)
if baseline_age_days < min_training_days:
continue
history = [row[0] for row in rows]
z_score = (current_value - mean(history)) / (pstdev(history) or 1.0)
if z_score < z_threshold:
@@ -366,7 +385,7 @@ class BaselineStore:
score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst")
samples = [_sample_event(event, detector) for event in matching[:5]]
if score >= MIN_REPORTED_DEVIATION_SCORE:
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
# Detect selected categorical values that have not appeared for this entity in prior data.
rare_counts: Counter[tuple[str, str]] = Counter()
for event in events:
@@ -386,6 +405,10 @@ class BaselineStore:
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 30:
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
baseline_age_days = _age_days(oldest, _event_epoch(event, int(time.time())))
if baseline_age_days < min_training_days:
continue
if rare_counts[(stream, entity)] >= MAX_RARE_VALUES_PER_ENTITY:
continue
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and entity in profile_entities(item, profile) and item.fields.get(field) == value]
@@ -393,7 +416,7 @@ class BaselineStore:
score, weight = _weighted_score(base_score, profile, field, "rare_value")
if score < MIN_REPORTED_DEVIATION_SCORE:
continue
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
rare_counts[(stream, entity)] += 1
@@ -449,14 +472,16 @@ class BaselineStore:
connection.execute("vacuum")
return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0}
def profile_readiness(self, profiles: dict[str, object]) -> list[dict[str, object]]:
def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
with self._connect() as connection:
for stream_id, profile in profiles.items():
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
for field in fields:
count = connection.execute("select count(distinct bucket_start) from profile_buckets where stream_id=? and field=?", (stream_id, str(field).lower())).fetchone()[0]
rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "ready": count >= 12})
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and field=?", (stream_id, str(field).lower())).fetchone()[0]
age_days = _age_days(oldest, int(time.time()))
rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days})
return rows
def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]: