Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix retention

Browse Source
This commit is contained in:
larssand
2026-07-06 12:41:34 +02:00
parent 6a0df3bc74
commit 841f94dc9f
6 changed files with 159 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
README.md
View File

@@ -458,16 +458,19 @@ five-minute windows. Historical rate and hitcount-rate deviations then contribut
to its anomaly score. Set `FGAI_BASELINE_DB` to use another location.
SignalScope prunes old baseline buckets during each monitor cycle. The defaults
keep 14 days of buckets and dedupe history, prune stale one-off categorical
values after 7 days, and cap high-cardinality values per stream/entity/field.
keep 7 days of buckets and dedupe history, prune stale one-off categorical
values after 3 days, and cap high-cardinality values per stream/entity/field.
Very noisy fields such as raw messages, URLs, payloads, request/response bodies,
tokens, sessions, hashes and long values are still counted in bucket baselines
but are not stored as distinct rare-value candidates.
Tune these in the dashboard or in `state/fgai-config.json`:
```json
{
"baseline_training_days": 7,
"baseline_retention_days": 14,
"baseline_value_retention_days": 7,
"baseline_max_values_per_field": 2000
"baseline_retention_days": 7,
"baseline_value_retention_days": 3,
"baseline_max_values_per_field": 500
}
```
@@ -480,11 +483,12 @@ If an existing baseline database has already grown large, stop the monitor and
run a manual prune plus SQLite compaction:
```bash
signalscope baseline-maintenance --baseline-db state/fgai-baseline.sqlite3 --retention-days 14 --value-retention-days 7 --max-values-per-field 2000 --vacuum
signalscope baseline-maintenance --baseline-db state/fgai-baseline.sqlite3 --retention-days 7 --value-retention-days 3 --max-values-per-field 500 --vacuum
```
`VACUUM` can take time on a large database and should not be run while the
monitor is actively writing.
`VACUUM` can take time on a large database, needs free disk space roughly equal
to the database size, and should not be run while the monitor is actively
writing. Without `--vacuum`, SQLite may delete rows but keep the file size.
Analyze local logs:

99
src/fgai/baseline.py
View File

@@ -15,12 +15,37 @@ from .detectors import DETECTOR_MINIMUMS, event_detector_categories
from .normalization import canonical_value
from .query_details import event_query_details
DEFAULT_RETENTION_DAYS = 14
DEFAULT_VALUE_RETENTION_DAYS = 7
DEFAULT_MAX_VALUES_PER_FIELD = 2000
DEFAULT_RETENTION_DAYS = 7
DEFAULT_VALUE_RETENTION_DAYS = 3
DEFAULT_MAX_VALUES_PER_FIELD = 500
DEFAULT_TRAINING_DAYS = 7
MIN_REPORTED_DEVIATION_SCORE = 15
MAX_RARE_VALUES_PER_ENTITY = 5
MAX_PROFILE_VALUE_LENGTH = 160
NOISY_VALUE_FIELD_PARTS = (
"answer",
"body",
"commandline",
"command_line",
"context",
"cookie",
"ephemeral",
"fingerprint",
"full_message",
"hash",
"message",
"payload",
"queryparameter",
"raw",
"request",
"response",
"session",
"stack",
"token",
"uri",
"url",
"user_agent",
)
def _number(value: str | None) -> int:
@@ -97,6 +122,17 @@ def _relationship_key(left: str, right: str) -> str:
return f"relationship:{left.lower()}->{right.lower()}"
def _should_store_profile_value(field: str, value: str) -> bool:
field = field.lower()
if field.startswith("relationship:"):
return True
if len(value) > MAX_PROFILE_VALUE_LENGTH:
return False
if any(part in field for part in NOISY_VALUE_FIELD_PARTS):
return False
return True
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -206,7 +242,14 @@ class BaselineStore:
)
return inserted
def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int:
def ingest_profile_fields(
self,
events: list[LogEvent],
profiles: dict[str, object],
*,
observed_at: int | None = None,
max_values_per_field: int = DEFAULT_MAX_VALUES_PER_FIELD,
) -> int:
observed_at = observed_at or int(time.time())
pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
@@ -253,7 +296,7 @@ class BaselineStore:
temporal_pending[temporal][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
if raw_value and _should_store_profile_value(key[2], raw_value):
pending_values[(*key[:3], raw_value)] += 1
for (stream_id, entity, field, bucket), values in pending.items():
connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
@@ -261,7 +304,23 @@ class BaselineStore:
for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items():
connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values))
value_counts: dict[tuple[str, str, str], int] = {}
for key, count in pending_values.items():
if max_values_per_field > 0:
group_key = key[:3]
known = connection.execute(
"select 1 from profile_values where stream_id=? and entity=? and field=? and value=?",
key,
).fetchone()
if known is None:
if group_key not in value_counts:
value_counts[group_key] = int(connection.execute(
"select count(*) from profile_values where stream_id=? and entity=? and field=?",
group_key,
).fetchone()[0])
if value_counts[group_key] >= max_values_per_field:
continue
value_counts[group_key] += 1
connection.execute("""insert into profile_values values (?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count,last_seen=excluded.last_seen""", (*key, count, observed_at))
for (stream_id, entity, detector, bucket), count in detector_pending.items():
@@ -538,7 +597,35 @@ class BaselineStore:
if vacuum:
with self._connect() as connection:
connection.execute("vacuum")
return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0}
stats = self.stats()
return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, **stats}
def stats(self) -> dict[str, object]:
with self._connect() as connection:
tables = (
"seen_events",
"source_buckets",
"source_values",
"profile_seen_events",
"profile_buckets",
"profile_temporal_buckets",
"profile_detector_buckets",
"profile_detector_temporal_buckets",
"profile_values",
)
rows = {table: int(connection.execute(f"select count(*) from {table}").fetchone()[0]) for table in tables}
page_size = int(connection.execute("pragma page_size").fetchone()[0])
page_count = int(connection.execute("pragma page_count").fetchone()[0])
freelist_count = int(connection.execute("pragma freelist_count").fetchone()[0])
size_bytes = self.path.stat().st_size if self.path.exists() else 0
return {
"size_bytes": size_bytes,
"page_size": page_size,
"page_count": page_count,
"freelist_count": freelist_count,
"reclaimable_bytes": freelist_count * page_size,
"rows": rows,
}
def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []

6
src/fgai/config.py
View File

@@ -19,9 +19,9 @@ DEFAULT_CONFIG: dict[str, object] = {
"graylog_raw_sample_events": 5000,
"graylog_mcp_call_timeout_seconds": 8,
"graylog_mcp_poll_timeout_seconds": 120,
"baseline_retention_days": 14,
"baseline_value_retention_days": 7,
"baseline_max_values_per_field": 2000,
"baseline_retention_days": 7,
"baseline_value_retention_days": 3,
"baseline_max_values_per_field": 500,
"baseline_training_days": 7,
"graylog_field_mapping": "",
"llm_enabled": False,

10
src/fgai/dashboard.py
View File

File diff suppressed because one or more lines are too long

15
src/fgai/monitor.py
View File

@@ -392,12 +392,19 @@ def build_status(
deviation["score"] = 0
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
baseline_events = baseline.ingest(events) if baseline else 0
profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
profile_baseline_fields = (
baseline.ingest_profile_fields(
events,
stream_profiles,
max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500),
)
if baseline else 0
)
baseline_maintenance = (
baseline.maintenance(
retention_days=int(runtime_values.get("baseline_retention_days", 14) or 14),
value_retention_days=int(runtime_values.get("baseline_value_retention_days", 7) or 7),
max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 2000) or 2000),
retention_days=int(runtime_values.get("baseline_retention_days", 7) or 7),
value_retention_days=int(runtime_values.get("baseline_value_retention_days", 3) or 3),
max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500),
)
if baseline
else {}

35
tests/test_baseline.py
View File

@@ -109,6 +109,41 @@ class BaselineTests(unittest.TestCase):
self.assertGreaterEqual(result["deleted"]["profile_buckets"], 1)
self.assertGreaterEqual(result["deleted"]["seen_events"], 1)
self.assertGreaterEqual(result["deleted"]["profile_seen_events"], 1)
self.assertIn("rows", result)
self.assertIn("reclaimable_bytes", result)
def test_profile_value_storage_skips_noisy_high_cardinality_fields(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "web", "entity_field": "srcip", "categorical_fields": ["url", "action"]}])
events = [
parse_log_line(f"fgai_stream_id=web srcip=10.0.0.5 action=allow url=/download/{index}/very/noisy/path event={index}")
for index in range(10)
]
store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000)
stats = store.stats()
self.assertEqual(stats["rows"]["profile_buckets"], 2)
with store._connect() as connection:
fields = {row[0] for row in connection.execute("select distinct field from profile_values").fetchall()}
self.assertIn("action", fields)
self.assertNotIn("url", fields)
def test_profile_value_storage_caps_new_values_during_ingest(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}])
events = [
parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=value-{index}.example event={index}")
for index in range(5)
]
store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000, max_values_per_field=2)
with store._connect() as connection:
stored = connection.execute("select count(*) from profile_values").fetchone()[0]
self.assertEqual(stored, 2)
def test_rare_values_are_limited_per_entity(self):
with tempfile.TemporaryDirectory() as directory: