Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix retention

Browse Source
This commit is contained in:
larssand
2026-07-06 12:41:34 +02:00
parent 6a0df3bc74
commit 841f94dc9f
6 changed files with 159 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
tests/test_baseline.py
View File

@@ -109,6 +109,41 @@ class BaselineTests(unittest.TestCase):
self.assertGreaterEqual(result["deleted"]["profile_buckets"], 1)
self.assertGreaterEqual(result["deleted"]["seen_events"], 1)
self.assertGreaterEqual(result["deleted"]["profile_seen_events"], 1)
self.assertIn("rows", result)
self.assertIn("reclaimable_bytes", result)
def test_profile_value_storage_skips_noisy_high_cardinality_fields(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "web", "entity_field": "srcip", "categorical_fields": ["url", "action"]}])
events = [
parse_log_line(f"fgai_stream_id=web srcip=10.0.0.5 action=allow url=/download/{index}/very/noisy/path event={index}")
for index in range(10)
]
store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000)
stats = store.stats()
self.assertEqual(stats["rows"]["profile_buckets"], 2)
with store._connect() as connection:
fields = {row[0] for row in connection.execute("select distinct field from profile_values").fetchall()}
self.assertIn("action", fields)
self.assertNotIn("url", fields)
def test_profile_value_storage_caps_new_values_during_ingest(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}])
events = [
parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=value-{index}.example event={index}")
for index in range(5)
]
store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000, max_values_per_field=2)
with store._connect() as connection:
stored = connection.execute("select count(*) from profile_values").fetchone()[0]
self.assertEqual(stored, 2)
def test_rare_values_are_limited_per_entity(self):
with tempfile.TemporaryDirectory() as directory: