Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pool baseline state across streams sharing an entity concept

Browse Source 
Completes the last deferred item from ROADMAP §0: entities recognized as
the same real-world concept across enabled streams (e.g. srcip in both a
firewall and an IIS stream) now share baseline knowledge of entity
presence/age and known relationship-target values, instead of each stream
learning it cold. Implemented additively (synthetic shared_entity:<group>
namespace in the existing baseline.py tables, no schema migration), scoped
to entity presence + relationship known-values only — per-stream
distribution baselines stay untouched. Also fixes a latent alias-map
collision bug found while building this (ALIAS_CANONICAL_BY_FIELD let
"context" silently steal "hostname"/"dns_query" from their own canonical
groups).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-23 18:50:14 +02:00
parent 8c155ec9cf
commit 83ffef3570
12 changed files with 305 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
tests/test_monitor.py
View File

@@ -232,6 +232,49 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile")
self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile")
def test_shared_entity_baselines_lists_streams_pooled_by_entity_concept(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"
config_path.write_text(
json.dumps(
{
"graylog_streams": [
{"id": "firewall", "title": "Firewall", "enabled": True},
{"id": "iis", "title": "IIS", "enabled": True},
],
"graylog_stream_profiles": [
{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]},
{"stream_id": "iis", "name": "IIS profile", "entity_field": "client_ip", "categorical_fields": ["action"]},
],
}
),
encoding="utf-8",
)
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
self.assertEqual(len(status["shared_entity_baselines"]), 1)
pool = status["shared_entity_baselines"][0]
self.assertEqual(pool["group"], "srcip")
self.assertEqual(set(pool["streams"]), {"firewall", "iis"})
def test_shared_entity_baselines_empty_for_single_stream(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"
config_path.write_text(
json.dumps(
{
"graylog_streams": [{"id": "firewall", "title": "Firewall", "enabled": True}],
"graylog_stream_profiles": [{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}],
}
),
encoding="utf-8",
)
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
self.assertEqual(status["shared_entity_baselines"], [])
def test_stream_coverage_marks_missing_profiles(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"