Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pool baseline state across streams sharing an entity concept

Browse Source 
Completes the last deferred item from ROADMAP §0: entities recognized as
the same real-world concept across enabled streams (e.g. srcip in both a
firewall and an IIS stream) now share baseline knowledge of entity
presence/age and known relationship-target values, instead of each stream
learning it cold. Implemented additively (synthetic shared_entity:<group>
namespace in the existing baseline.py tables, no schema migration), scoped
to entity presence + relationship known-values only — per-stream
distribution baselines stay untouched. Also fixes a latent alias-map
collision bug found while building this (ALIAS_CANONICAL_BY_FIELD let
"context" silently steal "hostname"/"dns_query" from their own canonical
groups).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-23 18:50:14 +02:00
parent 8c155ec9cf
commit 83ffef3570
12 changed files with 305 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
tests/test_baseline.py
View File

@@ -183,6 +183,57 @@ class BaselineTests(unittest.TestCase):
self.assertNotIn("alice", store.profile_deviations(burst, profiles, min_training_days=7))
self.assertIn("alice", store.profile_deviations(burst, profiles, min_training_days=0))
def test_entity_pools_share_known_relationship_values_across_streams(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([
{
"stream_id": "firewall",
"entity_field": "srcip",
"entity_fields": ["srcip"],
"categorical_fields": ["action"],
"relationship_fields": [{"left": "srcip", "right": "dstip", "name": "src to dst"}],
},
{
"stream_id": "iis",
"entity_field": "client_ip",
"entity_fields": ["client_ip"],
"categorical_fields": ["action"],
"relationship_fields": [{"left": "client_ip", "right": "dstip", "name": "client to dst"}],
},
])
entity_pools = {
"firewall": {"srcip": "shared_entity:srcip"},
"iis": {"client_ip": "shared_entity:srcip"},
}
for index in range(12):
# firewall establishes pooled knowledge of srcip=10.0.0.10 -> dstip=8.8.8.8
fw_event = parse_log_line(f"fgai_stream_id=firewall srcip=10.0.0.10 dstip=8.8.8.8 action=allow baseline={index}")
store.ingest_profile_fields([fw_event], profiles, observed_at=1_700_000_000 + index * 300, entity_pools=entity_pools)
# iis independently establishes its own (unpooled) history for a different destination
iis_history_event = parse_log_line(f"fgai_stream_id=iis client_ip=10.0.0.10 dstip=9.9.9.9 action=allow baseline={index}")
store.ingest_profile_fields([iis_history_event], profiles, observed_at=1_700_000_000 + index * 300)
iis_event = [parse_log_line("fgai_stream_id=iis client_ip=10.0.0.10 dstip=8.8.8.8 action=allow current=1")]
# Pooled: dstip=8.8.8.8 is already known via the firewall stream's contribution to the shared pool.
pooled = store.profile_deviations(iis_event, profiles, entity_pools=entity_pools)
self.assertNotIn("new_relationship", [item["detector"] for item in pooled.get("10.0.0.10", [])])
# Unpooled: iis's own history only knows dstip=9.9.9.9, so 8.8.8.8 is still reported new — proves
# the feature is additive and today's per-stream behavior is unchanged without entity_pools.
unpooled = store.profile_deviations(iis_event, profiles)
relation = next(item for item in unpooled["10.0.0.10"] if item["detector"] == "new_relationship")
self.assertEqual(relation["field"], "relationship:client_ip->dstip")
self.assertEqual(relation["value"], "8.8.8.8")
# Genuinely new everywhere: pooling never suppresses a value nobody has ever seen.
genuinely_new = [parse_log_line("fgai_stream_id=iis client_ip=10.0.0.10 dstip=1.2.3.4 current=1")]
still_new = store.profile_deviations(genuinely_new, profiles, entity_pools=entity_pools)
relation = next(item for item in still_new["10.0.0.10"] if item["detector"] == "new_relationship")
self.assertEqual(relation["value"], "1.2.3.4")
def test_custom_relationship_flags_new_user_source_ip(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))

5
tests/test_dashboard.py
View File

@@ -99,6 +99,11 @@ class DashboardTests(unittest.TestCase):
self.assertIn("-entity pattern", HTML)
self.assertIn("cluster_dampened", HTML)
def test_dashboard_shows_shared_entity_baselines(self):
self.assertIn("shared-entity-baselines", HTML)
self.assertIn("Shared Entity Baselines", HTML)
self.assertIn("shared_entity_baselines", HTML)
def test_dashboard_shows_shape_mismatch_warnings(self):
self.assertIn('id="profileShapeWarnings"', HTML)
self.assertIn("profile_shape_warnings", HTML)

43
tests/test_monitor.py
View File

@@ -232,6 +232,49 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile")
self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile")
def test_shared_entity_baselines_lists_streams_pooled_by_entity_concept(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"
config_path.write_text(
json.dumps(
{
"graylog_streams": [
{"id": "firewall", "title": "Firewall", "enabled": True},
{"id": "iis", "title": "IIS", "enabled": True},
],
"graylog_stream_profiles": [
{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]},
{"stream_id": "iis", "name": "IIS profile", "entity_field": "client_ip", "categorical_fields": ["action"]},
],
}
),
encoding="utf-8",
)
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
self.assertEqual(len(status["shared_entity_baselines"]), 1)
pool = status["shared_entity_baselines"][0]
self.assertEqual(pool["group"], "srcip")
self.assertEqual(set(pool["streams"]), {"firewall", "iis"})
def test_shared_entity_baselines_empty_for_single_stream(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"
config_path.write_text(
json.dumps(
{
"graylog_streams": [{"id": "firewall", "title": "Firewall", "enabled": True}],
"graylog_stream_profiles": [{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}],
}
),
encoding="utf-8",
)
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
self.assertEqual(status["shared_entity_baselines"], [])
def test_stream_coverage_marks_missing_profiles(self):
with tempfile.TemporaryDirectory() as tmp:
config_path = Path(tmp) / "config.json"

33
tests/test_profile_suggestions.py
View File

@@ -1,7 +1,7 @@
import unittest
from fgai.logs import parse_log_line
from fgai.profile_suggestions import apply_profile_advice, detect_shape_mismatches, suggest_stream_profiles
from fgai.profile_suggestions import ALIAS_CANONICAL_BY_FIELD, apply_profile_advice, detect_shape_mismatches, shared_entity_pools, suggest_stream_profiles
from fgai.stream_profiles import RelationshipField, StreamProfile
@@ -456,6 +456,37 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("srcip", suggestion["field_shapes"])
self.assertEqual(suggestion["field_shapes"]["srcip"], "ip")
def test_alias_canonical_map_prefers_specific_group_over_context_fallback(self):
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["hostname"], "hostname")
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["dns_query"], "dns_query")
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["query_domain"], "dns_query")
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["qh"], "dns_query")
def test_shared_entity_pools_groups_streams_with_the_same_entity_concept(self):
stream_profiles = {
"firewall": StreamProfile(stream_id="firewall", name="Firewall", entity_field="srcip", timestamp_field="timestamp", entity_fields=("srcip",)),
"iis": StreamProfile(stream_id="iis", name="IIS", entity_field="client_ip", timestamp_field="timestamp", entity_fields=("client_ip",)),
"billing": StreamProfile(stream_id="billing", name="Billing", entity_field="actor_id", timestamp_field="timestamp", entity_fields=("actor_id",)),
}
pools = shared_entity_pools(stream_profiles)
self.assertEqual(pools["firewall"]["srcip"], pools["iis"]["client_ip"])
self.assertTrue(pools["firewall"]["srcip"].startswith("shared_entity:"))
self.assertNotIn("billing", pools)
def test_shared_entity_pools_keeps_source_and_destination_ip_separate(self):
stream_profiles = {
"firewall": StreamProfile(stream_id="firewall", name="Firewall", entity_field="srcip", timestamp_field="timestamp", entity_fields=("srcip", "dstip")),
"proxy": StreamProfile(stream_id="proxy", name="Proxy", entity_field="client_ip", timestamp_field="timestamp", entity_fields=("client_ip", "server_ip")),
}
pools = shared_entity_pools(stream_profiles)
self.assertEqual(pools["firewall"]["srcip"], pools["proxy"]["client_ip"])
self.assertEqual(pools["firewall"]["dstip"], pools["proxy"]["server_ip"])
self.assertNotEqual(pools["firewall"]["srcip"], pools["firewall"]["dstip"])
if __name__ == "__main__":
unittest.main()