Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pool baseline state across streams sharing an entity concept

Browse Source 
Completes the last deferred item from ROADMAP §0: entities recognized as
the same real-world concept across enabled streams (e.g. srcip in both a
firewall and an IIS stream) now share baseline knowledge of entity
presence/age and known relationship-target values, instead of each stream
learning it cold. Implemented additively (synthetic shared_entity:<group>
namespace in the existing baseline.py tables, no schema migration), scoped
to entity presence + relationship known-values only — per-stream
distribution baselines stay untouched. Also fixes a latent alias-map
collision bug found while building this (ALIAS_CANONICAL_BY_FIELD let
"context" silently steal "hostname"/"dns_query" from their own canonical
groups).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-23 18:50:14 +02:00
parent 8c155ec9cf
commit 83ffef3570
12 changed files with 305 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
tests/test_profile_suggestions.py
View File

@@ -1,7 +1,7 @@
import unittest
from fgai.logs import parse_log_line
from fgai.profile_suggestions import apply_profile_advice, detect_shape_mismatches, suggest_stream_profiles
from fgai.profile_suggestions import ALIAS_CANONICAL_BY_FIELD, apply_profile_advice, detect_shape_mismatches, shared_entity_pools, suggest_stream_profiles
from fgai.stream_profiles import RelationshipField, StreamProfile
@@ -456,6 +456,37 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("srcip", suggestion["field_shapes"])
self.assertEqual(suggestion["field_shapes"]["srcip"], "ip")
def test_alias_canonical_map_prefers_specific_group_over_context_fallback(self):
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["hostname"], "hostname")
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["dns_query"], "dns_query")
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["query_domain"], "dns_query")
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["qh"], "dns_query")
def test_shared_entity_pools_groups_streams_with_the_same_entity_concept(self):
stream_profiles = {
"firewall": StreamProfile(stream_id="firewall", name="Firewall", entity_field="srcip", timestamp_field="timestamp", entity_fields=("srcip",)),
"iis": StreamProfile(stream_id="iis", name="IIS", entity_field="client_ip", timestamp_field="timestamp", entity_fields=("client_ip",)),
"billing": StreamProfile(stream_id="billing", name="Billing", entity_field="actor_id", timestamp_field="timestamp", entity_fields=("actor_id",)),
}
pools = shared_entity_pools(stream_profiles)
self.assertEqual(pools["firewall"]["srcip"], pools["iis"]["client_ip"])
self.assertTrue(pools["firewall"]["srcip"].startswith("shared_entity:"))
self.assertNotIn("billing", pools)
def test_shared_entity_pools_keeps_source_and_destination_ip_separate(self):
stream_profiles = {
"firewall": StreamProfile(stream_id="firewall", name="Firewall", entity_field="srcip", timestamp_field="timestamp", entity_fields=("srcip", "dstip")),
"proxy": StreamProfile(stream_id="proxy", name="Proxy", entity_field="client_ip", timestamp_field="timestamp", entity_fields=("client_ip", "server_ip")),
}
pools = shared_entity_pools(stream_profiles)
self.assertEqual(pools["firewall"]["srcip"], pools["proxy"]["client_ip"])
self.assertEqual(pools["firewall"]["dstip"], pools["proxy"]["server_ip"])
self.assertNotEqual(pools["firewall"]["srcip"], pools["firewall"]["dstip"])
if __name__ == "__main__":
unittest.main()