Pool baseline state across streams sharing an entity concept

Completes the last deferred item from ROADMAP §0: entities recognized as
the same real-world concept across enabled streams (e.g. srcip in both a
firewall and an IIS stream) now share baseline knowledge of entity
presence/age and known relationship-target values, instead of each stream
learning it cold. Implemented additively (synthetic shared_entity:<group>
namespace in the existing baseline.py tables, no schema migration), scoped
to entity presence + relationship known-values only — per-stream
distribution baselines stay untouched. Also fixes a latent alias-map
collision bug found while building this (ALIAS_CANONICAL_BY_FIELD let
"context" silently steal "hostname"/"dns_query" from their own canonical
groups).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/fgai/baseline.py

@@ -10,7 +10,8 @@ from statistics import mean, pstdev
 
 from .logs import THREAT_ACTIONS, is_utm_event
 from .models import LogEvent
-from .entities import profile_entities
+from .entities import profile_entities, profile_entity_fields
+from .profile_suggestions import _semantic_field_group
 from .detectors import DETECTOR_MINIMUMS, event_detector_categories
 from .field_shapes import parse_iso_timestamp
 from .normalization import canonical_value
@@ -121,6 +122,15 @@ def _relationship_key(left: str, right: str) -> str:
     return f"relationship:{left.lower()}->{right.lower()}"
 
 
+def _pooled_relationship_key(pool_id: str, right: str) -> str:
+    # Different streams name the same relationship differently (relationship:srcip->dstip
+    # vs relationship:client_ip->dstip), so the pooled key must be canonicalized on both
+    # sides — the pool's own group for the left/entity side, and the right side's
+    # semantic group — or contributions from different streams would land under
+    # different keys and never actually pool together.
+    return _relationship_key(pool_id.split(":", 1)[1], _semantic_field_group(right))
+
+
 def _should_store_profile_value(field: str, value: str) -> bool:
     field = field.lower()
     if field.startswith("relationship:"):
@@ -248,8 +258,10 @@ class BaselineStore:
         *,
         observed_at: int | None = None,
         max_values_per_field: int = DEFAULT_MAX_VALUES_PER_FIELD,
+        entity_pools: dict[str, dict[str, str]] | None = None,
     ) -> int:
         observed_at = observed_at or int(time.time())
+        entity_pools = entity_pools or {}
         pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
         temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
         detector_pending: Counter[tuple[str, str, str, int]] = Counter()
@@ -264,14 +276,20 @@ class BaselineStore:
                 fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
                 if connection.execute("insert or ignore into profile_seen_events(fingerprint, first_seen) values (?, ?)", (fingerprint, observed_at)).rowcount != 1:
                     continue
-                entities = profile_entities(event, profile)
-                if not entities:
+                entity_pairs = profile_entity_fields(event, profile)
+                if not entity_pairs:
                     continue
+                entities = tuple(value for _field, value in entity_pairs)
                 timestamp = _event_epoch(event, observed_at)
                 bucket = timestamp - (timestamp % self.bucket_seconds)
                 moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
                 fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
                 numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
+                stream_pools = entity_pools.get(stream_id, {})
+                for entity_field, entity_value in entity_pairs:
+                    pool_id = stream_pools.get(entity_field)
+                    if pool_id:
+                        pending[(pool_id, entity_value, "__entity_presence__", bucket)][0] += 1
                 for entity in entities:
                     for detector in event_detector_categories(event):
                         detector_pending[(stream_id, entity, detector, bucket)] += 1
@@ -283,6 +301,9 @@ class BaselineStore:
                         right_value = event.fields.get(right)
                         if left_value and right_value:
                             pending_values[(stream_id, left_value, _relationship_key(left, right), right_value)] += 1
+                            pool_id = stream_pools.get(left)
+                            if pool_id:
+                                pending_values[(pool_id, left_value, _pooled_relationship_key(pool_id, right), right_value)] += 1
                     for field in fields:
                         key = (stream_id, entity, str(field).lower(), bucket)
                         value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
@@ -330,7 +351,8 @@ class BaselineStore:
                   on conflict(stream_id, entity, detector, weekday, hour, bucket_start) do update set events=events+excluded.events""", (stream_id, entity, detector, weekday, hour, bucket, count))
         return len(pending)
 
-    def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object], *, min_training_days: int = 0) -> dict[str, list[dict[str, object]]]:
+    def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object], *, min_training_days: int = 0, entity_pools: dict[str, dict[str, str]] | None = None) -> dict[str, list[dict[str, object]]]:
+        entity_pools = entity_pools or {}
         current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
         entity_events: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
         detector_current: Counter[tuple[str, str, str]] = Counter()
@@ -477,15 +499,18 @@ class BaselineStore:
             relationship_limit: Counter[tuple[str, str]] = Counter()
             for (stream, left_value, left, right, right_value), count in relationship_counts.items():
                 field = _relationship_key(left, right)
-                known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, left_value, field, right_value)).fetchone()
-                known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0]
+                pool_id = entity_pools.get(stream, {}).get(left)
+                lookup_scope = pool_id or stream
+                lookup_field = _pooled_relationship_key(pool_id, right) if pool_id else field
+                known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (lookup_scope, left_value, lookup_field, right_value)).fetchone()
+                known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (lookup_scope, left_value, lookup_field)).fetchone()[0]
                 if known is not None or int(known_total or 0) < 12:
                     continue
                 if relationship_limit[(stream, left_value)] >= MAX_RARE_VALUES_PER_ENTITY:
                     continue
-                oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=?", (stream, left_value)).fetchone()[0]
+                oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=?", (lookup_scope, left_value)).fetchone()[0]
                 if oldest is None:
-                    oldest = connection.execute("select min(last_seen) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0]
+                    oldest = connection.execute("select min(last_seen) from profile_values where stream_id=? and entity=? and field=?", (lookup_scope, left_value, lookup_field)).fetchone()[0]
                 baseline_age_days = _age_days(oldest, _event_epoch(relationship_samples[(stream, left_value, left, right, right_value)][0], int(time.time())))
                 if baseline_age_days < min_training_days:
                     continue

src/fgai/dashboard.py

@@ -1066,6 +1066,7 @@ async function refresh() {
   const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id}));
   document.getElementById('diagnostics').innerHTML =
     collapsibleSection('stream-coverage', 'Stream Coverage', table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage')) +
+    collapsibleSection('shared-entity-baselines', 'Shared Entity Baselines', (data.shared_entity_baselines || []).length ? table(data.shared_entity_baselines || [], [{label:'Entity concept', key:'group'}, {label:'Shared across streams', render:r => esc((r.stream_names || r.streams || []).join(', '))}], 'shared-entity-baselines') : '<p class="muted">No entity concept is currently recognized across two or more enabled streams. Baselines stay per-stream until that happens.</p>') +
     collapsibleSection('correlations', 'Cross-Source Correlations', table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations')) +
     collapsibleSection('entities', 'Entities', table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities')) +
     collapsibleSection('profile-readiness', 'Profile Baseline Readiness', table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `<button type="button" class="delete-stream-profile small-action danger-action" title="Delete profile" data-stream-id="${esc(r.stream_id)}" data-stream-name="${esc(r.stream_title || r.profile_name || r.stream_id)}">Remove</button>` : '-'}], 'profile-readiness')) +

src/fgai/entities.py

@@ -99,14 +99,20 @@ def profile_entity(event: LogEvent, field: str) -> str:
     return str(event.fields.get(field.lower(), "")).strip()
 
 
-def profile_entities(event: LogEvent, profile: object) -> tuple[str, ...]:
+def profile_entity_fields(event: LogEvent, profile: object) -> tuple[tuple[str, str], ...]:
     fields = tuple(str(field) for field in getattr(profile, "entity_fields", ()) if field) or (str(getattr(profile, "entity_field", "")),)
-    values = []
+    pairs: list[tuple[str, str]] = []
+    seen_values: set[str] = set()
     for field in fields:
         value = profile_entity(event, field)
-        if value and value not in values:
-            values.append(value)
-    return tuple(values)
+        if value and value not in seen_values:
+            seen_values.add(value)
+            pairs.append((field.lower(), value))
+    return tuple(pairs)
+
+
+def profile_entities(event: LogEvent, profile: object) -> tuple[str, ...]:
+    return tuple(value for _field, value in profile_entity_fields(event, profile))
 
 
 def _timeline_sample(event: LogEvent) -> dict[str, object]:

src/fgai/monitor.py

@@ -24,7 +24,7 @@ from .llm import ollama_dashboard_assessment, ollama_profile_advice
 from .logs import local_in_failures, read_events, summarize_events, top_field_values
 from .mitigation import parse_allowlist, suggest_block_candidates
 from .policies import audit_policies, read_policies
-from .profile_suggestions import apply_profile_advice, suggest_stream_profiles
+from .profile_suggestions import apply_profile_advice, shared_entity_pools, suggest_stream_profiles
 from .recommendations import build_recommendations
 from .sequences import detect_sequences
 from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
@@ -393,6 +393,7 @@ def build_status(
     runtime_values = config_store.read() if config_exists and config_store else {}
     runtime_config = _public_runtime_config(runtime_values) if config_store else {}
     stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
+    entity_pools = shared_entity_pools(stream_profiles)
     stream_titles = _stream_titles(runtime_values)
     events = read_events(log_path) if Path(log_path).exists() else []
     mcp_status: dict[str, object] = {"status": "not_configured"}
@@ -527,7 +528,7 @@ def build_status(
     baseline = BaselineStore(baseline_path) if baseline_path else None
     profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
     baseline_training_days = int(runtime_values.get("baseline_training_days", 7) or 7)
-    field_deviations = baseline.profile_deviations(events, stream_profiles, min_training_days=baseline_training_days) if baseline else {}
+    field_deviations = baseline.profile_deviations(events, stream_profiles, min_training_days=baseline_training_days, entity_pools=entity_pools) if baseline else {}
     sequence_findings = detect_sequences(events)
     for entity, findings in sequence_findings.items():
         field_deviations.setdefault(entity, []).extend(findings)
@@ -569,6 +570,7 @@ def build_status(
             events,
             stream_profiles,
             max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500),
+            entity_pools=entity_pools,
         )
         if baseline else 0
     )
@@ -701,6 +703,18 @@ def build_status(
         for suggestion in profile_suggestions
         for warning in suggestion.get("shape_warnings", [])
     ]
+    pool_streams: dict[str, set[str]] = defaultdict(set)
+    for pool_stream_id, fields in entity_pools.items():
+        for pool_id in fields.values():
+            pool_streams[pool_id].add(pool_stream_id)
+    shared_entity_baselines = [
+        {
+            "group": pool_id.split(":", 1)[1],
+            "streams": sorted(pool_streams[pool_id]),
+            "stream_names": sorted(_stream_name(sid, stream_titles, stream_profiles.get(sid)) for sid in pool_streams[pool_id]),
+        }
+        for pool_id in sorted(pool_streams)
+    ]
     status = {
         "status_schema": 2,
         "stale": False,
@@ -718,6 +732,7 @@ def build_status(
         "stream_coverage": stream_coverage,
         "profile_suggestions": profile_suggestions,
         "profile_shape_warnings": profile_shape_warnings,
+        "shared_entity_baselines": shared_entity_baselines,
         "profile_readiness": profile_readiness,
         "diagnostics": {
             "top_source_ips": top_field_values(events, "srcip", limit=10),

src/fgai/profile_suggestions.py

@@ -73,11 +73,19 @@ def detect_shape_mismatches(existing: object, field_shapes: dict[str, str], fiel
     return warnings
 
 
-ALIAS_CANONICAL_BY_FIELD = {
-    alias: canonical
-    for canonical, aliases in FIELD_ALIASES.items()
-    for alias in aliases
-}
+def _build_alias_canonical_map() -> dict[str, str]:
+    # First-wins: FIELD_ALIASES lists specific groups before the generic
+    # "context" fallback, and some fallback aliases (hostname, dns_query)
+    # collide with a specific group's own alias — the specific group must
+    # win, or _semantic_field_group("hostname") would resolve to "context".
+    mapping: dict[str, str] = {}
+    for canonical, aliases in FIELD_ALIASES.items():
+        for alias in aliases:
+            mapping.setdefault(alias, canonical)
+    return mapping
+
+
+ALIAS_CANONICAL_BY_FIELD = _build_alias_canonical_map()
 
 ENTITY_PRIORITY = (
     "srcip",
@@ -717,6 +725,32 @@ def _shared_profile_fields(
     return list(dict.fromkeys(categorical))[:8], list(dict.fromkeys(numeric))[:5], stats[:16]
 
 
+def shared_entity_pools(stream_profiles: dict[str, object]) -> dict[str, dict[str, str]]:
+    """Map each stream's entity fields to a pool id shared with other streams
+    that recognize the same real-world entity concept, via the same canonical
+    alias grouping used for cross-stream field-sharing recommendations.
+
+    Returns {stream_id: {entity_field_lower: pool_id}}. A field is included
+    only when at least one other stream shares its canonical group, so a
+    stream with no matching peers gets nothing and baseline behavior for it
+    is unchanged.
+    """
+    group_streams: dict[str, set[str]] = defaultdict(set)
+    stream_entity_groups: dict[str, dict[str, str]] = {}
+    for stream_id, profile in stream_profiles.items():
+        fields = tuple(str(field).lower() for field in getattr(profile, "entity_fields", ()) if field)
+        groups = {field: _semantic_field_group(field) for field in fields}
+        stream_entity_groups[stream_id] = groups
+        for group in groups.values():
+            group_streams[group].add(stream_id)
+    pools: dict[str, dict[str, str]] = {}
+    for stream_id, groups in stream_entity_groups.items():
+        mapping = {field: f"shared_entity:{group}" for field, group in groups.items() if len(group_streams[group]) >= 2}
+        if mapping:
+            pools[stream_id] = mapping
+    return pools
+
+
 def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
     profile = suggestion.get("profile", {}) if isinstance(suggestion.get("profile"), dict) else {}
     fields = {str(item.get("field", "")) for item in suggestion.get("common_fields", []) if isinstance(item, dict)}