Fortsatte roadmapen med multi-entity stream profiles

README.md

@@ -59,9 +59,9 @@ or a complete `Basic <value>` header. Tokens are stored only in the local runtim
 configuration and are never returned by the dashboard API.
 
 Use `Edit profile` on a stream to load its fields. The field table shows Graylog
-datatype/capability metadata and lets you select an entity field, a time field,
+datatype/capability metadata and lets you select one or more entity fields, a
-and categorical/numeric fields for the stream profile. Profiles are stored under
+time field, and categorical/numeric fields for the stream profile. Profiles are
-`graylog_stream_profiles` in `state/fgai-config.json`.
+stored under `graylog_stream_profiles` in `state/fgai-config.json`.
 
 The settings page treats stream enablement and profile editing separately. The
 checkboxes decide which streams are monitored. Click `Edit profile` on one stream
@@ -79,6 +79,11 @@ The dashboard and Ollama then correlate behavior across sources, for example a
 client IP appearing in FortiGate, AdGuard/DNS, Windows Security, Nginx, Squid,
 VPN, or Proxmox.
 
+A stream profile can track multiple entities from the same event, such as
+`username`, `srcip`, and `hostname`. SignalScope stores profile baselines for
+each selected entity value, which makes cross-source investigation work even when
+one source is user-centric and another is IP- or host-centric.
+
 SignalScope keeps a common alias map for fields such as source IP, destination
 IP, ports, action, severity, service/protocol, DNS query, URL, message, and event
 type. This lets Related Activity and correlations work with firewall/proxy/DNS

ROADMAP.md

@@ -40,7 +40,7 @@ Acceptance: each finding shows its detector, confidence, baseline sample count,
 
 Goal: make one incident answer what happened, to whom, and across which sources.
 
-- [ ] Allow multiple entity fields per stream, such as user plus source IP plus hostname.
+- [x] Allow multiple entity fields per stream, such as user plus source IP plus hostname.
 - [ ] Add entity aliasing: map DHCP, VPN, DNS, and endpoint identities to the same host where evidence supports it.
 - [ ] Add configurable incident grouping windows and incident lifecycle: open, acknowledged, resolved, reopened.
 - [ ] Persist incident state and analyst notes separately from transient detection output.

src/fgai/baseline.py

@@ -10,7 +10,7 @@ from statistics import mean, pstdev
 
 from .logs import THREAT_ACTIONS, is_utm_event
 from .models import LogEvent
-from .entities import profile_entity
+from .entities import profile_entities
 from .detectors import DETECTOR_MINIMUMS, event_detector_categories
 from .normalization import canonical_value
 
@@ -180,17 +180,18 @@ class BaselineStore:
                 fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
                 if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1:
                     continue
-                entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
+                entities = profile_entities(event, profile)
-                if not entity:
+                if not entities:
                     continue
                 timestamp = _event_epoch(event, observed_at)
                 bucket = timestamp - (timestamp % self.bucket_seconds)
                 moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+                fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
+                numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
+                for entity in entities:
                     for detector in event_detector_categories(event):
                         detector_pending[(stream_id, entity, detector, bucket)] += 1
                         detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
-                fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
-                numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
                     for field in fields:
                         key = (stream_id, entity, str(field).lower(), bucket)
                         value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
@@ -230,13 +231,14 @@ class BaselineStore:
             profile = profiles.get(event.fields.get("fgai_stream_id", ""))
             if not profile:
                 continue
-            entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
+            entities = profile_entities(event, profile)
-            if not entity:
+            if not entities:
                 continue
+            numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
+            for entity in entities:
                 entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
                 for detector in event_detector_categories(event):
                     detector_current[(event.fields.get("fgai_stream_id", ""), entity, detector)] += 1
-            numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
                 for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
                     key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
                     current[key][0] += 1
@@ -246,7 +248,7 @@ class BaselineStore:
         with self._connect() as connection:
             for (stream, entity, field), values in current.items():
                 profile = profiles.get(stream)
-                matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
+                matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and entity in profile_entities(event, profiles.get(stream)) and event.fields.get(field)]
                 current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
                 moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
                 rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
@@ -336,10 +338,11 @@ class BaselineStore:
                 profile = profiles.get(event.fields.get("fgai_stream_id", ""))
                 if not profile:
                     continue
-                entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
+                entities = profile_entities(event, profile)
-                if not entity:
+                if not entities:
                     continue
                 stream = event.fields.get("fgai_stream_id", "")
+                for entity in entities:
                     for field in getattr(profile, "categorical_fields", ()):
                         field = str(field).lower()
                         value = event.fields.get(field)
@@ -348,7 +351,7 @@ class BaselineStore:
                         known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
                         known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
                         if known is None and known_total >= 30:
-                        samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
+                            samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and entity in profile_entities(item, profile) and item.fields.get(field) == value]
                             score, weight = _weighted_score(12, profile, field, "rare_value")
                             evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
                             if evidence not in output[entity]:

src/fgai/cli.py

@@ -230,6 +230,7 @@ def _profile_fields(profile: object | None) -> tuple[str, ...]:
         return tuple(
             field for field in (
                 str(getattr(profile, "entity_field", "")),
+                *tuple(str(item) for item in getattr(profile, "entity_fields", ())),
                 str(getattr(profile, "timestamp_field", "")),
             *tuple(str(item) for item in getattr(profile, "categorical_fields", ())),
             *tuple(str(item) for item in getattr(profile, "numeric_fields", ())),

src/fgai/dashboard.py

@@ -271,7 +271,8 @@ async function editStreamProfile(streamId, title) {
   document.querySelector('[name="profile_name"]').value = profile.name || `${title || streamId} profile`;
   document.querySelector('[name="profile_detectors"]').value = Object.keys(profile.detectors || {}).length ? JSON.stringify(profile.detectors, null, 2) : '';
   document.querySelector('[name="profile_field_weights"]').value = Object.keys(profile.field_weights || {}).length ? JSON.stringify(profile.field_weights, null, 2) : '';
-  const rows = (payload.fields || []).map(field => { const name=field.name||field.field, type=(field.type||{}).type||'', props=(field.type||{}).properties||[]; return `<div class="field-row"><code>${esc(name)}</code><span>${esc(type)}</span><span>${esc(props.join(', '))}</span><div class="field-controls"><label><input type="radio" name="profile_entity" value="${esc(name)}" ${profile.entity_field===name?'checked':''}> Entity</label><label><input type="radio" name="profile_timestamp" value="${esc(name)}" ${profile.timestamp_field===name?'checked':''}> Time</label>${props.includes('enumerable')?`<label><input class="profile-categorical" type="checkbox" value="${esc(name)}" ${(profile.categorical_fields||[]).includes(name)?'checked':''}> Categorical</label>`:''}${props.includes('numeric')?`<label><input class="profile-numeric" type="checkbox" value="${esc(name)}" ${(profile.numeric_fields||[]).includes(name)?'checked':''}> Numeric</label>`:''}</div></div>`; }).join('');
+  const entityFields = new Set(profile.entity_fields || (profile.entity_field ? [profile.entity_field] : []));
+  const rows = (payload.fields || []).map(field => { const name=field.name||field.field, type=(field.type||{}).type||'', props=(field.type||{}).properties||[]; return `<div class="field-row"><code>${esc(name)}</code><span>${esc(type)}</span><span>${esc(props.join(', '))}</span><div class="field-controls"><label><input class="profile-entity" type="checkbox" value="${esc(name)}" ${entityFields.has(name)?'checked':''}> Entity</label><label><input type="radio" name="profile_timestamp" value="${esc(name)}" ${profile.timestamp_field===name?'checked':''}> Time</label>${props.includes('enumerable')?`<label><input class="profile-categorical" type="checkbox" value="${esc(name)}" ${(profile.categorical_fields||[]).includes(name)?'checked':''}> Categorical</label>`:''}${props.includes('numeric')?`<label><input class="profile-numeric" type="checkbox" value="${esc(name)}" ${(profile.numeric_fields||[]).includes(name)?'checked':''}> Numeric</label>`:''}</div></div>`; }).join('');
   document.getElementById('fieldPicker').innerHTML = rows ? `<div class="field-header"><span>Field</span><span>Type</span><span>Capabilities</span><span>Use In Profile</span></div>${rows}` : esc(payload.error || 'No fields found.');
   window.activeProfileStream = streamId;
   window.activeProfileTitle = title || streamId;
@@ -294,7 +295,7 @@ document.getElementById('settingsForm').addEventListener('submit', async event =
   values.llm_enabled = form.elements.llm_enabled.checked;
   values.threat_intel_enabled = form.elements.threat_intel_enabled.checked;
   values.graylog_streams = [...document.querySelectorAll('.graylog-stream')].map(item => ({id:item.dataset.id, title:item.dataset.title, enabled:item.checked}));
-  if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:form.querySelector('[name="profile_entity"]:checked')?.value||'',timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
+  if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const entityFields=[...form.querySelectorAll('.profile-entity:checked')].map(item=>item.value); if (!entityFields.length) { document.getElementById('settingsResult').textContent='Select at least one Entity field for the active profile.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:entityFields[0]||'',entity_fields:entityFields,timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
   const savedProfile = window.activeProfileStream ? ` Profile saved for ${window.activeProfileTitle || window.activeProfileStream}.` : ' No profile editor active, so profiles were not changed.';
   const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(values)});
   document.getElementById('settingsResult').textContent = response.ok ? `Saved enabled streams and global settings.${savedProfile} Monitor applies supported settings on its next cycle.` : 'Could not save configuration.';

src/fgai/entities.py

@@ -42,6 +42,16 @@ def profile_entity(event: LogEvent, field: str) -> str:
     return str(event.fields.get(field.lower(), "")).strip()
 
 
+def profile_entities(event: LogEvent, profile: object) -> tuple[str, ...]:
+    fields = tuple(str(field) for field in getattr(profile, "entity_fields", ()) if field) or (str(getattr(profile, "entity_field", "")),)
+    values = []
+    for field in fields:
+        value = profile_entity(event, field)
+        if value and value not in values:
+            values.append(value)
+    return tuple(values)
+
+
 def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]:
     samples = [
         {

src/fgai/monitor.py

@@ -79,7 +79,7 @@ def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[st
             "enabled": enabled,
             "profile": _profile_name(stream_id, stream_titles, profile) if profile else "",
             "profile_ready": bool(profile),
-            "entity_field": str(getattr(profile, "entity_field", "")) if profile else "",
+            "entity_field": ", ".join(getattr(profile, "entity_fields", ()) or (str(getattr(profile, "entity_field", "")),)) if profile else "",
             "tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) if profile else 0,
             "ready_fields": ready_fields,
             "total_fields": total_fields,
@@ -131,6 +131,7 @@ def build_status(
                     profile = stream_profiles.get(stream_id)
                     profile_fields = (
                         str(getattr(profile, "entity_field", "")),
+                        *tuple(str(field) for field in getattr(profile, "entity_fields", ())),
                         str(getattr(profile, "timestamp_field", "")),
                         *tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
                         *tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
@@ -225,7 +226,7 @@ def build_status(
         "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields},
         "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
         "configuration": runtime_config,
-        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
+        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
         "stream_coverage": stream_coverage,
         "profile_readiness": profile_readiness,
         "diagnostics": {

src/fgai/stream_profiles.py

@@ -9,6 +9,7 @@ class StreamProfile:
     name: str
     entity_field: str
     timestamp_field: str
+    entity_fields: tuple[str, ...] = ()
     categorical_fields: tuple[str, ...] = ()
     numeric_fields: tuple[str, ...] = ()
     detectors: dict[str, dict[str, object]] = field(default_factory=dict)
@@ -59,14 +60,17 @@ def parse_profiles(value: object) -> dict[str, StreamProfile]:
         if not isinstance(item, dict):
             continue
         stream_id = str(item.get("stream_id", "")).strip()
-        entity = str(item.get("entity_field", "")).strip()
+        entity_fields = tuple(dict.fromkeys(str(field).strip() for field in item.get("entity_fields", []) if str(field).strip())) if isinstance(item.get("entity_fields", []), list) else ()
+        entity = str(item.get("entity_field", "")).strip() or (entity_fields[0] if entity_fields else "")
+        if not entity_fields and entity:
+            entity_fields = (entity,)
         timestamp = str(item.get("timestamp_field", "timestamp")).strip()
         if not stream_id or not entity:
             continue
         detectors = _detectors(item.get("detectors", {}))
         field_weights = _field_weights(item.get("field_weights", {}))
         profiles[stream_id] = StreamProfile(
-            stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp,
+            stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp, entity_fields,
             tuple(str(field) for field in item.get("categorical_fields", []) if field),
             tuple(str(field) for field in item.get("numeric_fields", []) if field),
             detectors,

tests/test_baseline.py

@@ -28,6 +28,17 @@ class BaselineTests(unittest.TestCase):
             self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000), 1)
             self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_300), 0)
 
+    def test_profile_can_baseline_multiple_entities_from_same_event(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
+            profiles = parse_profiles([{"stream_id": "vpn", "entity_fields": ["username", "srcip"], "categorical_fields": ["action"]}])
+            events = [parse_log_line("fgai_stream_id=vpn username=alice srcip=10.0.0.5 action=login")]
+
+            self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000), 2)
+            readiness = store.profile_readiness(profiles)
+
+            self.assertEqual(readiness[0]["buckets"], 1)
+
     def test_profile_rate_burst_is_one_confident_detector(self):
         with tempfile.TemporaryDirectory() as directory:
             store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))

tests/test_stream_profiles.py

@@ -8,8 +8,14 @@ class StreamProfileTests(unittest.TestCase):
         profiles = parse_profiles([{"stream_id": "dns", "name": "DNS client behavior", "entity_field": "IP", "categorical_fields": ["QH"], "numeric_fields": ["Elapsed"]}])
         self.assertEqual(profiles["dns"].name, "DNS client behavior")
         self.assertEqual(profiles["dns"].entity_field, "IP")
+        self.assertEqual(profiles["dns"].entity_fields, ("IP",))
         self.assertEqual(profiles["dns"].numeric_fields, ("Elapsed",))
 
+    def test_parses_multiple_entity_fields(self):
+        profiles = parse_profiles([{"stream_id": "vpn", "entity_fields": ["username", "srcip", "hostname"], "categorical_fields": ["action"]}])
+        self.assertEqual(profiles["vpn"].entity_field, "username")
+        self.assertEqual(profiles["vpn"].entity_fields, ("username", "srcip", "hostname"))
+
     def test_parses_detector_thresholds(self):
         profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "detectors": {"auth_failure": {"enabled": False, "minimum": 7, "z_threshold": 4.5}}}])
         self.assertEqual(profiles["windows"].detectors["auth_failure"], {"enabled": False, "minimum": 7, "z_threshold": 4.5})