Implemented the next multi-source detection layer in this repository.

src/fgai/baseline.py

@@ -4,11 +4,13 @@ import hashlib
 import sqlite3
 import time
 from collections import Counter, defaultdict
+from datetime import datetime, timezone
 from pathlib import Path
 from statistics import mean, pstdev
 
 from .logs import THREAT_ACTIONS, is_utm_event
 from .models import LogEvent
+from .entities import profile_entity
 
 
 def _number(value: str | None) -> int:
@@ -18,6 +20,19 @@ def _number(value: str | None) -> int:
         return 0
 
 
+def _event_epoch(event: LogEvent, fallback: int) -> int:
+    value = event.fields.get("eventtime", event.fields.get("timestamp", ""))
+    if not value:
+        return fallback
+    try:
+        return int(float(value))
+    except ValueError:
+        try:
+            return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
+        except ValueError:
+            return fallback
+
+
 class BaselineStore:
     """Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
 
@@ -48,6 +63,13 @@ class BaselineStore:
                   stream_id text not null, entity text not null, field text not null, value text not null,
                   seen_count integer not null, primary key (stream_id, entity, field, value)
                 );
+                create table if not exists profile_seen_events (fingerprint text primary key);
+                create table if not exists profile_temporal_buckets (
+                  stream_id text not null, entity text not null, field text not null,
+                  weekday integer not null, hour integer not null, bucket_start integer not null,
+                  events integer not null, numeric_sum real not null, numeric_sum_squares real not null,
+                  primary key (stream_id, entity, field, weekday, hour, bucket_start)
+                );
                 """
             )
 
@@ -92,34 +114,46 @@ class BaselineStore:
 
     def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int:
         observed_at = observed_at or int(time.time())
-        bucket = observed_at - (observed_at % self.bucket_seconds)
-        pending: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
+        pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
+        temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
         pending_values: Counter[tuple[str, str, str, str]] = Counter()
-        for event in events:
-            stream_id = event.fields.get("fgai_stream_id", "")
-            profile = profiles.get(stream_id)
-            if not profile:
-                continue
-            entity_field = str(getattr(profile, "entity_field", "")).lower()
-            entity = event.fields.get(entity_field)
-            if not entity:
-                continue
-            fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
-            numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
-            for field in fields:
-                key = (stream_id, entity, str(field).lower())
-                value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
-                pending[key][0] += 1
-                pending[key][1] += value
-                pending[key][2] += value * value
-                if key[2] not in numeric:
-                    raw_value = event.fields.get(key[2])
-                    if raw_value:
-                        pending_values[(*key, raw_value)] += 1
         with self._connect() as connection:
-            for (stream_id, entity, field), values in pending.items():
+            for event in events:
+                stream_id = event.fields.get("fgai_stream_id", "")
+                profile = profiles.get(stream_id)
+                if not profile:
+                    continue
+                fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
+                if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1:
+                    continue
+                entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
+                if not entity:
+                    continue
+                timestamp = _event_epoch(event, observed_at)
+                bucket = timestamp - (timestamp % self.bucket_seconds)
+                moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+                fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
+                numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
+                for field in fields:
+                    key = (stream_id, entity, str(field).lower(), bucket)
+                    value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
+                    pending[key][0] += 1
+                    pending[key][1] += value
+                    pending[key][2] += value * value
+                    temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
+                    temporal_pending[temporal][0] += 1
+                    temporal_pending[temporal][1] += value
+                    temporal_pending[temporal][2] += value * value
+                    if key[2] not in numeric:
+                        raw_value = event.fields.get(key[2])
+                        if raw_value:
+                            pending_values[(*key[:3], raw_value)] += 1
+            for (stream_id, entity, field, bucket), values in pending.items():
                 connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
                   on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
+            for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items():
+                connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                  on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values))
             for key, count in pending_values.items():
                 connection.execute("""insert into profile_values values (?, ?, ?, ?, ?)
                   on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count""", (*key, count))
@@ -131,7 +165,7 @@ class BaselineStore:
             profile = profiles.get(event.fields.get("fgai_stream_id", ""))
             if not profile:
                 continue
-            entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
+            entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
             if not entity:
                 continue
             numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
@@ -143,20 +177,26 @@ class BaselineStore:
         output: dict[str, list[dict[str, object]]] = defaultdict(list)
         with self._connect() as connection:
             for (stream, entity, field), values in current.items():
-                rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
+                matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
+                current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
+                moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
+                rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
+                baseline_scope = "same weekday/hour"
+                if len(rows) < 12:
+                    rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
+                    baseline_scope = "all observed periods"
                 if len(rows) < 12:
                     continue
                 if values[1] == 0:
                     history = [row[0] for row in rows]
                     current_value = values[0]
-                    reason = f"{field} event rate deviates from its stream baseline"
+                    reason = f"{field} event rate deviates from its {baseline_scope} baseline"
                 else:
                     history = [row[1] / row[0] if row[0] else 0 for row in rows]
                     current_value = values[1] / values[0] if values[0] else 0
-                    reason = f"{field} value deviates from its stream baseline"
+                    reason = f"{field} value deviates from its {baseline_scope} baseline"
                 deviation = abs(current_value - mean(history))
                 if deviation > (pstdev(history) or 1.0) * 3:
-                    matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and event.fields.get(str(getattr(profiles.get(stream), "entity_field", "")).lower()) == entity and event.fields.get(field)]
                     samples = sorted({event.fields.get(field, "") for event in matching})[:5]
                     evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
                     output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": samples, "sample_events": evidence_events})
@@ -165,7 +205,7 @@ class BaselineStore:
                 profile = profiles.get(event.fields.get("fgai_stream_id", ""))
                 if not profile:
                     continue
-                entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
+                entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
                 if not entity:
                     continue
                 stream = event.fields.get("fgai_stream_id", "")
@@ -177,7 +217,8 @@ class BaselineStore:
                     known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
                     known_total = connection.execute("select count(*) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
                     if known is None and known_total >= 10:
-                        evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value}
+                        samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
+                        evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
                         if evidence not in output[entity]:
                             output[entity].append(evidence)
         return output

src/fgai/correlation.py

@@ -2,29 +2,33 @@ from __future__ import annotations
 
 from collections import defaultdict
 
+from .entities import event_entities, sample_timeline
 from .logs import THREAT_ACTIONS, is_utm_event
 from .models import LogEvent
 
 
 def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
-    grouped: dict[str, list[LogEvent]] = defaultdict(list)
+    """Correlate IPs, users, and hosts across independently configured Graylog streams."""
+    grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
     for event in events:
-        if event.src_ip:
-            grouped[event.src_ip].append(event)
+        for identity in event_entities(event):
+            grouped[(identity["entity_type"], identity["entity"])].append(event)
     correlations = []
-    for source_ip, source_events in grouped.items():
+    for (kind, entity), source_events in grouped.items():
         streams = sorted({event.fields.get("fgai_stream", "local_syslog") for event in source_events})
         if len(streams) < 2:
             continue
         threat_events = sum(event.action in THREAT_ACTIONS or is_utm_event(event) for event in source_events)
-        samples = [
-            {
-                "stream": event.fields.get("fgai_stream", "local_syslog"), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
-                "type": event.fields.get("type", ""), "subtype": event.subtype, "action": event.action,
-                "severity": event.severity, "destination": event.dst_ip or "", "service": event.fields.get("service", ""),
-                "context": event.fields.get("query_domain", event.fields.get("qh", event.fields.get("message", ""))),
-            }
-            for event in source_events[:20]
-        ]
-        correlations.append({"source_ip": source_ip, "streams": streams, "events": len(source_events), "security_events": threat_events, "samples": samples})
+        samples = sample_timeline(source_events)
+        correlations.append({
+            "entity": entity,
+            "entity_type": kind,
+            "source_ip": entity if kind == "ip" else "",
+            "streams": streams,
+            "events": len(source_events),
+            "security_events": threat_events,
+            "samples": samples,
+            "first_seen": samples[0]["timestamp"] if samples else "",
+            "last_seen": samples[-1]["timestamp"] if samples else "",
+        })
     return sorted(correlations, key=lambda item: (int(item["security_events"]), int(item["events"])), reverse=True)[:limit]

src/fgai/dashboard.py

@@ -74,7 +74,7 @@ HTML = """<!doctype html>
       </div>
     </section>
     <nav class="tabs" aria-label="Dashboard views"><button class="tab active" data-tab="overview">Overview</button><button class="tab" data-tab="findings">Findings</button><button class="tab" data-tab="diagnostics">Diagnostics</button><button class="tab" data-tab="settings">Settings</button></nav>
-    <div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
+    <div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
     <div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
     <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
     <div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
@@ -151,27 +151,35 @@ async function refresh() {
     {label:'Policies', render:r => esc((r.related_policy_ids || []).join(', '))},
     {label:'Services', render:r => esc((r.related_services || []).join(', '))}
   ]);
+  document.getElementById('incidents').innerHTML = table(data.incidents || [], [
+    {label:'Entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)},
+    {label:'Score', key:'score'},
+    {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
+    {label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')},
+    {label:'Evidence', render:r => esc((r.evidence || []).join('; '))},
+    {label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('<br>'); return rows ? `<details><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }}
+  ]);
   document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
     {label:'Source', key:'src_ip'},
     {label:'Score', key:'score'},
     {label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
   ]);
   const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
-  const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({source_ip: correlation.source_ip, ...sample})));
+  const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
   const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
   const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item})));
   document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [
-    {label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark confirmed</button></div>`}
+    {label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
   ]);
   document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
     const note = prompt('Review note (optional):') || '';
     const days = prompt('Expiry in days (0 = no expiry):', '0') || '0';
-    const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
+    const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, value:button.dataset.value, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
     document.getElementById('feedbackNotice').textContent = response.ok ? 'Review saved. The matching pattern will be labeled on the next refresh.' : 'Could not save review.';
     refresh();
   }));
   document.getElementById('relatedActivity').innerHTML = table(relatedRows, [
-    {label:'Source IP', key:'source_ip'}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
+    {label:'Entity', render:r => esc(r.entity || r.source_ip)}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
     {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'},
     {label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'}
   ]);
@@ -195,7 +203,7 @@ async function refresh() {
   const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id}));
   const correlations = data.cross_source_correlations || [];
   document.getElementById('diagnostics').innerHTML =
-    '<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Source IP', key:'source_ip'}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
+    '<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
     '<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) +
     '<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', render:r => r.ready ? 'ready' : 'learning'}]) +
     '<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +

src/fgai/entities.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import ipaddress
+from collections.abc import Iterable
+
+from .models import LogEvent
+
+
+ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
+    "ip": ("srcip", "src_ip", "source_ip", "client_ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "event_data_ipaddress"),
+    "user": ("username", "user", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog_event_data_subjectusername"),
+    "host": ("hostname", "host", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "agent_name"),
+}
+
+
+def entity_type(value: str) -> str:
+    try:
+        ipaddress.ip_address(value)
+        return "ip"
+    except ValueError:
+        return "entity"
+
+
+def event_entities(event: LogEvent) -> list[dict[str, str]]:
+    """Return normalized identities shared across network, endpoint, DNS, and web logs."""
+    identities: list[dict[str, str]] = []
+    seen: set[tuple[str, str]] = set()
+    for kind, fields in ENTITY_FIELDS.items():
+        for field in fields:
+            value = str(event.fields.get(field, "")).strip()
+            if not value or value in {"-", "unknown", "n/a"}:
+                continue
+            key = (kind, value.lower() if kind != "ip" else value)
+            if key not in seen:
+                seen.add(key)
+                identities.append({"entity": value, "entity_type": kind, "field": field})
+    return identities
+
+
+def profile_entity(event: LogEvent, field: str) -> str:
+    return str(event.fields.get(field.lower(), "")).strip()
+
+
+def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]:
+    samples = [
+        {
+            "stream": event.fields.get("fgai_stream", "local_syslog"),
+            "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
+            "type": event.fields.get("type", ""),
+            "subtype": event.subtype,
+            "action": event.action,
+            "severity": event.severity,
+            "destination": event.dst_ip or event.fields.get("query_domain", event.fields.get("url", "")),
+            "service": event.fields.get("service", event.fields.get("query_type", "")),
+            "context": event.fields.get("query_domain", event.fields.get("qh", event.fields.get("url", event.fields.get("message", event.fields.get("msg", "")))))[:240],
+        }
+        for event in events
+    ]
+    return sorted(samples, key=lambda item: item["timestamp"])[-limit:]

src/fgai/feedback.py

@@ -21,7 +21,7 @@ class FeedbackStore:
         status = str(item.get("status", "")).lower()
         if status not in {"false_positive", "expected", "confirmed"}:
             raise ValueError("invalid feedback status")
-        entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)}
-        entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field")) != (entry["stream_id"], entry["entity"], entry["field"])]
+        entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "value": str(item.get("value", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)}
+        entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field"), value.get("value", "")) != (entry["stream_id"], entry["entity"], entry["field"], entry["value"])]
         entries.append(entry); self.path.parent.mkdir(parents=True, exist_ok=True); self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8")
         return entry

src/fgai/graylog_source.py

@@ -41,11 +41,12 @@ def _records(value: object) -> Iterable[dict[str, object]]:
 
 
 class GraylogStreamSource:
-    def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "") -> None:
+    def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "", profile_fields: tuple[str, ...] = ()) -> None:
         self.client = client
         self.stream = stream
         self.query = query or "*"
         self.stream_label = stream_label or stream
+        self.profile_fields = tuple(str(field).lower() for field in profile_fields if field)
         try:
             self.mapping = json.loads(field_mapping) if field_mapping else {}
         except json.JSONDecodeError as exc:
@@ -59,7 +60,7 @@ class GraylogStreamSource:
         arguments: dict[str, object] = {
             "query": self.query,
             "range_seconds": 300,
-            "fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields])),
+            "fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields, *self.profile_fields])),
         }
         if self.stream:
             arguments["streams"] = [self.stream]

src/fgai/incidents.py

@@ -1,17 +1,54 @@
 from __future__ import annotations
 
+from collections import defaultdict
+
+from .entities import entity_type
 from .models import AnomalyFinding
 
 
 def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]:
-    correlation_by_ip = {str(item.get("source_ip")): item for item in correlations}
-    incidents = []
+    """Build investigation units from any log source, not only network source IPs."""
+    groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []})
     for anomaly in anomalies:
-        fields = field_deviations.get(anomaly.subject, [])
-        correlation = correlation_by_ip.get(anomaly.subject)
-        evidence = [*anomaly.reasons, *[str(item.get("reason", "")) for item in fields]]
-        score = min(100, anomaly.score + min(15, sum(int(item.get("score", 0)) for item in fields)))
-        if correlation:
-            score = min(100, score + 10); evidence.append(f"observed across {len(correlation.get('streams', []))} streams")
-        incidents.append({"entity": anomaly.subject, "score": score, "severity": anomaly.severity, "evidence": evidence[:8], "field_deviations": len(fields), "correlated_streams": correlation.get("streams", []) if correlation else []})
+        group = groups[anomaly.subject]
+        group["score"] = max(int(group["score"]), anomaly.score)
+        group["evidence"].extend(anomaly.reasons)
+    for entity, deviations in field_deviations.items():
+        group = groups[entity]
+        group["fields"].extend(deviations)
+        active = [item for item in deviations if item.get("feedback") not in {"expected", "false_positive"}]
+        group["score"] = min(100, int(group["score"]) + min(30, sum(int(item.get("score", 0)) for item in active)))
+        group["evidence"].extend(str(item.get("reason", "")) for item in active)
+        for item in active:
+            group["timeline"].extend(item.get("sample_events", []))
+    for correlation in correlations:
+        entity = str(correlation.get("entity") or correlation.get("source_ip") or "")
+        if not entity:
+            continue
+        group = groups[entity]
+        group["correlations"].append(correlation)
+        group["score"] = min(100, int(group["score"]) + 10 + min(15, int(correlation.get("security_events", 0)) * 2))
+        group["evidence"].append(f"observed across {len(correlation.get('streams', []))} streams")
+        group["timeline"].extend(correlation.get("samples", []))
+
+    incidents = []
+    for entity, group in groups.items():
+        if not group["evidence"]:
+            continue
+        score = int(group["score"])
+        severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
+        streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")})
+        timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
+        incidents.append({
+            "entity": entity,
+            "entity_type": entity_type(entity),
+            "score": score,
+            "severity": severity,
+            "evidence": list(dict.fromkeys(str(item) for item in group["evidence"] if item))[:8],
+            "field_deviations": len(group["fields"]),
+            "correlated_streams": streams,
+            "timeline": timeline,
+            "first_seen": timeline[0].get("timestamp", "") if timeline else "",
+            "last_seen": timeline[-1].get("timestamp", "") if timeline else "",
+        })
     return sorted(incidents, key=lambda item: int(item["score"]), reverse=True)

src/fgai/llm.py

@@ -63,6 +63,7 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
         "diagnostics": analysis.get("diagnostics", {}),
         "capabilities": analysis.get("capabilities", {}),
         "cross_source_correlations": analysis.get("cross_source_correlations", [])[:20],
+        "incidents": analysis.get("incidents", [])[:10],
         "field_deviations": analysis.get("field_deviations", {}),
         "feedback": analysis.get("feedback", []),
     }

src/fgai/metrics.py

@@ -7,6 +7,7 @@ def prometheus_metrics(status: dict[str, object]) -> str:
     baseline = status.get("baseline", {}) if isinstance(status.get("baseline"), dict) else {}
     capabilities = status.get("capabilities", {}) if isinstance(status.get("capabilities"), dict) else {}
     mcp = capabilities.get("graylog_mcp", {}) if isinstance(capabilities.get("graylog_mcp"), dict) else {}
+    incidents = status.get("incidents", []) if isinstance(status.get("incidents"), list) else []
     lines = ["# HELP fgai_events_total Events in the latest analysis window.", "# TYPE fgai_events_total gauge"]
     for key in ("total", "utm", "threat_actions", "critical_or_high"):
         lines.append(f'fgai_events_total{{kind="{key}"}} {int(summary.get(key, 0) or 0)}')
@@ -16,4 +17,7 @@ def prometheus_metrics(status: dict[str, object]) -> str:
     lines += ["# HELP fgai_baseline_sources_ready Sources with sufficient baseline history.", "# TYPE fgai_baseline_sources_ready gauge", f'fgai_baseline_sources_ready {int(baseline.get("sources_ready", 0) or 0)}']
     lines += ["# HELP fgai_graylog_mcp_connected Graylog MCP connectivity state.", "# TYPE fgai_graylog_mcp_connected gauge", f'fgai_graylog_mcp_connected {1 if mcp.get("status") == "connected" else 0}']
     lines += ["# HELP fgai_graylog_events_fetched Events fetched from Graylog in the latest poll.", "# TYPE fgai_graylog_events_fetched gauge", f'fgai_graylog_events_fetched {int(mcp.get("events_fetched", 0) or 0)}']
+    lines += ["# HELP signalscope_incidents Open incident candidates by severity.", "# TYPE signalscope_incidents gauge"]
+    for severity in ("critical", "high", "medium", "low"):
+        lines.append(f'signalscope_incidents{{severity="{severity}"}} {sum(1 for item in incidents if isinstance(item, dict) and item.get("severity") == severity)}')
     return "\n".join(lines) + "\n"

src/fgai/monitor.py

@@ -58,7 +58,14 @@ def build_status(
                 events = []
                 for stream_config in stream_configs:
                     stream_id = str(stream_config["id"])
-                    stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id))).fetch()
+                    profile = stream_profiles.get(stream_id)
+                    profile_fields = (
+                        str(getattr(profile, "entity_field", "")),
+                        str(getattr(profile, "timestamp_field", "")),
+                        *tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
+                        *tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
+                    ) if profile else ()
+                    stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id)), profile_fields).fetch()
                     events.extend(stream_events)
                     stream_statuses.append({"stream_id": stream_id, **stream_status})
                 mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
@@ -71,7 +78,13 @@ def build_status(
     feedback = FeedbackStore().entries()
     for entity, deviations in field_deviations.items():
         for deviation in deviations:
-            match = next((item for item in feedback if item.get("entity") == entity and item.get("stream_id") == deviation.get("stream_id") and item.get("field") == deviation.get("field")), None)
+            match = next((
+                item for item in feedback
+                if item.get("entity") == entity
+                and item.get("stream_id") == deviation.get("stream_id")
+                and item.get("field") == deviation.get("field")
+                and (not item.get("value") or item.get("value") == deviation.get("value", ""))
+            ), None)
             if match:
                 deviation["feedback"] = match["status"]
                 if match["status"] in {"false_positive", "expected"}: