Implemented the next multi-source detection layer in this repository.

src/fgai/incidents.py

@@ -1,17 +1,54 @@
 from __future__ import annotations
 
+from collections import defaultdict
+
+from .entities import entity_type
 from .models import AnomalyFinding
 
 
 def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]:
-    correlation_by_ip = {str(item.get("source_ip")): item for item in correlations}
-    incidents = []
+    """Build investigation units from any log source, not only network source IPs."""
+    groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []})
     for anomaly in anomalies:
-        fields = field_deviations.get(anomaly.subject, [])
-        correlation = correlation_by_ip.get(anomaly.subject)
-        evidence = [*anomaly.reasons, *[str(item.get("reason", "")) for item in fields]]
-        score = min(100, anomaly.score + min(15, sum(int(item.get("score", 0)) for item in fields)))
-        if correlation:
-            score = min(100, score + 10); evidence.append(f"observed across {len(correlation.get('streams', []))} streams")
-        incidents.append({"entity": anomaly.subject, "score": score, "severity": anomaly.severity, "evidence": evidence[:8], "field_deviations": len(fields), "correlated_streams": correlation.get("streams", []) if correlation else []})
+        group = groups[anomaly.subject]
+        group["score"] = max(int(group["score"]), anomaly.score)
+        group["evidence"].extend(anomaly.reasons)
+    for entity, deviations in field_deviations.items():
+        group = groups[entity]
+        group["fields"].extend(deviations)
+        active = [item for item in deviations if item.get("feedback") not in {"expected", "false_positive"}]
+        group["score"] = min(100, int(group["score"]) + min(30, sum(int(item.get("score", 0)) for item in active)))
+        group["evidence"].extend(str(item.get("reason", "")) for item in active)
+        for item in active:
+            group["timeline"].extend(item.get("sample_events", []))
+    for correlation in correlations:
+        entity = str(correlation.get("entity") or correlation.get("source_ip") or "")
+        if not entity:
+            continue
+        group = groups[entity]
+        group["correlations"].append(correlation)
+        group["score"] = min(100, int(group["score"]) + 10 + min(15, int(correlation.get("security_events", 0)) * 2))
+        group["evidence"].append(f"observed across {len(correlation.get('streams', []))} streams")
+        group["timeline"].extend(correlation.get("samples", []))
+
+    incidents = []
+    for entity, group in groups.items():
+        if not group["evidence"]:
+            continue
+        score = int(group["score"])
+        severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
+        streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")})
+        timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
+        incidents.append({
+            "entity": entity,
+            "entity_type": entity_type(entity),
+            "score": score,
+            "severity": severity,
+            "evidence": list(dict.fromkeys(str(item) for item in group["evidence"] if item))[:8],
+            "field_deviations": len(group["fields"]),
+            "correlated_streams": streams,
+            "timeline": timeline,
+            "first_seen": timeline[0].get("timestamp", "") if timeline else "",
+            "last_seen": timeline[-1].get("timestamp", "") if timeline else "",
+        })
     return sorted(incidents, key=lambda item: int(item["score"]), reverse=True)