add new structure

src/fgai/monitor.py

@@ -5,6 +5,7 @@ import json
 import signal
 import threading
 import time
+from collections import defaultdict
 from pathlib import Path
 
 from .anomaly import anomaly_summary, detect_source_anomalies
@@ -251,6 +252,40 @@ def _detect_profile_drift(
     return drift
 
 
+CLUSTER_SIZE_DAMPEN_THRESHOLD = 5
+CLUSTER_DAMPENED_SCORE_CAP = 25
+
+
+def _cluster_field_deviations(field_deviations: dict[str, list[dict[str, object]]]) -> None:
+    """Recognize the same (detector, field) signature repeating across many
+    entities as one systemic pattern instead of N independent findings.
+
+    Mutates deviations in place: tags cluster_size, and caps score for large
+    clusters — a signature hitting many entities identically usually means a
+    detector/profile problem, not N separate critical events. Signature
+    deliberately excludes stream_id: the incident this guards against (a
+    misconfigured relationship field) recurred identically across multiple
+    streams, and that cross-stream repetition is itself part of the pattern.
+    Already-suppressed deviations don't count toward another entity's cluster
+    size, so a false-positive rule doesn't inflate the appearance of a pattern.
+    """
+    signature_entities: dict[tuple[str, str], set[str]] = defaultdict(set)
+    for entity, deviations in field_deviations.items():
+        for deviation in deviations:
+            if deviation.get("feedback") in SUPPRESSING_STATUSES:
+                continue
+            signature = (str(deviation.get("detector", "")), str(deviation.get("field", "")))
+            signature_entities[signature].add(entity)
+    for entity, deviations in field_deviations.items():
+        for deviation in deviations:
+            signature = (str(deviation.get("detector", "")), str(deviation.get("field", "")))
+            cluster_size = len(signature_entities.get(signature, ()))
+            deviation["cluster_size"] = cluster_size
+            if cluster_size >= CLUSTER_SIZE_DAMPEN_THRESHOLD:
+                deviation["score"] = min(int(deviation.get("score", 0) or 0), CLUSTER_DAMPENED_SCORE_CAP)
+                deviation["cluster_dampened"] = True
+
+
 def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
     return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
 
@@ -524,6 +559,7 @@ def build_status(
                 deviation["feedback_rule_id"] = match.get("id", "")
                 if match["status"] in SUPPRESSING_STATUSES:
                     deviation["score"] = 0
+    _cluster_field_deviations(field_deviations)
     generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3)
     feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values)
     anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations, feedback=feedback)
@@ -660,6 +696,11 @@ def build_status(
         summary["total"] = int(mcp_status.get("aggregate_events", 0) or 0)
         summary["raw_sample_total"] = len(events)
         summary["aggregate_backed"] = True
+    profile_shape_warnings = [
+        {"stream_id": suggestion.get("stream_id", ""), "stream_name": suggestion.get("stream_name", ""), **warning}
+        for suggestion in profile_suggestions
+        for warning in suggestion.get("shape_warnings", [])
+    ]
     status = {
         "status_schema": 2,
         "stale": False,
@@ -676,6 +717,7 @@ def build_status(
         "profile_drift": profile_drift,
         "stream_coverage": stream_coverage,
         "profile_suggestions": profile_suggestions,
+        "profile_shape_warnings": profile_shape_warnings,
         "profile_readiness": profile_readiness,
         "diagnostics": {
             "top_source_ips": top_field_values(events, "srcip", limit=10),