add new structure

tests/test_dashboard.py

@@ -94,6 +94,17 @@ class DashboardTests(unittest.TestCase):
         self.assertIn("openDetailIds", HTML)
         self.assertIn("addEventListener('toggle'", HTML)
 
+    def test_dashboard_annotates_field_deviations_with_cluster_size(self):
+        self.assertIn("cluster_size", HTML)
+        self.assertIn("-entity pattern", HTML)
+        self.assertIn("cluster_dampened", HTML)
+
+    def test_dashboard_shows_shape_mismatch_warnings(self):
+        self.assertIn('id="profileShapeWarnings"', HTML)
+        self.assertIn("profile_shape_warnings", HTML)
+        self.assertIn("looks like free text", HTML)
+        self.assertIn("Open profile editor", HTML)
+
     def test_dashboard_has_anomaly_bulk_bar(self):
         self.assertIn('id="anomalyBulkBar"', HTML)
         self.assertIn('anomaly-select', HTML)

tests/test_monitor.py

@@ -8,7 +8,7 @@ from fgai.config import ConfigStore
 from fgai.feedback import FeedbackStore
 from fgai.history import StatusSnapshotStore
 from fgai.history import FieldDiscoveryStore
-from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
+from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status, _cluster_field_deviations
 
 
 class MonitorTests(unittest.TestCase):
@@ -39,6 +39,75 @@ class MonitorTests(unittest.TestCase):
             self.assertFalse(status["status_cache"]["served_from_cache"])
             self.assertGreaterEqual(len(status["anomalies"]), 1)
 
+    def test_cluster_field_deviations_dampens_large_identical_clusters(self):
+        field_deviations = {
+            f"10.0.0.{index}": [{"detector": "relationship", "field": "client_ip->context", "score": 40}]
+            for index in range(1, 6)
+        }
+
+        _cluster_field_deviations(field_deviations)
+
+        for entity, deviations in field_deviations.items():
+            deviation = deviations[0]
+            self.assertEqual(deviation["cluster_size"], 5)
+            self.assertEqual(deviation["score"], 25)
+            self.assertTrue(deviation["cluster_dampened"])
+
+    def test_cluster_field_deviations_leaves_small_clusters_untouched(self):
+        field_deviations = {
+            f"10.0.0.{index}": [{"detector": "relationship", "field": "client_ip->context", "score": 40}]
+            for index in range(1, 4)
+        }
+
+        _cluster_field_deviations(field_deviations)
+
+        for entity, deviations in field_deviations.items():
+            deviation = deviations[0]
+            self.assertEqual(deviation["cluster_size"], 3)
+            self.assertEqual(deviation["score"], 40)
+            self.assertNotIn("cluster_dampened", deviation)
+
+    def test_cluster_field_deviations_excludes_suppressed_entities_from_cluster_size(self):
+        field_deviations = {
+            f"10.0.0.{index}": [{"detector": "relationship", "field": "client_ip->context", "score": 40}]
+            for index in range(1, 5)
+        }
+        field_deviations["10.0.0.99"] = [{"detector": "relationship", "field": "client_ip->context", "score": 40, "feedback": "false_positive"}]
+
+        _cluster_field_deviations(field_deviations)
+
+        self.assertEqual(field_deviations["10.0.0.1"][0]["cluster_size"], 4)
+        self.assertNotIn("cluster_dampened", field_deviations["10.0.0.1"][0])
+        self.assertEqual(field_deviations["10.0.0.99"][0]["score"], 40)
+
+    def test_cluster_field_deviations_dampen_large_clusters_in_build_status(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            srcips = [f"203.0.113.{index}" for index in range(1, 6)]
+            log_path = Path(tmp) / "fg.log"
+            log_path.write_text(
+                "\n".join(
+                    f"type=utm subtype=ips srcip={ip} dstip=10.0.0.10 service=https action=blocked severity=critical"
+                    for ip in srcips
+                ),
+                encoding="utf-8",
+            )
+            canned_deviations = {
+                ip: [{"detector": "relationship", "field": "client_ip->context", "stream_id": "utm", "score": 40, "reason": "new relationship: client_ip->context"}]
+                for ip in srcips
+            }
+
+            with patch("fgai.monitor.BaselineStore.profile_deviations", return_value=canned_deviations):
+                status = build_status(str(log_path), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json"))
+
+            for ip in srcips:
+                deviation = status["field_deviations"][ip][0]
+                self.assertEqual(deviation["cluster_size"], 5)
+                self.assertEqual(deviation["score"], 25)
+                self.assertTrue(deviation["cluster_dampened"])
+
+            anomalies_by_subject = {item["subject"]: item for item in status["anomalies"]}
+            self.assertIn("new relationship: client_ip->context", anomalies_by_subject[srcips[0]]["reasons"])
+
     def test_entity_scope_feedback_suppresses_anomaly_in_build_status(self):
         with tempfile.TemporaryDirectory() as tmp:
             log_path = Path(tmp) / "fg.log"

tests/test_profile_suggestions.py

@@ -1,7 +1,8 @@
 import unittest
 
 from fgai.logs import parse_log_line
-from fgai.profile_suggestions import apply_profile_advice, suggest_stream_profiles
+from fgai.profile_suggestions import apply_profile_advice, detect_shape_mismatches, suggest_stream_profiles
+from fgai.stream_profiles import RelationshipField, StreamProfile
 
 
 class ProfileSuggestionTests(unittest.TestCase):
@@ -364,6 +365,84 @@ class ProfileSuggestionTests(unittest.TestCase):
         self.assertIn("deny_action", detectors)
         self.assertEqual(detectors["deny_action"].get("source"), "value_shape")
 
+    def test_name_matched_free_text_field_is_vetoed_from_categorical_and_relationships(self):
+        messages = [
+            "GET /checkout/cart/item-{0}?ref=promo&session=abc{0} HTTP/1.1 200",
+            "POST /api/v2/orders/{0}/confirm referrer=partner-{0} status=accepted",
+            "DELETE /users/{0}/sessions/active reason=timeout-{0} agent=curl/8.1",
+            "PUT /inventory/warehouse-{0}/restock qty={0} note=manual-adjustment",
+        ]
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=proxy fgai_stream=Proxy srcip=10.0.0.{index % 7} "
+                f"context='{messages[index % len(messages)].format(index)}' "
+                f"timestamp=2026-06-29T10:00:{index:02d}Z"
+            )
+            for index in range(1, 40)
+        ]
+
+        suggestion = suggest_stream_profiles(events)[0]
+        profile = suggestion["profile"]
+
+        self.assertEqual(suggestion["field_shapes"]["context"], "free_text")
+        self.assertNotIn("context", profile["entity_fields"])
+        self.assertNotIn("context", profile["categorical_fields"])
+        relationship_fields = {item["left"] for item in profile["relationship_fields"]} | {
+            item["right"] for item in profile["relationship_fields"]
+        }
+        self.assertNotIn("context", relationship_fields)
+        self.assertIn("srcip", profile["entity_fields"])
+
+    def test_low_cardinality_name_matched_field_is_not_vetoed(self):
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index % 7} "
+                f"action={'deny' if index % 2 else 'allow'} timestamp=2026-06-29T10:00:{index:02d}Z"
+            )
+            for index in range(1, 30)
+        ]
+
+        suggestion = suggest_stream_profiles(events)[0]
+        profile = suggestion["profile"]
+
+        self.assertIn("action", profile["categorical_fields"])
+
+    def test_existing_profile_relationship_field_now_free_text_is_flagged(self):
+        field_shapes = {"context": "free_text", "srcip": "ip"}
+        field_confidence = {"context": 43, "srcip": 90}
+        existing = StreamProfile(
+            stream_id="proxy",
+            name="Proxy",
+            entity_field="srcip",
+            timestamp_field="timestamp",
+            categorical_fields=("context",),
+            relationship_fields=(RelationshipField(left="srcip", right="context", name="srcip -> context"),),
+        )
+
+        warnings = detect_shape_mismatches(existing, field_shapes, field_confidence)
+
+        flagged = {(item["field"], item["role"]) for item in warnings}
+        self.assertIn(("context", "categorical"), flagged)
+        self.assertIn(("context", "relationship"), flagged)
+        self.assertNotIn(("srcip", "relationship"), flagged)
+
+    def test_existing_profile_with_fitting_fields_produces_no_warnings(self):
+        field_shapes = {"action": "categorical_low_cardinality", "srcip": "ip"}
+        field_confidence = {"action": 85, "srcip": 90}
+        existing = StreamProfile(
+            stream_id="firewall",
+            name="Firewall",
+            entity_field="srcip",
+            timestamp_field="timestamp",
+            categorical_fields=("action",),
+            relationship_fields=(RelationshipField(left="srcip", right="action", name="srcip -> action"),),
+        )
+
+        self.assertEqual(detect_shape_mismatches(existing, field_shapes, field_confidence), [])
+
+    def test_missing_existing_profile_produces_no_warnings(self):
+        self.assertEqual(detect_shape_mismatches(None, {}, {}), [])
+
     def test_every_suggestion_carries_field_confidence_and_shapes(self):
         events = [
             parse_log_line(f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index} dstip=8.8.8.8 dstport=443 action=deny timestamp=2026-06-29T10:00:{index:02d}Z hitcount={index}")