add value-shape field classification and auto-apply

Field roles are now inferred from observed values instead of names,
with src/dst disambiguation, value-pattern detector hints, an Ollama
advisor scoped to ambiguous fields only, confidence-gated auto-apply
with manual-field protection, and drift flagging for review

tests/test_monitor.py

@@ -4,6 +4,7 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch
 
+from fgai.config import ConfigStore
 from fgai.history import StatusSnapshotStore
 from fgai.history import FieldDiscoveryStore
 from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
@@ -230,7 +231,15 @@ class MonitorTests(unittest.TestCase):
     def test_profile_advisor_skips_when_all_streams_have_profiles(self):
         with tempfile.TemporaryDirectory() as tmp:
             log_path = Path(tmp) / "events.log"
-            log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice\n", encoding="utf-8")
+            names = ["alice", "bob", "carol"]
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=windows fgai_stream=Windows username={names[index % 3]}"
+                    for index in range(25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
             config_path = Path(tmp) / "config.json"
             config_path.write_text(
                 json.dumps(
@@ -269,6 +278,139 @@ class MonitorTests(unittest.TestCase):
             advisor.assert_called_once()
             self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty")
 
+    def test_profile_advisor_runs_for_low_confidence_field_even_with_full_profile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            # Mixed-shape, low-sample-count values keep "username"'s classified
+            # confidence below the default threshold even though its name already
+            # makes it the entity field and the profile's field set is unchanged.
+            log_path.write_text(
+                "fgai_stream_id=windows fgai_stream=Windows username=alice\n"
+                "fgai_stream_id=windows fgai_stream=Windows username=99\n",
+                encoding="utf-8",
+            )
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(
+                json.dumps(
+                    {
+                        "profile_advisor_enabled": True,
+                        "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}],
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            with patch("fgai.monitor.ollama_profile_advice", return_value=[]) as advisor:
+                status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            advisor.assert_called_once()
+            _, kwargs = advisor.call_args
+            self.assertIn("username", kwargs["ambiguous_fields_by_stream"].get("windows", []))
+            self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty")
+
+    def test_auto_apply_writes_high_confidence_new_stream_profile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
+                    for index in range(1, 25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8")
+
+            build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            saved = ConfigStore(str(config_path)).read()
+            profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]}
+            self.assertIn("custom_vendor", profiles)
+            self.assertIn("x7_field_a", profiles["custom_vendor"]["entity_fields"])
+            self.assertEqual(profiles["custom_vendor"]["field_origin"]["x7_field_a"], "auto")
+
+    def test_auto_apply_never_overwrites_a_manual_origin_field(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
+                    for index in range(1, 25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(
+                json.dumps({
+                    "auto_apply_profile_suggestions": True,
+                    "graylog_stream_profiles": [{
+                        "stream_id": "custom_vendor",
+                        "name": "Custom Vendor profile",
+                        "entity_field": "x7_field_a",
+                        "entity_fields": ["x7_field_a"],
+                        "timestamp_field": "timestamp",
+                        "field_origin": {"x7_field_a": "manual"},
+                    }],
+                }),
+                encoding="utf-8",
+            )
+
+            build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            saved = ConfigStore(str(config_path)).read()
+            profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]}
+            self.assertEqual(profiles["custom_vendor"]["field_origin"], {"x7_field_a": "manual"})
+
+    def test_profile_drift_flags_confidence_drop_without_changing_live_profile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            config_path = Path(tmp) / "config.json"
+            history_path = Path(tmp) / "history.sqlite3"
+            config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8")
+
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
+                    for index in range(1, 25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            first = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
+            applied_profile = {item["stream_id"]: item for item in first["stream_profiles"]}["custom_vendor"]
+            self.assertEqual(applied_profile["field_origin"]["x7_field_a"], "auto")
+            self.assertEqual(first["profile_drift"], [])
+
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=some odd free text value number {index}"
+                    for index in range(1, 30)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            second = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            drift_entries = {(item["stream_id"], item["field"]) for item in second["profile_drift"]}
+            self.assertIn(("custom_vendor", "x7_field_a"), drift_entries)
+            still_applied_profile = {item["stream_id"]: item for item in second["stream_profiles"]}["custom_vendor"]
+            self.assertIn("x7_field_a", still_applied_profile["entity_fields"])
+
+    def test_confidence_history_maintenance_runs_each_cycle(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            log_path.write_text("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login\n", encoding="utf-8")
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(json.dumps({}), encoding="utf-8")
+            history_path = Path(tmp) / "history.sqlite3"
+
+            status = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            self.assertIn("confidence_history_maintenance", status["profile_auto_apply"])
+            self.assertIn("remaining", status["profile_auto_apply"]["confidence_history_maintenance"])
+
     def test_profile_suggestions_use_cached_discovered_fields(self):
         with tempfile.TemporaryDirectory() as tmp:
             history_path = str(Path(tmp) / "history.sqlite3")