add value-shape field classification and auto-apply

Field roles are now inferred from observed values instead of names,
with src/dst disambiguation, value-pattern detector hints, an Ollama
advisor scoped to ambiguous fields only, confidence-gated auto-apply
with manual-field protection, and drift flagging for review

src/fgai/monitor.py

@@ -114,13 +114,141 @@ def _suggestion_field_set(suggestion: dict[str, object]) -> set[str]:
     return {field for field in fields if field}
 
 
-def _needs_profile_advisor(suggestion: dict[str, object], stream_profiles: dict[str, object]) -> bool:
-    if not suggestion.get("profile_exists"):
-        return True
+def _ambiguous_fields(suggestion: dict[str, object], threshold: int) -> list[str]:
+    field_confidence = suggestion.get("field_confidence", {})
+    if not isinstance(field_confidence, dict):
+        return []
+    selected = _suggestion_field_set(suggestion)
+    # Only fields that were actually observed and shape-classified are "ambiguous" —
+    # a field with no field_confidence entry (e.g. a placeholder timestamp field name
+    # used when no time-like field was discovered at all) has nothing to disambiguate.
+    return [field for field in selected if field in field_confidence and int(field_confidence[field] or 0) < threshold]
+
+
+def _needs_profile_advisor(suggestion: dict[str, object], stream_profiles: dict[str, object], *, confidence_threshold: int = 60) -> bool:
     stream_id = str(suggestion.get("stream_id", ""))
     suggested_fields = _suggestion_field_set(suggestion)
+    if not suggestion.get("profile_exists"):
+        return True
     existing_fields = _profile_field_set(stream_profiles.get(stream_id))
-    return bool(suggested_fields - existing_fields)
+    if suggested_fields - existing_fields:
+        return True
+    return bool(_ambiguous_fields(suggestion, confidence_threshold))
+
+
+def _suggestion_is_high_confidence(suggestion: dict[str, object], threshold: int) -> bool:
+    field_confidence = suggestion.get("field_confidence", {})
+    if not isinstance(field_confidence, dict):
+        return False
+    observed = [field for field in _suggestion_field_set(suggestion) if field in field_confidence]
+    if not observed:
+        return False
+    return all(int(field_confidence[field] or 0) >= threshold for field in observed)
+
+
+def _profile_is_auto_apply_safe(existing: object) -> bool:
+    """A profile with any manual-origin field is left alone entirely by auto-apply."""
+    if existing is None:
+        return True
+    field_origin = getattr(existing, "field_origin", None) or {}
+    return all(origin != "manual" for origin in field_origin.values())
+
+
+def _auto_apply_profile_suggestions(
+    profile_suggestions: list[dict[str, object]],
+    stream_profiles: dict[str, object],
+    runtime_values: dict[str, object],
+    config_path: str | None,
+) -> list[str]:
+    """Write high-confidence, provenance-safe suggestions straight to config.
+
+    Mirrors the dashboard's own POST /api/config write path (ConfigStore.update),
+    just invoked here unattended each cycle instead of from a human clicking Apply.
+    """
+    if not config_path or not runtime_values.get("auto_apply_profile_suggestions"):
+        return []
+    threshold = int(runtime_values.get("field_confidence_threshold", 60) or 60)
+    updated_profiles = [item for item in runtime_values.get("graylog_stream_profiles", []) if isinstance(item, dict)]
+    applied: list[str] = []
+    now = int(time.time())
+    for suggestion in profile_suggestions:
+        if not isinstance(suggestion, dict):
+            continue
+        stream_id = str(suggestion.get("stream_id", ""))
+        profile = suggestion.get("profile", {})
+        if not stream_id or not isinstance(profile, dict):
+            continue
+        if not _suggestion_is_high_confidence(suggestion, threshold):
+            continue
+        if not _profile_is_auto_apply_safe(stream_profiles.get(stream_id)):
+            continue
+        all_fields = {*profile.get("entity_fields", []), *profile.get("categorical_fields", []), *profile.get("numeric_fields", [])}
+        new_profile = dict(profile)
+        new_profile["field_origin"] = {str(field): "auto" for field in all_fields if field}
+        new_profile["updated_at"] = now
+        updated_profiles = [item for item in updated_profiles if item.get("stream_id") != stream_id]
+        updated_profiles.append(new_profile)
+        applied.append(stream_id)
+    if applied:
+        ConfigStore(config_path).update({"graylog_stream_profiles": updated_profiles})
+    return applied
+
+
+DRIFT_CONFIDENCE_DROP_THRESHOLD = 20
+
+
+def _detect_profile_drift(
+    profile_suggestions: list[dict[str, object]],
+    stream_profiles: dict[str, object],
+    discovery_store: object | None,
+) -> list[dict[str, object]]:
+    """Flag auto-applied fields whose classification has drifted since they were applied.
+
+    Flag-only: this never touches the live profile, only surfaces a diagnostic for a
+    human to review. A silently "self-corrected" auto-applied field could suppress or
+    misdirect real findings with nobody noticing, which is worse than a stale mapping
+    that gets flagged and reviewed.
+    """
+    if discovery_store is None:
+        return []
+    drift: list[dict[str, object]] = []
+    for suggestion in profile_suggestions:
+        if not isinstance(suggestion, dict):
+            continue
+        stream_id = str(suggestion.get("stream_id", ""))
+        if not stream_id:
+            continue
+        existing = stream_profiles.get(stream_id)
+        field_origin = getattr(existing, "field_origin", None) or {}
+        auto_fields = [field for field, origin in field_origin.items() if origin == "auto"]
+        if not auto_fields:
+            continue
+        applied = discovery_store.applied_confidence(stream_id)
+        if not applied:
+            continue
+        field_confidence = suggestion.get("field_confidence", {})
+        field_shapes = suggestion.get("field_shapes", {})
+        if not isinstance(field_confidence, dict) or not isinstance(field_shapes, dict):
+            continue
+        for field in auto_fields:
+            baseline = applied.get(field)
+            if not baseline or field not in field_confidence:
+                continue
+            current_confidence = int(field_confidence.get(field, 0) or 0)
+            current_shape = str(field_shapes.get(field, ""))
+            confidence_drop = int(baseline["applied_confidence"]) - current_confidence
+            shape_changed = bool(current_shape) and current_shape != str(baseline["applied_shape"])
+            if confidence_drop > DRIFT_CONFIDENCE_DROP_THRESHOLD or shape_changed:
+                drift.append({
+                    "stream_id": stream_id,
+                    "field": field,
+                    "applied_confidence": baseline["applied_confidence"],
+                    "current_confidence": current_confidence,
+                    "applied_shape": baseline["applied_shape"],
+                    "current_shape": current_shape,
+                    "applied_at": baseline["applied_at"],
+                })
+    return drift
 
 
 def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
@@ -427,13 +555,19 @@ def build_status(
     stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
     discovery_cache_events = []
     discovered_profile_fields = 0
+    discovery_store = None
     if history_path:
         discovery_store = FieldDiscoveryStore(history_path)
         discovered_profile_fields = discovery_store.ingest(events)
         discovery_cache_events = discovery_store.synthetic_events()
     profile_suggestions = suggest_stream_profiles([*discovery_cache_events, *events], existing_profiles=stream_profiles)
+    confidence_threshold = int(runtime_values.get("field_confidence_threshold", 60) or 60)
     profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"}
-    advisor_candidates = [item for item in profile_suggestions if _needs_profile_advisor(item, stream_profiles)]
+    advisor_candidates = [item for item in profile_suggestions if _needs_profile_advisor(item, stream_profiles, confidence_threshold=confidence_threshold)]
+    ambiguous_fields_by_stream = {
+        str(item.get("stream_id", "")): _ambiguous_fields(item, confidence_threshold)
+        for item in advisor_candidates
+    }
     if runtime_values.get("profile_advisor_enabled") and not run_profile_advisor:
         profile_advisor_status = {"enabled": True, "status": "pending" if advisor_candidates else "skipped_no_profile_changes", "candidates": len(advisor_candidates)}
     elif runtime_values.get("profile_advisor_enabled") and profile_suggestions and not advisor_candidates:
@@ -444,6 +578,7 @@ def build_status(
                 advisor_candidates[:10],
                 model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
                 timeout=int(runtime_values.get("profile_advisor_timeout", 240) or 240),
+                ambiguous_fields_by_stream=ambiguous_fields_by_stream,
             )
             profile_suggestions = apply_profile_advice(profile_suggestions, advice)
             profile_advisor_status = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
@@ -457,6 +592,28 @@ def build_status(
             }
             for suggestion in profile_suggestions:
                 suggestion.setdefault("profile_advisor", {"status": "heuristic", "error": str(exc)})
+    auto_applied_streams = _auto_apply_profile_suggestions(profile_suggestions, stream_profiles, runtime_values, config_path)
+    if auto_applied_streams:
+        if discovery_store:
+            suggestions_by_stream = {str(item.get("stream_id", "")): item for item in profile_suggestions if isinstance(item, dict)}
+            for stream_id in auto_applied_streams:
+                suggestion = suggestions_by_stream.get(stream_id)
+                if not suggestion:
+                    continue
+                profile = suggestion.get("profile", {})
+                if not isinstance(profile, dict):
+                    continue
+                applied_fields = [*profile.get("entity_fields", []), *profile.get("categorical_fields", []), *profile.get("numeric_fields", [])]
+                discovery_store.record_applied_confidence(
+                    stream_id,
+                    applied_fields,
+                    suggestion.get("field_confidence", {}) if isinstance(suggestion.get("field_confidence"), dict) else {},
+                    suggestion.get("field_shapes", {}) if isinstance(suggestion.get("field_shapes"), dict) else {},
+                )
+        if config_store:
+            stream_profiles = parse_profiles(config_store.read().get("graylog_stream_profiles", []))
+    confidence_history_maintenance = discovery_store.confidence_history_maintenance(retention_days=int(runtime_values.get("field_confidence_retention_days", 90) or 90)) if discovery_store else {}
+    profile_drift = _detect_profile_drift(profile_suggestions, stream_profiles, discovery_store)
     intel_ips = sorted(
         {
             ip
@@ -512,7 +669,9 @@ def build_status(
         "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "discovery_fields_recorded": discovered_profile_fields, "discovery_cache_events": len(discovery_cache_events), "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
         "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
         "configuration": runtime_config,
-        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights, "relationship_fields": [{"left": relation.left, "right": relation.right, "name": relation.name} for relation in item.relationship_fields]} for item in stream_profiles.values()],
+        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights, "relationship_fields": [{"left": relation.left, "right": relation.right, "name": relation.name} for relation in item.relationship_fields], "field_origin": item.field_origin, "updated_at": item.updated_at} for item in stream_profiles.values()],
+        "profile_auto_apply": {"applied_streams": auto_applied_streams, "confidence_history_maintenance": confidence_history_maintenance},
+        "profile_drift": profile_drift,
         "stream_coverage": stream_coverage,
         "profile_suggestions": profile_suggestions,
         "profile_readiness": profile_readiness,
@@ -626,7 +785,12 @@ def add_profile_advisor(status: dict[str, object], runtime_values: dict[str, obj
     if not isinstance(profile_suggestions, list):
         return
     stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
-    advisor_candidates = [item for item in profile_suggestions if isinstance(item, dict) and _needs_profile_advisor(item, stream_profiles)]
+    confidence_threshold = int(runtime_values.get("field_confidence_threshold", 60) or 60)
+    advisor_candidates = [item for item in profile_suggestions if isinstance(item, dict) and _needs_profile_advisor(item, stream_profiles, confidence_threshold=confidence_threshold)]
+    ambiguous_fields_by_stream = {
+        str(item.get("stream_id", "")): _ambiguous_fields(item, confidence_threshold)
+        for item in advisor_candidates
+    }
     capabilities = status.setdefault("capabilities", {})
     if not isinstance(capabilities, dict):
         return
@@ -641,6 +805,7 @@ def add_profile_advisor(status: dict[str, object], runtime_values: dict[str, obj
             advisor_candidates[:10],
             model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
             timeout=int(runtime_values.get("profile_advisor_timeout", 240) or 240),
+            ambiguous_fields_by_stream=ambiguous_fields_by_stream,
         )
         status["profile_suggestions"] = apply_profile_advice(profile_suggestions, advice)
         capabilities["profile_advisor"] = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
@@ -740,6 +905,7 @@ def monitor_loop(
     print(f"Writing status to {output}", flush=True)
     last_llm_at = 0
     last_llm_text: str | None = None
+    last_advisor_at = 0
     while True:
         runtime = ConfigStore(config_path).read() if config_path and Path(config_path).exists() else {}
         effective_llm = bool(runtime.get("llm_enabled")) if runtime else llm
@@ -804,16 +970,21 @@ def monitor_loop(
                     "generated_at": last_llm_at,
                     "text": last_llm_text or "",
                 }
-            if runtime.get("profile_advisor_enabled"):
+            advisor_interval = int(runtime.get("profile_advisor_interval_seconds", 300) or 300)
+            if runtime.get("profile_advisor_enabled") and now - last_advisor_at >= advisor_interval:
                 add_profile_advisor(status, runtime)
+                last_advisor_at = now
                 write_status(status, output)
         else:
             status["llm_assessment"] = {"enabled": False, "status": "disabled", "text": ""}
-            if runtime.get("profile_advisor_enabled"):
+            advisor_interval = int(runtime.get("profile_advisor_interval_seconds", 300) or 300)
+            now = int(time.time())
+            if runtime.get("profile_advisor_enabled") and now - last_advisor_at >= advisor_interval:
                 if status_cache_path and isinstance(mcp, dict) and mcp.get("status") not in {"error", "refreshing"}:
                     StatusSnapshotStore(status_cache_path).save("last_good", status)
                 write_status(status, output)
                 add_profile_advisor(status, runtime)
+                last_advisor_at = now
         if status_cache_path and isinstance(mcp, dict) and mcp.get("status") not in {"error", "refreshing"}:
             StatusSnapshotStore(status_cache_path).save("last_good", status)
         write_status(status, output)