add value-shape field classification and auto-apply

Field roles are now inferred from observed values instead of names,
with src/dst disambiguation, value-pattern detector hints, an Ollama
advisor scoped to ambiguous fields only, confidence-gated auto-apply
with manual-field protection, and drift flagging for review

src/fgai/anomaly.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
-import ipaddress
 from collections import Counter, defaultdict
 from datetime import datetime
 from statistics import mean, pstdev
 
+from .entities import is_public_ip as _is_public_ip
 from .feedback import SUPPRESSING_STATUSES
 from .logs import THREAT_ACTIONS, event_score, is_utm_event
 from .models import AnomalyFinding, LogEvent
@@ -37,13 +37,6 @@ def _confidence(event_count: int, reason_count: int) -> str:
     return "low"
 
 
-def _is_public_ip(value: str) -> bool:
-    try:
-        return ipaddress.ip_address(value).is_global
-    except ValueError:
-        return False
-
-
 def _event_timestamp(event: LogEvent) -> float | None:
     """Return the FortiGate event time in seconds when present."""
     eventtime = event.fields.get("eventtime")

src/fgai/baseline.py

@@ -12,6 +12,7 @@ from .logs import THREAT_ACTIONS, is_utm_event
 from .models import LogEvent
 from .entities import profile_entities
 from .detectors import DETECTOR_MINIMUMS, event_detector_categories
+from .field_shapes import parse_iso_timestamp
 from .normalization import canonical_value
 from .query_details import event_query_details
 
@@ -62,10 +63,8 @@ def _event_epoch(event: LogEvent, fallback: int) -> int:
     try:
         return int(float(value))
     except ValueError:
-        try:
-            return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
-        except ValueError:
-            return fallback
+        parsed = parse_iso_timestamp(value)
+        return int(parsed) if parsed is not None else fallback
 
 
 def _baseline_confidence(samples: int, *, temporal: bool) -> str:

src/fgai/config.py

@@ -30,6 +30,10 @@ DEFAULT_CONFIG: dict[str, object] = {
     "profile_advisor_enabled": False,
     "profile_advisor_model": "qwen3:8b",
     "profile_advisor_timeout": 240,
+    "profile_advisor_interval_seconds": 300,
+    "field_confidence_threshold": 60,
+    "field_confidence_retention_days": 90,
+    "auto_apply_profile_suggestions": False,
     "threat_intel_enabled": False,
     "threat_intel_provider": "auto",
     "abuseipdb_api_key": "",
@@ -68,18 +72,23 @@ class ConfigStore:
                 continue
             if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "":
                 continue
-            if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled", "graylog_tls_verify"}:
+            if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled", "graylog_tls_verify", "auto_apply_profile_suggestions"}:
                 current[key] = bool(value)
             elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
                 current[key] = value
             elif key == "graylog_fetch_mode" and value in {"auto", "raw", "aggregate"}:
                 current[key] = value
-            elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "graylog_mcp_call_timeout_seconds", "graylog_mcp_poll_timeout_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "llm_timeout", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
+            elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "graylog_mcp_call_timeout_seconds", "graylog_mcp_poll_timeout_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "llm_timeout", "profile_advisor_timeout", "profile_advisor_interval_seconds", "field_confidence_retention_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
                 try:
-                    minimum = 60 if key in {"graylog_range_seconds", "graylog_mcp_poll_timeout_seconds"} else 1
+                    minimum = 60 if key in {"graylog_range_seconds", "graylog_mcp_poll_timeout_seconds", "profile_advisor_interval_seconds"} else 1
                     current[key] = max(minimum, int(value))
                 except (TypeError, ValueError):
                     continue
+            elif key == "field_confidence_threshold":
+                try:
+                    current[key] = min(100, max(0, int(value)))
+                except (TypeError, ValueError):
+                    continue
             elif key == "threat_intel_provider" and value in {"auto", "abuseipdb", "virustotal"}:
                 current[key] = value
             elif key == "graylog_streams" and isinstance(value, list):

src/fgai/entities.py

@@ -44,6 +44,13 @@ def entity_type(value: str) -> str:
         return "entity"
 
 
+def is_public_ip(value: str) -> bool:
+    try:
+        return ipaddress.ip_address(str(value)).is_global
+    except ValueError:
+        return False
+
+
 def _related_value(events: Iterable[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str:
     counts: Counter[str] = Counter()
     excluded = exclude.lower()

src/fgai/field_shapes.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import ipaddress
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from .entities import is_public_ip
+
+MAX_SAMPLE_CREDIT = 20
+SHAPE_FIT_WEIGHT = 70
+PORT_SHAPE_FIT_WEIGHT = 60
+SAMPLE_ADEQUACY_WEIGHT = 15
+NAME_BONUS_CAP = 15
+
+PORT_MIN, PORT_MAX = 0, 65535
+EPOCH_MIN, EPOCH_MAX = 946684800, 4102444800  # 2000-01-01 .. 2100-01-01
+
+NAME_TOKENS_BY_SHAPE = {
+    "ip": ("ip", "addr"),
+    "timestamp": ("time", "date", "timestamp"),
+    "port": ("port",),
+}
+
+
+def parse_iso_timestamp(value: str) -> float | None:
+    """Parse an ISO-8601-ish timestamp string to a unix epoch float, or None."""
+    try:
+        return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        return None
+
+
+def _is_ip(value: str) -> bool:
+    try:
+        ipaddress.ip_address(value)
+        return True
+    except ValueError:
+        return False
+
+
+def _is_timestamp(value: str) -> bool:
+    text = str(value).strip()
+    if not text:
+        return False
+    try:
+        number = float(text)
+        if EPOCH_MIN <= number <= EPOCH_MAX:
+            return True
+        if EPOCH_MIN <= number / 1000 <= EPOCH_MAX:
+            return True
+    except ValueError:
+        pass
+    return parse_iso_timestamp(text) is not None
+
+
+def _is_port(value: str) -> bool:
+    text = str(value).strip()
+    if "." in text:
+        return False
+    try:
+        number = int(text)
+    except ValueError:
+        return False
+    return PORT_MIN <= number <= PORT_MAX
+
+
+def _is_numeric(value: str) -> bool:
+    try:
+        float(value)
+        return True
+    except (TypeError, ValueError):
+        return False
+
+
+@dataclass
+class FieldShape:
+    primary_shape: str
+    confidence: int
+    scores: dict[str, int] = field(default_factory=dict)
+    evidence: list[str] = field(default_factory=list)
+
+
+def _name_bonus(field_name: str, shape: str) -> int:
+    if not field_name:
+        return 0
+    lower = field_name.lower()
+    tokens = NAME_TOKENS_BY_SHAPE.get(shape, ())
+    return NAME_BONUS_CAP if any(token in lower for token in tokens) else 0
+
+
+def classify_field_shape(sample_values, *, field_name: str = "") -> FieldShape:
+    """Classify a field's semantic shape from its observed values.
+
+    Value shape is the primary signal; ``field_name`` only contributes a small,
+    capped bonus that can nudge an already-plausible shape, never manufacture one.
+    """
+    values = [str(item).strip() for item in sample_values if str(item).strip()]
+    if not values:
+        return FieldShape(primary_shape="free_text", confidence=0, evidence=["no sample values"])
+
+    total = len(values)
+    cardinality = len(set(values))
+    low_cardinality = cardinality <= max(10, int(total * 0.5))
+
+    ip_ratio = sum(_is_ip(v) for v in values) / total
+    timestamp_ratio = sum(_is_timestamp(v) for v in values) / total
+    port_ratio = sum(_is_port(v) for v in values) / total if low_cardinality else 0.0
+    numeric_ratio = sum(_is_numeric(v) for v in values) / total
+    non_numeric_ratio = 1 - numeric_ratio
+    avg_words = sum(len(v.split()) for v in values) / total
+    multiword = avg_words >= 3
+    high_cardinality = cardinality > max(8, int(total * 0.5))
+    categorical_ratio = non_numeric_ratio if (1 < cardinality <= max(8, int(total * 0.5)) and not multiword) else 0.0
+    free_text_ratio = 0.4 if ((high_cardinality or multiword) and numeric_ratio < 0.5) else 0.0
+
+    fit_by_shape = {
+        "ip": (ip_ratio, SHAPE_FIT_WEIGHT),
+        "timestamp": (timestamp_ratio, SHAPE_FIT_WEIGHT),
+        "port": (port_ratio, PORT_SHAPE_FIT_WEIGHT),
+        "numeric": (numeric_ratio if numeric_ratio >= 0.8 else 0.0, SHAPE_FIT_WEIGHT),
+        "categorical_low_cardinality": (categorical_ratio, SHAPE_FIT_WEIGHT),
+        "free_text": (free_text_ratio, SHAPE_FIT_WEIGHT),
+    }
+    sample_adequacy = int(round(min(total, MAX_SAMPLE_CREDIT) / MAX_SAMPLE_CREDIT * SAMPLE_ADEQUACY_WEIGHT))
+
+    scores: dict[str, int] = {}
+    for shape, (fit_ratio, weight) in fit_by_shape.items():
+        if fit_ratio <= 0:
+            continue
+        shape_fit = int(round(fit_ratio * weight))
+        scores[shape] = min(100, shape_fit + sample_adequacy + _name_bonus(field_name, shape))
+
+    if not scores:
+        scores["free_text"] = sample_adequacy
+
+    primary = max(scores, key=scores.get)
+    evidence = [f"primary={primary} cardinality={cardinality} samples={total}"]
+    return FieldShape(primary_shape=primary, confidence=scores[primary], scores=scores, evidence=evidence)
+
+
+def private_ip_ratio(sample_values) -> float:
+    """Fraction of ip-shaped sample values that are private/internal addresses.
+
+    Used as a secondary, name-ambiguity-breaking signal only: the field more often
+    holding private addresses is treated as source-like relative to a peer field that
+    is more often public, never as a replacement for a clear name match.
+    """
+    ip_values = [str(item).strip() for item in sample_values if _is_ip(str(item).strip())]
+    if not ip_values:
+        return 0.0
+    return sum(not is_public_ip(value) for value in ip_values) / len(ip_values)

src/fgai/history.py

@@ -82,6 +82,13 @@ class FieldDiscoveryStore:
                     primary key (stream_id, field)
                 )"""
             )
+            connection.execute(
+                """create table if not exists field_confidence_history (
+                    stream_id text not null, field text not null,
+                    applied_confidence integer not null, applied_shape text not null, applied_at integer not null,
+                    primary key (stream_id, field)
+                )"""
+            )
 
     def ingest_catalog(self, stream_id: str, stream_name: str, fields: list[dict[str, object]], *, retention_days: int = 30) -> int:
         now = int(time.time())
@@ -196,3 +203,46 @@ class FieldDiscoveryStore:
             fields = {"fgai_stream_id": str(stream_id), "fgai_stream": str(stream_name), str(field): value}
             events.append(LogEvent(raw=json.dumps(fields, sort_keys=True), fields=fields))
         return events
+
+    def record_applied_confidence(self, stream_id: str, fields: list[str], field_confidence: dict[str, object], field_shapes: dict[str, object]) -> None:
+        """Snapshot a field's confidence/shape the first time it is auto-applied.
+
+        This is what a later drift check compares the current classification
+        against — only written when auto-apply actually fires (Phase 4), so this
+        table only grows on auto-apply events and stays small. Deliberately
+        insert-only (never overwritten): the drift baseline must stay pinned to
+        the original apply, not the most recent one, or a field that drifts
+        gradually while staying just above the auto-apply threshold would keep
+        re-stamping its own baseline every cycle and drift would never trigger.
+        """
+        now = int(time.time())
+        with sqlite3.connect(self.path) as connection:
+            for field in fields:
+                if field not in field_confidence:
+                    continue
+                connection.execute(
+                    "insert or ignore into field_confidence_history values (?, ?, ?, ?, ?)",
+                    (stream_id, field, int(field_confidence.get(field, 0) or 0), str(field_shapes.get(field, "")), now),
+                )
+
+    def applied_confidence(self, stream_id: str) -> dict[str, dict[str, object]]:
+        with sqlite3.connect(self.path) as connection:
+            rows = connection.execute(
+                "select field, applied_confidence, applied_shape, applied_at from field_confidence_history where stream_id = ?",
+                (stream_id,),
+            ).fetchall()
+        return {
+            str(field): {"applied_confidence": int(confidence), "applied_shape": str(shape), "applied_at": int(applied_at)}
+            for field, confidence, shape, applied_at in rows
+        }
+
+    def confidence_history_maintenance(self, *, retention_days: int = 90, vacuum: bool = False) -> dict[str, object]:
+        cutoff = int(time.time()) - retention_days * 86400
+        with sqlite3.connect(self.path) as connection:
+            before = int(connection.execute("select count(*) from field_confidence_history").fetchone()[0])
+            connection.execute("delete from field_confidence_history where applied_at < ?", (cutoff,))
+            connection.execute("pragma optimize")
+            after = int(connection.execute("select count(*) from field_confidence_history").fetchone()[0])
+            if vacuum:
+                connection.execute("vacuum")
+        return {"removed": before - after, "remaining": after}

src/fgai/llm.py

@@ -306,10 +306,10 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
     )
 
 
-def _compact_profile_suggestion(item: dict[str, object]) -> dict[str, object]:
+def _compact_profile_suggestion(item: dict[str, object], *, fields_only: list[str] | None = None) -> dict[str, object]:
     discovery = item.get("discovery", {}) if isinstance(item.get("discovery"), dict) else {}
     selected = discovery.get("selected_fields", {}) if isinstance(discovery.get("selected_fields"), dict) else {}
-    return {
+    compact = {
         "stream_id": item.get("stream_id"),
         "stream_name": item.get("stream_name"),
         "events": item.get("events"),
@@ -321,38 +321,81 @@ def _compact_profile_suggestion(item: dict[str, object]) -> dict[str, object]:
         "rejected_fields": discovery.get("rejected_fields", [])[:10],
         "heuristic_profile": item.get("profile", {}),
     }
+    if fields_only:
+        wanted = set(fields_only)
+        compact["fields_to_classify"] = sorted(wanted)
+        compact["common_fields"] = [row for row in compact["common_fields"] if isinstance(row, dict) and row.get("field") in wanted]
+        compact["top_fields"] = [row for row in compact["top_fields"] if isinstance(row, dict) and row.get("field") in wanted]
+        field_shapes = item.get("field_shapes", {})
+        field_confidence = item.get("field_confidence", {})
+        if isinstance(field_shapes, dict):
+            compact["field_shapes"] = {field: field_shapes[field] for field in wanted if field in field_shapes}
+        if isinstance(field_confidence, dict):
+            compact["field_confidence"] = {field: field_confidence[field] for field in wanted if field in field_confidence}
+    return compact
 
 
-def ollama_profile_advice(suggestions: list[dict[str, object]], model: str | None = None, timeout: int | None = None) -> list[dict[str, object]]:
+_FULL_PROFILE_ADVISOR_PROMPT_TEMPLATE = (
+    "You are SignalScope's local profile advisor. Infer stream profile mappings from observed field statistics. "
+    "Return only valid JSON with this schema: "
+    "{{\"profiles\":[{{\"stream_id\":\"...\",\"entity_fields\":[\"...\"],\"timestamp_field\":\"...\","
+    "\"categorical_fields\":[\"...\"],\"numeric_fields\":[\"...\"],\"detectors\":{{\"auth_failure\":{{\"enabled\":true,\"minimum\":5,\"z_threshold\":3}}}},"
+    "\"relationship_fields\":[{{\"left\":\"username\",\"right\":\"srcip\",\"name\":\"user source IP\"}}],"
+    "\"field_roles\":{{\"field\":\"entity|identity|asset|session|status|action|resource|metric|context\"}},"
+    "\"correlation_roles\":{{\"entity\":[\"...\"],\"identity\":[\"...\"],\"asset\":[\"...\"],\"session\":[\"...\"],\"activity\":[\"...\"],\"resource\":[\"...\"],\"status\":[\"...\"]}},"
+    "\"reason\":\"short reason\"}}]}}. "
+    "Use only field names present in common_fields, shared_fields aliases, top_fields, selected_fields, or heuristic_profile. "
+    "Do not invent fields. Do not include raw message/full_message/raw/answer fields. "
+    "Treat shared_fields as deterministic evidence that a field or semantic group appears across streams. "
+    "For custom namespaces such as lcs_* infer roles from the real field names and observed stats, for example tenant/customer/request/session/correlation IDs. "
+    "Prefer profile fields that connect streams: source or client IPs, usernames/accounts, hostnames, event IDs, session/correlation/request IDs, status/result/action/type, service/app, URL/domain/resource, and stable numeric metrics. "
+    "Use categorical_fields for low-cardinality behavior dimensions, numeric_fields for counters/durations/sizes, and entity_fields for fields that identify a user, host, IP, account, device, or application actor. "
+    "Use relationship_fields for behavior pairs where a left identity should learn normal right-side values, such as username->srcip, username->hostname, host->process.name, srcip->dstip, or account->application. "
+    "Allowed detectors are auth_failure, dns_query, deny_action. Prefer canonical fields such as username, hostname, eventid, srcip, dstip when present, but keep useful stream-specific fields too. "
+    "\n\nObserved streams:\n{compact}"
+)
+
+_SCOPED_PROFILE_ADVISOR_PROMPT_TEMPLATE = (
+    "You are SignalScope's local profile advisor. For each stream, classify ONLY the fields listed in "
+    "fields_to_classify into a semantic role, using the observed field_shapes, field_confidence, top_fields, and "
+    "common_fields as evidence. These fields already have a deterministic guess but low confidence, so focus on "
+    "resolving the ambiguity rather than re-deriving the whole profile. "
+    "Return only valid JSON with this schema: "
+    "{{\"profiles\":[{{\"stream_id\":\"...\",\"field_roles\":{{\"field_name\":\"entity|categorical|numeric|reject\"}},\"reason\":\"short reason\"}}]}}. "
+    "Use only field names present in fields_to_classify for that stream. Do not invent fields. "
+    "\"entity\" means the field identifies a user, host, IP, account, device, or application actor. "
+    "\"categorical\" means a low-cardinality behavior dimension useful for baselining. "
+    "\"numeric\" means a counter, duration, or size. \"reject\" means the field carries no useful signal. "
+    "\n\nStreams with fields to classify:\n{compact}"
+)
+
+
+def ollama_profile_advice(
+    suggestions: list[dict[str, object]],
+    model: str | None = None,
+    timeout: int | None = None,
+    *,
+    ambiguous_fields_by_stream: dict[str, list[str]] | None = None,
+) -> list[dict[str, object]]:
     host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
     selected_model = model or os.getenv("FGAI_PROFILE_ADVISOR_MODEL", "qwen3:8b")
     selected_timeout = timeout or int(os.getenv("FGAI_PROFILE_ADVISOR_TIMEOUT", "240"))
-    compact = [_compact_profile_suggestion(item) for item in suggestions[:10]]
+    scoped = bool(ambiguous_fields_by_stream)
+    compact = [
+        _compact_profile_suggestion(
+            item,
+            fields_only=(ambiguous_fields_by_stream or {}).get(str(item.get("stream_id", ""))) if scoped else None,
+        )
+        for item in suggestions[:10]
+    ]
+    template = _SCOPED_PROFILE_ADVISOR_PROMPT_TEMPLATE if scoped else _FULL_PROFILE_ADVISOR_PROMPT_TEMPLATE
     body = json.dumps(
         {
             "model": selected_model,
             "stream": False,
             "format": "json",
             "options": {"num_predict": 1200, "temperature": 0.1},
-            "prompt": (
-                "You are SignalScope's local profile advisor. Infer stream profile mappings from observed field statistics. "
-                "Return only valid JSON with this schema: "
-                "{\"profiles\":[{\"stream_id\":\"...\",\"entity_fields\":[\"...\"],\"timestamp_field\":\"...\","
-                "\"categorical_fields\":[\"...\"],\"numeric_fields\":[\"...\"],\"detectors\":{\"auth_failure\":{\"enabled\":true,\"minimum\":5,\"z_threshold\":3}},"
-                "\"relationship_fields\":[{\"left\":\"username\",\"right\":\"srcip\",\"name\":\"user source IP\"}],"
-                "\"field_roles\":{\"field\":\"entity|identity|asset|session|status|action|resource|metric|context\"},"
-                "\"correlation_roles\":{\"entity\":[\"...\"],\"identity\":[\"...\"],\"asset\":[\"...\"],\"session\":[\"...\"],\"activity\":[\"...\"],\"resource\":[\"...\"],\"status\":[\"...\"]},"
-                "\"reason\":\"short reason\"}]}. "
-                "Use only field names present in common_fields, shared_fields aliases, top_fields, selected_fields, or heuristic_profile. "
-                "Do not invent fields. Do not include raw message/full_message/raw/answer fields. "
-                "Treat shared_fields as deterministic evidence that a field or semantic group appears across streams. "
-                "For custom namespaces such as lcs_* infer roles from the real field names and observed stats, for example tenant/customer/request/session/correlation IDs. "
-                "Prefer profile fields that connect streams: source or client IPs, usernames/accounts, hostnames, event IDs, session/correlation/request IDs, status/result/action/type, service/app, URL/domain/resource, and stable numeric metrics. "
-                "Use categorical_fields for low-cardinality behavior dimensions, numeric_fields for counters/durations/sizes, and entity_fields for fields that identify a user, host, IP, account, device, or application actor. "
-                "Use relationship_fields for behavior pairs where a left identity should learn normal right-side values, such as username->srcip, username->hostname, host->process.name, srcip->dstip, or account->application. "
-                "Allowed detectors are auth_failure, dns_query, deny_action. Prefer canonical fields such as username, hostname, eventid, srcip, dstip when present, but keep useful stream-specific fields too. "
-                f"\n\nObserved streams:\n{json.dumps(compact, sort_keys=True)}"
-            ),
+            "prompt": template.format(compact=json.dumps(compact, sort_keys=True)),
         }
     ).encode("utf-8")
     req = request.Request(f"{host}/api/generate", data=body, method="POST", headers={"Content-Type": "application/json"})

src/fgai/mitigation.py

@@ -10,6 +10,7 @@ import json
 import ssl
 from urllib.parse import urlencode
 
+from .entities import is_public_ip
 from .logs import event_score, is_utm_event
 from .models import BlockCandidate, LogEvent
 
@@ -26,12 +27,9 @@ def parse_allowlist(value: str | None = None) -> list[ipaddress._BaseNetwork]:
 
 
 def is_blockable_public_ip(ip: str, allowlist: Iterable[ipaddress._BaseNetwork] = ()) -> bool:
-    try:
-        address = ipaddress.ip_address(ip)
-    except ValueError:
-        return False
-    if not address.is_global:
+    if not is_public_ip(ip):
         return False
+    address = ipaddress.ip_address(ip)
     return not any(address in network for network in allowlist)
 
 

src/fgai/monitor.py

@@ -114,13 +114,141 @@ def _suggestion_field_set(suggestion: dict[str, object]) -> set[str]:
     return {field for field in fields if field}
 
 
-def _needs_profile_advisor(suggestion: dict[str, object], stream_profiles: dict[str, object]) -> bool:
-    if not suggestion.get("profile_exists"):
-        return True
+def _ambiguous_fields(suggestion: dict[str, object], threshold: int) -> list[str]:
+    field_confidence = suggestion.get("field_confidence", {})
+    if not isinstance(field_confidence, dict):
+        return []
+    selected = _suggestion_field_set(suggestion)
+    # Only fields that were actually observed and shape-classified are "ambiguous" —
+    # a field with no field_confidence entry (e.g. a placeholder timestamp field name
+    # used when no time-like field was discovered at all) has nothing to disambiguate.
+    return [field for field in selected if field in field_confidence and int(field_confidence[field] or 0) < threshold]
+
+
+def _needs_profile_advisor(suggestion: dict[str, object], stream_profiles: dict[str, object], *, confidence_threshold: int = 60) -> bool:
     stream_id = str(suggestion.get("stream_id", ""))
     suggested_fields = _suggestion_field_set(suggestion)
+    if not suggestion.get("profile_exists"):
+        return True
     existing_fields = _profile_field_set(stream_profiles.get(stream_id))
-    return bool(suggested_fields - existing_fields)
+    if suggested_fields - existing_fields:
+        return True
+    return bool(_ambiguous_fields(suggestion, confidence_threshold))
+
+
+def _suggestion_is_high_confidence(suggestion: dict[str, object], threshold: int) -> bool:
+    field_confidence = suggestion.get("field_confidence", {})
+    if not isinstance(field_confidence, dict):
+        return False
+    observed = [field for field in _suggestion_field_set(suggestion) if field in field_confidence]
+    if not observed:
+        return False
+    return all(int(field_confidence[field] or 0) >= threshold for field in observed)
+
+
+def _profile_is_auto_apply_safe(existing: object) -> bool:
+    """A profile with any manual-origin field is left alone entirely by auto-apply."""
+    if existing is None:
+        return True
+    field_origin = getattr(existing, "field_origin", None) or {}
+    return all(origin != "manual" for origin in field_origin.values())
+
+
+def _auto_apply_profile_suggestions(
+    profile_suggestions: list[dict[str, object]],
+    stream_profiles: dict[str, object],
+    runtime_values: dict[str, object],
+    config_path: str | None,
+) -> list[str]:
+    """Write high-confidence, provenance-safe suggestions straight to config.
+
+    Mirrors the dashboard's own POST /api/config write path (ConfigStore.update),
+    just invoked here unattended each cycle instead of from a human clicking Apply.
+    """
+    if not config_path or not runtime_values.get("auto_apply_profile_suggestions"):
+        return []
+    threshold = int(runtime_values.get("field_confidence_threshold", 60) or 60)
+    updated_profiles = [item for item in runtime_values.get("graylog_stream_profiles", []) if isinstance(item, dict)]
+    applied: list[str] = []
+    now = int(time.time())
+    for suggestion in profile_suggestions:
+        if not isinstance(suggestion, dict):
+            continue
+        stream_id = str(suggestion.get("stream_id", ""))
+        profile = suggestion.get("profile", {})
+        if not stream_id or not isinstance(profile, dict):
+            continue
+        if not _suggestion_is_high_confidence(suggestion, threshold):
+            continue
+        if not _profile_is_auto_apply_safe(stream_profiles.get(stream_id)):
+            continue
+        all_fields = {*profile.get("entity_fields", []), *profile.get("categorical_fields", []), *profile.get("numeric_fields", [])}
+        new_profile = dict(profile)
+        new_profile["field_origin"] = {str(field): "auto" for field in all_fields if field}
+        new_profile["updated_at"] = now
+        updated_profiles = [item for item in updated_profiles if item.get("stream_id") != stream_id]
+        updated_profiles.append(new_profile)
+        applied.append(stream_id)
+    if applied:
+        ConfigStore(config_path).update({"graylog_stream_profiles": updated_profiles})
+    return applied
+
+
+DRIFT_CONFIDENCE_DROP_THRESHOLD = 20
+
+
+def _detect_profile_drift(
+    profile_suggestions: list[dict[str, object]],
+    stream_profiles: dict[str, object],
+    discovery_store: object | None,
+) -> list[dict[str, object]]:
+    """Flag auto-applied fields whose classification has drifted since they were applied.
+
+    Flag-only: this never touches the live profile, only surfaces a diagnostic for a
+    human to review. A silently "self-corrected" auto-applied field could suppress or
+    misdirect real findings with nobody noticing, which is worse than a stale mapping
+    that gets flagged and reviewed.
+    """
+    if discovery_store is None:
+        return []
+    drift: list[dict[str, object]] = []
+    for suggestion in profile_suggestions:
+        if not isinstance(suggestion, dict):
+            continue
+        stream_id = str(suggestion.get("stream_id", ""))
+        if not stream_id:
+            continue
+        existing = stream_profiles.get(stream_id)
+        field_origin = getattr(existing, "field_origin", None) or {}
+        auto_fields = [field for field, origin in field_origin.items() if origin == "auto"]
+        if not auto_fields:
+            continue
+        applied = discovery_store.applied_confidence(stream_id)
+        if not applied:
+            continue
+        field_confidence = suggestion.get("field_confidence", {})
+        field_shapes = suggestion.get("field_shapes", {})
+        if not isinstance(field_confidence, dict) or not isinstance(field_shapes, dict):
+            continue
+        for field in auto_fields:
+            baseline = applied.get(field)
+            if not baseline or field not in field_confidence:
+                continue
+            current_confidence = int(field_confidence.get(field, 0) or 0)
+            current_shape = str(field_shapes.get(field, ""))
+            confidence_drop = int(baseline["applied_confidence"]) - current_confidence
+            shape_changed = bool(current_shape) and current_shape != str(baseline["applied_shape"])
+            if confidence_drop > DRIFT_CONFIDENCE_DROP_THRESHOLD or shape_changed:
+                drift.append({
+                    "stream_id": stream_id,
+                    "field": field,
+                    "applied_confidence": baseline["applied_confidence"],
+                    "current_confidence": current_confidence,
+                    "applied_shape": baseline["applied_shape"],
+                    "current_shape": current_shape,
+                    "applied_at": baseline["applied_at"],
+                })
+    return drift
 
 
 def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
@@ -427,13 +555,19 @@ def build_status(
     stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
     discovery_cache_events = []
     discovered_profile_fields = 0
+    discovery_store = None
     if history_path:
         discovery_store = FieldDiscoveryStore(history_path)
         discovered_profile_fields = discovery_store.ingest(events)
         discovery_cache_events = discovery_store.synthetic_events()
     profile_suggestions = suggest_stream_profiles([*discovery_cache_events, *events], existing_profiles=stream_profiles)
+    confidence_threshold = int(runtime_values.get("field_confidence_threshold", 60) or 60)
     profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"}
-    advisor_candidates = [item for item in profile_suggestions if _needs_profile_advisor(item, stream_profiles)]
+    advisor_candidates = [item for item in profile_suggestions if _needs_profile_advisor(item, stream_profiles, confidence_threshold=confidence_threshold)]
+    ambiguous_fields_by_stream = {
+        str(item.get("stream_id", "")): _ambiguous_fields(item, confidence_threshold)
+        for item in advisor_candidates
+    }
     if runtime_values.get("profile_advisor_enabled") and not run_profile_advisor:
         profile_advisor_status = {"enabled": True, "status": "pending" if advisor_candidates else "skipped_no_profile_changes", "candidates": len(advisor_candidates)}
     elif runtime_values.get("profile_advisor_enabled") and profile_suggestions and not advisor_candidates:
@@ -444,6 +578,7 @@ def build_status(
                 advisor_candidates[:10],
                 model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
                 timeout=int(runtime_values.get("profile_advisor_timeout", 240) or 240),
+                ambiguous_fields_by_stream=ambiguous_fields_by_stream,
             )
             profile_suggestions = apply_profile_advice(profile_suggestions, advice)
             profile_advisor_status = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
@@ -457,6 +592,28 @@ def build_status(
             }
             for suggestion in profile_suggestions:
                 suggestion.setdefault("profile_advisor", {"status": "heuristic", "error": str(exc)})
+    auto_applied_streams = _auto_apply_profile_suggestions(profile_suggestions, stream_profiles, runtime_values, config_path)
+    if auto_applied_streams:
+        if discovery_store:
+            suggestions_by_stream = {str(item.get("stream_id", "")): item for item in profile_suggestions if isinstance(item, dict)}
+            for stream_id in auto_applied_streams:
+                suggestion = suggestions_by_stream.get(stream_id)
+                if not suggestion:
+                    continue
+                profile = suggestion.get("profile", {})
+                if not isinstance(profile, dict):
+                    continue
+                applied_fields = [*profile.get("entity_fields", []), *profile.get("categorical_fields", []), *profile.get("numeric_fields", [])]
+                discovery_store.record_applied_confidence(
+                    stream_id,
+                    applied_fields,
+                    suggestion.get("field_confidence", {}) if isinstance(suggestion.get("field_confidence"), dict) else {},
+                    suggestion.get("field_shapes", {}) if isinstance(suggestion.get("field_shapes"), dict) else {},
+                )
+        if config_store:
+            stream_profiles = parse_profiles(config_store.read().get("graylog_stream_profiles", []))
+    confidence_history_maintenance = discovery_store.confidence_history_maintenance(retention_days=int(runtime_values.get("field_confidence_retention_days", 90) or 90)) if discovery_store else {}
+    profile_drift = _detect_profile_drift(profile_suggestions, stream_profiles, discovery_store)
     intel_ips = sorted(
         {
             ip
@@ -512,7 +669,9 @@ def build_status(
         "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "discovery_fields_recorded": discovered_profile_fields, "discovery_cache_events": len(discovery_cache_events), "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
         "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
         "configuration": runtime_config,
-        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights, "relationship_fields": [{"left": relation.left, "right": relation.right, "name": relation.name} for relation in item.relationship_fields]} for item in stream_profiles.values()],
+        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights, "relationship_fields": [{"left": relation.left, "right": relation.right, "name": relation.name} for relation in item.relationship_fields], "field_origin": item.field_origin, "updated_at": item.updated_at} for item in stream_profiles.values()],
+        "profile_auto_apply": {"applied_streams": auto_applied_streams, "confidence_history_maintenance": confidence_history_maintenance},
+        "profile_drift": profile_drift,
         "stream_coverage": stream_coverage,
         "profile_suggestions": profile_suggestions,
         "profile_readiness": profile_readiness,
@@ -626,7 +785,12 @@ def add_profile_advisor(status: dict[str, object], runtime_values: dict[str, obj
     if not isinstance(profile_suggestions, list):
         return
     stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
-    advisor_candidates = [item for item in profile_suggestions if isinstance(item, dict) and _needs_profile_advisor(item, stream_profiles)]
+    confidence_threshold = int(runtime_values.get("field_confidence_threshold", 60) or 60)
+    advisor_candidates = [item for item in profile_suggestions if isinstance(item, dict) and _needs_profile_advisor(item, stream_profiles, confidence_threshold=confidence_threshold)]
+    ambiguous_fields_by_stream = {
+        str(item.get("stream_id", "")): _ambiguous_fields(item, confidence_threshold)
+        for item in advisor_candidates
+    }
     capabilities = status.setdefault("capabilities", {})
     if not isinstance(capabilities, dict):
         return
@@ -641,6 +805,7 @@ def add_profile_advisor(status: dict[str, object], runtime_values: dict[str, obj
             advisor_candidates[:10],
             model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
             timeout=int(runtime_values.get("profile_advisor_timeout", 240) or 240),
+            ambiguous_fields_by_stream=ambiguous_fields_by_stream,
         )
         status["profile_suggestions"] = apply_profile_advice(profile_suggestions, advice)
         capabilities["profile_advisor"] = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
@@ -740,6 +905,7 @@ def monitor_loop(
     print(f"Writing status to {output}", flush=True)
     last_llm_at = 0
     last_llm_text: str | None = None
+    last_advisor_at = 0
     while True:
         runtime = ConfigStore(config_path).read() if config_path and Path(config_path).exists() else {}
         effective_llm = bool(runtime.get("llm_enabled")) if runtime else llm
@@ -804,16 +970,21 @@ def monitor_loop(
                     "generated_at": last_llm_at,
                     "text": last_llm_text or "",
                 }
-            if runtime.get("profile_advisor_enabled"):
+            advisor_interval = int(runtime.get("profile_advisor_interval_seconds", 300) or 300)
+            if runtime.get("profile_advisor_enabled") and now - last_advisor_at >= advisor_interval:
                 add_profile_advisor(status, runtime)
+                last_advisor_at = now
                 write_status(status, output)
         else:
             status["llm_assessment"] = {"enabled": False, "status": "disabled", "text": ""}
-            if runtime.get("profile_advisor_enabled"):
+            advisor_interval = int(runtime.get("profile_advisor_interval_seconds", 300) or 300)
+            now = int(time.time())
+            if runtime.get("profile_advisor_enabled") and now - last_advisor_at >= advisor_interval:
                 if status_cache_path and isinstance(mcp, dict) and mcp.get("status") not in {"error", "refreshing"}:
                     StatusSnapshotStore(status_cache_path).save("last_good", status)
                 write_status(status, output)
                 add_profile_advisor(status, runtime)
+                last_advisor_at = now
         if status_cache_path and isinstance(mcp, dict) and mcp.get("status") not in {"error", "refreshing"}:
             StatusSnapshotStore(status_cache_path).save("last_good", status)
         write_status(status, output)

src/fgai/profile_suggestions.py

@@ -4,9 +4,13 @@ from collections import Counter, defaultdict
 
 from .detectors import event_detector_categories
 from .entities import ENTITY_FIELDS
+from .field_shapes import classify_field_shape, private_ip_ratio
+from .logs import THREAT_ACTIONS
 from .models import LogEvent
 from .normalization import FIELD_ALIASES
 
+SHAPE_ENTITY_CONFIDENCE = 60
+
 ALIAS_CANONICAL_BY_FIELD = {
     alias: canonical
     for canonical, aliases in FIELD_ALIASES.items()
@@ -381,21 +385,28 @@ def _rejected_fields(
     return rejected
 
 
-def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
+def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str], *, field_shapes: dict[str, str] | None = None, field_confidence: dict[str, int] | None = None) -> list[str]:
+    field_shapes = field_shapes or {}
+    field_confidence = field_confidence or {}
     output = list(selected)
     for field, count in coverage.most_common():
         if len(output) >= 4:
             break
-        if field in output or field in IGNORED_DISCOVERY_FIELDS or not _looks_like_entity_field(field):
+        if field in output or field in IGNORED_DISCOVERY_FIELDS:
+            continue
+        shape_match = field_shapes.get(field) == "ip" and field_confidence.get(field, 0) >= SHAPE_ENTITY_CONFIDENCE
+        if not shape_match and not _looks_like_entity_field(field):
             continue
         coverage_ratio = count / max(1, total)
         cardinality = len(unique_values[field])
-        if coverage_ratio >= 0.05 and 1 < cardinality <= min(500, max(10, total)):
+        if shape_match or (coverage_ratio >= 0.05 and 1 < cardinality <= min(500, max(10, total))):
             output.append(field)
     return output
 
 
-def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
+def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str], *, field_shapes: dict[str, str] | None = None, field_confidence: dict[str, int] | None = None) -> list[str]:
+    field_shapes = field_shapes or {}
+    field_confidence = field_confidence or {}
     output = list(selected)
     for field, count in coverage.most_common():
         if len(output) >= 12:
@@ -404,19 +415,23 @@ def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str,
             continue
         coverage_ratio = count / max(1, total)
         cardinality = len(unique_values[field])
-        if coverage_ratio >= 0.1 and 1 < cardinality <= min(200, max(8, int(total * 0.7))):
+        shape_match = field_shapes.get(field) == "categorical_low_cardinality" and field_confidence.get(field, 0) >= SHAPE_ENTITY_CONFIDENCE
+        if shape_match or (coverage_ratio >= 0.1 and 1 < cardinality <= min(200, max(8, int(total * 0.7)))):
             output.append(field)
     return output
 
 
-def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str], total: int, selected: list[str]) -> list[str]:
+def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str], total: int, selected: list[str], *, field_shapes: dict[str, str] | None = None, field_confidence: dict[str, int] | None = None) -> list[str]:
+    field_shapes = field_shapes or {}
+    field_confidence = field_confidence or {}
     output = list(selected)
     for field, count in coverage.most_common():
         if len(output) >= 8:
             break
         if field in output or field in IGNORED_DISCOVERY_FIELDS:
             continue
-        if count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95:
+        shape_match = field_shapes.get(field) == "numeric" and field_confidence.get(field, 0) >= SHAPE_ENTITY_CONFIDENCE
+        if shape_match or (count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95):
             output.append(field)
     return output
 
@@ -425,6 +440,30 @@ def _relationship_name(left: str, right: str) -> str:
     return f"{left} to {right}"
 
 
+_AUTH_FAILURE_TOKENS = {"fail", "failed", "denied", "rejected", "invalid", "incorrect"}
+_DENY_ACTION_TOKENS = THREAT_ACTIONS | {"denied", "rejected", "blocked"}
+
+
+def _looks_domain_like(value: str) -> bool:
+    return "." in value and any(char.isalpha() for char in value) and " " not in value
+
+
+def _shape_detector_hints(field_shapes: dict[str, str], unique_values: dict[str, set[str]]) -> dict[str, dict[str, object]]:
+    """Suggest detectors from classified value patterns, independent of field/stream name."""
+    hints: dict[str, dict[str, object]] = {}
+    for field, shape in field_shapes.items():
+        if shape != "categorical_low_cardinality":
+            continue
+        values_lower = {str(value).lower() for value in unique_values.get(field, ())}
+        if any(token in value for value in values_lower for token in _AUTH_FAILURE_TOKENS):
+            hints["auth_failure"] = {"enabled": True, "minimum": 5, "z_threshold": 3.0, "source": "value_shape"}
+        if any(token in value for value in values_lower for token in _DENY_ACTION_TOKENS):
+            hints["deny_action"] = {"enabled": True, "minimum": 10, "z_threshold": 3.0, "source": "value_shape"}
+        if any(_looks_domain_like(value) for value in values_lower):
+            hints["dns_query"] = {"enabled": True, "minimum": 10, "z_threshold": 3.0, "source": "value_shape"}
+    return hints
+
+
 def _field_matches(field: str, groups: set[str], tokens: set[str]) -> bool:
     lower = field.lower()
     semantic = set(_semantic_field_groups(lower))
@@ -439,8 +478,10 @@ def _relationship_candidates(
     categorical_fields: list[str],
     numeric_fields: list[str],
     *,
+    field_shapes: dict[str, str] | None = None,
     limit: int = 6,
 ) -> list[dict[str, str]]:
+    field_shapes = field_shapes or {}
     def usable(field: str) -> bool:
         return (
             bool(field)
@@ -473,6 +514,20 @@ def _relationship_candidates(
     ]
     destination_fields.extend(field for field in categorical if field.lower() in {"dstport", "destination_port", "dest_port", "service", "application", "app", "url", "query_domain", "dns_query"})
 
+    # Ambiguous-name tie-break: two ip-shaped entity fields that neither name-matched
+    # source nor destination are split using which one leans more private (source-like)
+    # vs. more public (destination-like). Only fires when names give no signal at all.
+    ambiguous_ip_fields = [
+        field for field in entities
+        if field_shapes.get(field) == "ip" and field not in source_fields and field not in destination_fields
+    ]
+    if len(ambiguous_ip_fields) >= 2:
+        private_ratio = {field: private_ip_ratio(unique_values[field]) for field in ambiguous_ip_fields}
+        ranked = sorted(ambiguous_ip_fields, key=lambda field: private_ratio[field], reverse=True)
+        if private_ratio[ranked[0]] > private_ratio[ranked[-1]]:
+            source_fields.append(ranked[0])
+            destination_fields.append(ranked[-1])
+
     for left in identity_fields:
         for right in [*source_fields, *host_fields]:
             if left != right:
@@ -632,17 +687,49 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
             output.append(item)
             continue
         allowed_fields = _allowed_fields(item)
-        def valid_fields(key: str, fallback: list[str]) -> list[str]:
+        def whole_list(key: str) -> list[str] | None:
+            """Advisor-provided whole field list for this key, or None if it gave nothing usable."""
             value = advisor.get(key, [])
             fields = [str(field) for field in value if str(field) in allowed_fields] if isinstance(value, list) else []
-            return list(dict.fromkeys(fields or fallback))
+            return list(dict.fromkeys(fields)) or None
         heuristic = item.get("profile", {}) if isinstance(item.get("profile"), dict) else {}
-        entity_fields = valid_fields("entity_fields", list(heuristic.get("entity_fields", [])) if isinstance(heuristic.get("entity_fields"), list) else [])
+        heuristic_entity = list(heuristic.get("entity_fields", [])) if isinstance(heuristic.get("entity_fields"), list) else []
+        heuristic_categorical = list(heuristic.get("categorical_fields", [])) if isinstance(heuristic.get("categorical_fields"), list) else []
+        heuristic_numeric = list(heuristic.get("numeric_fields", [])) if isinstance(heuristic.get("numeric_fields"), list) else []
+        whole_entity = whole_list("entity_fields")
+        whole_categorical = whole_list("categorical_fields")
+        whole_numeric = whole_list("numeric_fields")
+        entity_fields = list(whole_entity) if whole_entity is not None else list(heuristic_entity)
+        categorical = list(whole_categorical) if whole_categorical is not None else list(heuristic_categorical)
+        numeric = list(whole_numeric) if whole_numeric is not None else list(heuristic_numeric)
         timestamp = str(advisor.get("timestamp_field", ""))
         if timestamp not in allowed_fields:
             timestamp = str(heuristic.get("timestamp_field", "timestamp"))
-        categorical = valid_fields("categorical_fields", list(heuristic.get("categorical_fields", [])) if isinstance(heuristic.get("categorical_fields"), list) else [])
-        numeric = valid_fields("numeric_fields", list(heuristic.get("numeric_fields", [])) if isinstance(heuristic.get("numeric_fields"), list) else [])
+
+        # Per-field role patches (scoped advisor calls answer for only a handful of
+        # ambiguous fields via field_roles instead of returning whole lists). A role in
+        # the controlled vocabulary below moves that single field between lists; other
+        # role values (the free-form correlation vocabulary used by full-profile calls)
+        # are left alone here and only recorded informationally further down.
+        raw_field_roles_for_merge = advisor.get("field_roles", {})
+        role_lists = {"entity": entity_fields, "categorical": categorical, "numeric": numeric}
+        reclassified_fields = 0
+        if isinstance(raw_field_roles_for_merge, dict):
+            for field, role in raw_field_roles_for_merge.items():
+                field, role = str(field), str(role)
+                if field not in allowed_fields or role not in {"entity", "categorical", "numeric", "reject"}:
+                    continue
+                if role == "reject" and field in role_lists["entity"] and len(role_lists["entity"]) <= 1:
+                    continue  # never reject the only entity field
+                for other_list in role_lists.values():
+                    if field in other_list:
+                        other_list.remove(field)
+                if role != "reject":
+                    role_lists[role].append(field)
+                reclassified_fields += 1
+        if not entity_fields:
+            entity_fields = list(heuristic_entity) or [next(iter(allowed_fields), "source")]
+
         relationships = []
         raw_relationships = advisor.get("relationship_fields", [])
         if isinstance(raw_relationships, list):
@@ -668,7 +755,8 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
                     continue
         if not detectors:
             detectors = dict(heuristic.get("detectors", {})) if isinstance(heuristic.get("detectors"), dict) else {}
-        if entity_fields:
+        advisor_gave_something_usable = bool(entity_fields and (whole_entity is not None or whole_categorical is not None or whole_numeric is not None or reclassified_fields))
+        if advisor_gave_something_usable:
             raw_field_roles = advisor.get("field_roles", {})
             field_roles = {}
             if isinstance(raw_field_roles, dict):
@@ -703,6 +791,8 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
                 "reason": str(advisor.get("reason", "")),
                 "field_roles": field_roles,
                 "correlation_roles": correlation_roles,
+                "fields_reclassified": reclassified_fields,
+                "fields_requested": len(field_roles),
             }
         else:
             item["profile_advisor"] = {"status": "invalid", "reason": "advisor returned no valid entity fields"}
@@ -759,19 +849,27 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
         entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
         categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
         numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
+        field_shapes: dict[str, str] = {}
+        field_confidence: dict[str, int] = {}
+        for field in coverage:
+            if field in IGNORED_DISCOVERY_FIELDS:
+                continue
+            shape = classify_field_shape(unique_values[field], field_name=field)
+            field_shapes[field] = shape.primary_shape
+            field_confidence[field] = shape.confidence
         shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields)
-        entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3))
+        entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3), field_shapes=field_shapes, field_confidence=field_confidence)
         timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
         if not timestamp:
             timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
         categorical_seed = list(dict.fromkeys([*_pick_present(categorical_priority, coverage, total, min_ratio=0.02, limit=10 if is_windows else 8), *shared_categorical]))
-        categorical = _generic_categorical_fields(coverage, unique_values, total, categorical_seed)
+        categorical = _generic_categorical_fields(coverage, unique_values, total, categorical_seed, field_shapes=field_shapes, field_confidence=field_confidence)
         numeric_seed = [
             field for field in numeric_priority
             if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
         ][:5]
         numeric_seed = list(dict.fromkeys([*numeric_seed, *shared_numeric]))
-        numeric = numeric_seed if is_windows else _generic_numeric_fields(coverage, numeric_counts, total, numeric_seed)
+        numeric = numeric_seed if is_windows else _generic_numeric_fields(coverage, numeric_counts, total, numeric_seed, field_shapes=field_shapes, field_confidence=field_confidence)
         if not entity_fields:
             entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
         if not entity_fields:
@@ -804,15 +902,17 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
                 if field not in set(categorical) and field not in {"eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code", "logon_type"}
             ][:5]
         detectors = {
-            name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
+            name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0, "source": "event_pattern"}
             for name, count in detector_counts.items()
             if count > 0
         }
         if is_windows and coverage["eventid"]:
-            detectors.setdefault("auth_failure", {"enabled": True, "minimum": 5, "z_threshold": 3.0})
+            detectors.setdefault("auth_failure", {"enabled": True, "minimum": 5, "z_threshold": 3.0, "source": "windows_eventid"})
+        for name, hint in _shape_detector_hints(field_shapes, unique_values).items():
+            detectors.setdefault(name, hint)
         if not detectors:
             detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
-        relationship_fields = _relationship_candidates(coverage, unique_values, total, entity_fields, categorical, numeric)
+        relationship_fields = _relationship_candidates(coverage, unique_values, total, entity_fields, categorical, numeric, field_shapes=field_shapes)
         high_coverage = [
             {"field": field, "coverage": round(count / max(1, total), 2), "unique_values": len(unique_values[field])}
             for field, count in coverage.most_common(12)
@@ -847,6 +947,8 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
             "profile_exists": bool(existing),
             "confidence": "high" if score >= 75 and total >= 100 else "medium" if score >= 55 and total >= 20 else "low",
             "score": score,
+            "field_confidence": field_confidence,
+            "field_shapes": field_shapes,
             "entity_fields": entity_fields,
             "timestamp_field": timestamp,
             "categorical_fields": categorical,

src/fgai/replay.py

@@ -2,11 +2,11 @@ from __future__ import annotations
 
 import tempfile
 from collections import Counter, defaultdict
-from datetime import datetime
 from pathlib import Path
 
 from .anomaly import anomaly_summary, detect_source_anomalies
 from .baseline import BaselineStore
+from .field_shapes import parse_iso_timestamp
 from .models import LogEvent
 
 
@@ -32,10 +32,8 @@ def _timestamp(event: LogEvent, fallback: int) -> int:
     try:
         return int(float(value))
     except (TypeError, ValueError):
-        try:
-            return int(datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp())
-        except ValueError:
-            return fallback
+        parsed = parse_iso_timestamp(value)
+        return int(parsed) if parsed is not None else fallback
 
 
 def _with_stream(events: list[LogEvent], stream_id: str, stream_name: str) -> list[LogEvent]:

src/fgai/sequences.py

@@ -5,6 +5,7 @@ from datetime import datetime
 
 from .detectors import event_detector_categories
 from .entities import event_entities
+from .field_shapes import parse_iso_timestamp
 from .logs import THREAT_ACTIONS
 from .models import LogEvent
 from .normalization import canonical_value
@@ -25,10 +26,9 @@ def _timestamp(event: LogEvent, fallback: int) -> int:
                 parsed /= 1_000
             return int(parsed)
         except ValueError:
-            try:
-                return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
-            except ValueError:
-                pass
+            parsed_timestamp = parse_iso_timestamp(value)
+            if parsed_timestamp is not None:
+                return int(parsed_timestamp)
     date = event.fields.get("date")
     clock = event.fields.get("time")
     if date and clock:

src/fgai/stream_profiles.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections.abc import Iterable
 from dataclasses import dataclass, field
 
 
@@ -22,6 +23,8 @@ class StreamProfile:
     detectors: dict[str, dict[str, object]] = field(default_factory=dict)
     field_weights: dict[str, object] = field(default_factory=dict)
     relationship_fields: tuple[RelationshipField, ...] = ()
+    field_origin: dict[str, str] = field(default_factory=dict)
+    updated_at: int = 0
 
 
 def _detectors(value: object) -> dict[str, dict[str, object]]:
@@ -81,6 +84,20 @@ def _relationship_fields(value: object) -> tuple[RelationshipField, ...]:
     return tuple(output)
 
 
+def _field_origin(value: object, fields: Iterable[str]) -> dict[str, str]:
+    """Per-field provenance, defaulting missing entries to "manual" for safety.
+
+    A profile written before this field existed, or hand-edited through the
+    dashboard, must never be silently treated as auto-overwritable.
+    """
+    raw = value if isinstance(value, dict) else {}
+    output: dict[str, str] = {}
+    for field_name in fields:
+        origin = str(raw.get(field_name, "manual")).strip().lower()
+        output[field_name] = origin if origin in {"auto", "manual"} else "manual"
+    return output
+
+
 def parse_profiles(value: object) -> dict[str, StreamProfile]:
     profiles: dict[str, StreamProfile] = {}
     for item in value if isinstance(value, list) else []:
@@ -97,12 +114,21 @@ def parse_profiles(value: object) -> dict[str, StreamProfile]:
         detectors = _detectors(item.get("detectors", {}))
         field_weights = _field_weights(item.get("field_weights", {}))
         relationship_fields = _relationship_fields(item.get("relationship_fields", []))
+        categorical_fields = tuple(str(field) for field in item.get("categorical_fields", []) if field)
+        numeric_fields = tuple(str(field) for field in item.get("numeric_fields", []) if field)
+        field_origin = _field_origin(item.get("field_origin"), {*entity_fields, *categorical_fields, *numeric_fields})
+        try:
+            updated_at = int(item.get("updated_at", 0) or 0)
+        except (TypeError, ValueError):
+            updated_at = 0
         profiles[stream_id] = StreamProfile(
             stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp, entity_fields,
-            tuple(str(field) for field in item.get("categorical_fields", []) if field),
-            tuple(str(field) for field in item.get("numeric_fields", []) if field),
+            categorical_fields,
+            numeric_fields,
             detectors,
             field_weights,
             relationship_fields,
+            field_origin,
+            updated_at,
         )
     return profiles

src/fgai/threat_intel.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import ipaddress
 import json
 import os
 import time
@@ -8,14 +7,7 @@ from pathlib import Path
 from urllib import error, request
 from urllib.parse import urlencode
 
-
-def is_public_ip(value: str | None) -> bool:
-    if not value:
-        return False
-    try:
-        return ipaddress.ip_address(value).is_global
-    except ValueError:
-        return False
+from .entities import is_public_ip
 
 
 class ThreatIntelClient:

tests/test_config.py

@@ -62,6 +62,26 @@ class ConfigTests(unittest.TestCase):
             public = store.update({"graylog_tls_verify": False})
             self.assertFalse(public["graylog_tls_verify"])
 
+    def test_profile_advisor_interval_seconds_is_numeric_and_bounded(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = ConfigStore(str(Path(directory) / "config.json"))
+            public = store.update({"profile_advisor_interval_seconds": "10"})
+            self.assertEqual(public["profile_advisor_interval_seconds"], 60)
+            public = store.update({"profile_advisor_interval_seconds": "600"})
+            self.assertEqual(public["profile_advisor_interval_seconds"], 600)
+
+    def test_field_confidence_threshold_is_numeric_and_clamped(self):
+        with tempfile.TemporaryDirectory() as directory:
+            store = ConfigStore(str(Path(directory) / "config.json"))
+            public = store.update({"field_confidence_threshold": "-5"})
+            self.assertEqual(public["field_confidence_threshold"], 0)
+            public = store.update({"field_confidence_threshold": "500"})
+            self.assertEqual(public["field_confidence_threshold"], 100)
+            public = store.update({"field_confidence_threshold": "70"})
+            self.assertEqual(public["field_confidence_threshold"], 70)
+            public = store.update({"field_confidence_threshold": "not-a-number"})
+            self.assertEqual(public["field_confidence_threshold"], 70)
+
     def test_empty_stream_update_does_not_clear_existing_streams(self):
         with tempfile.TemporaryDirectory() as directory:
             store = ConfigStore(str(Path(directory) / "config.json"))

tests/test_entities.py

@@ -0,0 +1,21 @@
+import unittest
+
+from fgai.entities import is_public_ip
+
+
+class EntitiesTests(unittest.TestCase):
+    def test_is_public_ip_true_for_global_address(self):
+        self.assertTrue(is_public_ip("8.8.8.8"))
+
+    def test_is_public_ip_false_for_private_address(self):
+        self.assertFalse(is_public_ip("10.0.0.5"))
+        self.assertFalse(is_public_ip("192.168.1.1"))
+
+    def test_is_public_ip_false_for_invalid_value(self):
+        self.assertFalse(is_public_ip("not-an-ip"))
+        self.assertFalse(is_public_ip(""))
+        self.assertFalse(is_public_ip(None))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_field_shapes.py

@@ -0,0 +1,88 @@
+import unittest
+
+from fgai.field_shapes import classify_field_shape, parse_iso_timestamp, private_ip_ratio
+
+
+class FieldShapesTests(unittest.TestCase):
+    def test_classifies_ip_shape_without_name_hint(self):
+        values = [f"203.0.113.{n}" for n in range(1, 12)]
+        result = classify_field_shape(values, field_name="x7_field_a")
+        self.assertEqual(result.primary_shape, "ip")
+        self.assertGreater(result.confidence, 60)
+
+    def test_classifies_timestamp_shape_iso(self):
+        values = [f"2026-09-0{n}T12:00:00Z" for n in range(1, 8)]
+        result = classify_field_shape(values, field_name="whatever")
+        self.assertEqual(result.primary_shape, "timestamp")
+
+    def test_classifies_timestamp_shape_epoch(self):
+        values = [str(1757000000 + n) for n in range(10)]
+        result = classify_field_shape(values, field_name="whatever")
+        self.assertEqual(result.primary_shape, "timestamp")
+
+    def test_classifies_port_shape_with_name_hint(self):
+        values = ["80", "443", "22", "3389", "80", "443", "80"]
+        without_name = classify_field_shape(values)
+        with_name = classify_field_shape(values, field_name="dst_port")
+        self.assertEqual(with_name.primary_shape, "port")
+        self.assertGreaterEqual(with_name.confidence, without_name.scores.get("port", 0))
+
+    def test_small_numeric_counter_prefers_numeric_over_port(self):
+        values = ["1", "2", "3", "4", "5", "2", "3"]
+        result = classify_field_shape(values, field_name="retry_count")
+        self.assertEqual(result.primary_shape, "numeric")
+
+    def test_classifies_categorical_low_cardinality(self):
+        values = ["allow", "deny", "allow", "reset", "allow", "deny"]
+        result = classify_field_shape(values, field_name="whatever")
+        self.assertEqual(result.primary_shape, "categorical_low_cardinality")
+
+    def test_classifies_free_text(self):
+        values = [
+            "user bob logged in from workstation A with elevated token set",
+            "connection reset by peer during TLS handshake on segment 4",
+            "unexpected EOF while parsing payload chunk 91 from upstream",
+            "cache miss for key group-7 falling back to origin lookup path",
+        ]
+        result = classify_field_shape(values, field_name="message")
+        self.assertEqual(result.primary_shape, "free_text")
+
+    def test_name_bonus_does_not_win_without_shape_evidence(self):
+        # Free-text values named "srcip" should not be classified as ip just because of the name.
+        values = [
+            "user bob logged in from workstation A with elevated token set",
+            "connection reset by peer during TLS handshake on segment 4",
+            "unexpected EOF while parsing payload chunk 91 from upstream",
+        ]
+        result = classify_field_shape(values, field_name="srcip")
+        self.assertNotEqual(result.primary_shape, "ip")
+
+    def test_name_bonus_is_bounded(self):
+        values = [f"203.0.113.{n}" for n in range(1, 5)]
+        without_name = classify_field_shape(values)
+        with_name = classify_field_shape(values, field_name="srcip")
+        self.assertLessEqual(with_name.confidence - without_name.confidence, 15)
+
+    def test_empty_values_return_zero_confidence(self):
+        result = classify_field_shape([], field_name="whatever")
+        self.assertEqual(result.confidence, 0)
+
+    def test_parse_iso_timestamp(self):
+        self.assertIsNotNone(parse_iso_timestamp("2026-09-07T12:00:00Z"))
+        self.assertIsNone(parse_iso_timestamp("not-a-timestamp"))
+
+    def test_private_ip_ratio_all_private(self):
+        self.assertEqual(private_ip_ratio(["10.0.0.1", "10.0.0.2", "192.168.1.1"]), 1.0)
+
+    def test_private_ip_ratio_all_public(self):
+        self.assertEqual(private_ip_ratio(["8.8.8.8", "1.1.1.1"]), 0.0)
+
+    def test_private_ip_ratio_mixed(self):
+        self.assertAlmostEqual(private_ip_ratio(["10.0.0.1", "8.8.8.8"]), 0.5)
+
+    def test_private_ip_ratio_ignores_non_ip_values(self):
+        self.assertEqual(private_ip_ratio(["not-an-ip", "also-not"]), 0.0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_history.py

@@ -0,0 +1,71 @@
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+from fgai.history import FieldDiscoveryStore
+
+
+class FieldDiscoveryConfidenceHistoryTests(unittest.TestCase):
+    def test_record_and_read_applied_confidence(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = FieldDiscoveryStore(str(Path(tmp) / "history.sqlite3"))
+            store.record_applied_confidence(
+                "custom_vendor",
+                ["x7_field_a", "x7_field_b"],
+                field_confidence={"x7_field_a": 85, "x7_field_b": 40},
+                field_shapes={"x7_field_a": "ip", "x7_field_b": "categorical_low_cardinality"},
+            )
+
+            applied = store.applied_confidence("custom_vendor")
+
+            self.assertEqual(applied["x7_field_a"]["applied_confidence"], 85)
+            self.assertEqual(applied["x7_field_a"]["applied_shape"], "ip")
+            self.assertEqual(applied["x7_field_b"]["applied_confidence"], 40)
+
+    def test_record_ignores_fields_without_confidence_entry(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = FieldDiscoveryStore(str(Path(tmp) / "history.sqlite3"))
+            store.record_applied_confidence(
+                "custom_vendor",
+                ["x7_field_a", "timestamp"],
+                field_confidence={"x7_field_a": 85},
+                field_shapes={"x7_field_a": "ip"},
+            )
+
+            applied = store.applied_confidence("custom_vendor")
+
+            self.assertIn("x7_field_a", applied)
+            self.assertNotIn("timestamp", applied)
+
+    def test_record_keeps_first_snapshot_and_ignores_later_calls(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = FieldDiscoveryStore(str(Path(tmp) / "history.sqlite3"))
+            store.record_applied_confidence("custom_vendor", ["x7_field_a"], {"x7_field_a": 65}, {"x7_field_a": "ip"})
+            store.record_applied_confidence("custom_vendor", ["x7_field_a"], {"x7_field_a": 90}, {"x7_field_a": "ip"})
+
+            applied = store.applied_confidence("custom_vendor")
+
+            self.assertEqual(applied["x7_field_a"]["applied_confidence"], 65)
+            self.assertEqual(len(applied), 1)
+
+    def test_confidence_history_maintenance_prunes_old_rows(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db_path = Path(tmp) / "history.sqlite3"
+            store = FieldDiscoveryStore(str(db_path))
+            store.record_applied_confidence("custom_vendor", ["x7_field_a"], {"x7_field_a": 85}, {"x7_field_a": "ip"})
+
+            import sqlite3
+            old_time = int(time.time()) - 200 * 86400
+            with sqlite3.connect(db_path) as connection:
+                connection.execute("update field_confidence_history set applied_at = ?", (old_time,))
+
+            result = store.confidence_history_maintenance(retention_days=90)
+
+            self.assertEqual(result["removed"], 1)
+            self.assertEqual(result["remaining"], 0)
+            self.assertEqual(store.applied_confidence("custom_vendor"), {})
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_llm.py

@@ -1,9 +1,46 @@
 import unittest
 
-from fgai.llm import _json_object_from_text, compact_dashboard_analysis
+from fgai.llm import _compact_profile_suggestion, _json_object_from_text, compact_dashboard_analysis
 
 
 class LlmTests(unittest.TestCase):
+    def test_compact_profile_suggestion_without_fields_only_keeps_everything(self):
+        item = {
+            "stream_id": "windows",
+            "stream_name": "Windows",
+            "events": 10,
+            "common_fields": [{"field": "username", "coverage": 1.0}, {"field": "hostname", "coverage": 0.9}],
+            "discovery": {"top_fields": [{"field": "username"}, {"field": "hostname"}], "selected_fields": {}, "rejected_fields": []},
+            "profile": {"entity_fields": ["username"]},
+            "field_shapes": {"username": "categorical_low_cardinality", "hostname": "categorical_low_cardinality"},
+            "field_confidence": {"username": 80, "hostname": 40},
+        }
+
+        compact = _compact_profile_suggestion(item)
+
+        self.assertNotIn("fields_to_classify", compact)
+        self.assertEqual(len(compact["common_fields"]), 2)
+
+    def test_compact_profile_suggestion_with_fields_only_trims_to_subset(self):
+        item = {
+            "stream_id": "windows",
+            "stream_name": "Windows",
+            "events": 10,
+            "common_fields": [{"field": "username", "coverage": 1.0}, {"field": "hostname", "coverage": 0.9}],
+            "discovery": {"top_fields": [{"field": "username"}, {"field": "hostname"}], "selected_fields": {}, "rejected_fields": []},
+            "profile": {"entity_fields": ["username"]},
+            "field_shapes": {"username": "categorical_low_cardinality", "hostname": "categorical_low_cardinality"},
+            "field_confidence": {"username": 80, "hostname": 40},
+        }
+
+        compact = _compact_profile_suggestion(item, fields_only=["hostname"])
+
+        self.assertEqual(compact["fields_to_classify"], ["hostname"])
+        self.assertEqual([row["field"] for row in compact["common_fields"]], ["hostname"])
+        self.assertEqual([row["field"] for row in compact["top_fields"]], ["hostname"])
+        self.assertEqual(compact["field_shapes"], {"hostname": "categorical_low_cardinality"})
+        self.assertEqual(compact["field_confidence"], {"hostname": 40})
+
     def test_json_object_from_text_accepts_markdown_wrapped_json(self):
         payload = _json_object_from_text(
             '```json\n{"profiles":[{"stream_id":"windows","entity_fields":["username"]}]}\n```'

tests/test_monitor.py

@@ -4,6 +4,7 @@ import unittest
 from pathlib import Path
 from unittest.mock import patch
 
+from fgai.config import ConfigStore
 from fgai.history import StatusSnapshotStore
 from fgai.history import FieldDiscoveryStore
 from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
@@ -230,7 +231,15 @@ class MonitorTests(unittest.TestCase):
     def test_profile_advisor_skips_when_all_streams_have_profiles(self):
         with tempfile.TemporaryDirectory() as tmp:
             log_path = Path(tmp) / "events.log"
-            log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice\n", encoding="utf-8")
+            names = ["alice", "bob", "carol"]
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=windows fgai_stream=Windows username={names[index % 3]}"
+                    for index in range(25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
             config_path = Path(tmp) / "config.json"
             config_path.write_text(
                 json.dumps(
@@ -269,6 +278,139 @@ class MonitorTests(unittest.TestCase):
             advisor.assert_called_once()
             self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty")
 
+    def test_profile_advisor_runs_for_low_confidence_field_even_with_full_profile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            # Mixed-shape, low-sample-count values keep "username"'s classified
+            # confidence below the default threshold even though its name already
+            # makes it the entity field and the profile's field set is unchanged.
+            log_path.write_text(
+                "fgai_stream_id=windows fgai_stream=Windows username=alice\n"
+                "fgai_stream_id=windows fgai_stream=Windows username=99\n",
+                encoding="utf-8",
+            )
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(
+                json.dumps(
+                    {
+                        "profile_advisor_enabled": True,
+                        "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}],
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            with patch("fgai.monitor.ollama_profile_advice", return_value=[]) as advisor:
+                status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            advisor.assert_called_once()
+            _, kwargs = advisor.call_args
+            self.assertIn("username", kwargs["ambiguous_fields_by_stream"].get("windows", []))
+            self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty")
+
+    def test_auto_apply_writes_high_confidence_new_stream_profile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
+                    for index in range(1, 25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8")
+
+            build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            saved = ConfigStore(str(config_path)).read()
+            profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]}
+            self.assertIn("custom_vendor", profiles)
+            self.assertIn("x7_field_a", profiles["custom_vendor"]["entity_fields"])
+            self.assertEqual(profiles["custom_vendor"]["field_origin"]["x7_field_a"], "auto")
+
+    def test_auto_apply_never_overwrites_a_manual_origin_field(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
+                    for index in range(1, 25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(
+                json.dumps({
+                    "auto_apply_profile_suggestions": True,
+                    "graylog_stream_profiles": [{
+                        "stream_id": "custom_vendor",
+                        "name": "Custom Vendor profile",
+                        "entity_field": "x7_field_a",
+                        "entity_fields": ["x7_field_a"],
+                        "timestamp_field": "timestamp",
+                        "field_origin": {"x7_field_a": "manual"},
+                    }],
+                }),
+                encoding="utf-8",
+            )
+
+            build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            saved = ConfigStore(str(config_path)).read()
+            profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]}
+            self.assertEqual(profiles["custom_vendor"]["field_origin"], {"x7_field_a": "manual"})
+
+    def test_profile_drift_flags_confidence_drop_without_changing_live_profile(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            config_path = Path(tmp) / "config.json"
+            history_path = Path(tmp) / "history.sqlite3"
+            config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8")
+
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
+                    for index in range(1, 25)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            first = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
+            applied_profile = {item["stream_id"]: item for item in first["stream_profiles"]}["custom_vendor"]
+            self.assertEqual(applied_profile["field_origin"]["x7_field_a"], "auto")
+            self.assertEqual(first["profile_drift"], [])
+
+            log_path.write_text(
+                "\n".join(
+                    f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=some odd free text value number {index}"
+                    for index in range(1, 30)
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            second = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            drift_entries = {(item["stream_id"], item["field"]) for item in second["profile_drift"]}
+            self.assertIn(("custom_vendor", "x7_field_a"), drift_entries)
+            still_applied_profile = {item["stream_id"]: item for item in second["stream_profiles"]}["custom_vendor"]
+            self.assertIn("x7_field_a", still_applied_profile["entity_fields"])
+
+    def test_confidence_history_maintenance_runs_each_cycle(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            log_path = Path(tmp) / "events.log"
+            log_path.write_text("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login\n", encoding="utf-8")
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(json.dumps({}), encoding="utf-8")
+            history_path = Path(tmp) / "history.sqlite3"
+
+            status = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
+
+            self.assertIn("confidence_history_maintenance", status["profile_auto_apply"])
+            self.assertIn("remaining", status["profile_auto_apply"]["confidence_history_maintenance"])
+
     def test_profile_suggestions_use_cached_discovered_fields(self):
         with tempfile.TemporaryDirectory() as tmp:
             history_path = str(Path(tmp) / "history.sqlite3")

tests/test_profile_suggestions.py

@@ -276,6 +276,39 @@ class ProfileSuggestionTests(unittest.TestCase):
         self.assertEqual(advised["profile_advisor"]["field_roles"], {"username": "identity"})
         self.assertEqual(advised["profile_advisor"]["correlation_roles"], {"identity": ["username"]})
 
+    def test_scoped_field_roles_patch_only_the_named_fields(self):
+        suggestion = suggest_stream_profiles([
+            parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure")
+        ])[0]
+
+        advised = apply_profile_advice([suggestion], [{
+            "stream_id": "windows",
+            "field_roles": {"hostname": "entity", "action": "reject"},
+            "reason": "scoped advisor pass",
+        }])[0]
+
+        self.assertEqual(advised["profile_advisor"]["status"], "ok")
+        self.assertEqual(advised["profile_advisor"]["fields_reclassified"], 2)
+        self.assertIn("username", advised["profile"]["entity_fields"])
+        self.assertIn("hostname", advised["profile"]["entity_fields"])
+        self.assertNotIn("action", advised["profile"]["categorical_fields"])
+
+    def test_scoped_advisor_never_rejects_the_only_entity_field(self):
+        events = [
+            parse_log_line(f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}")
+            for index in range(1, 25)
+        ]
+        suggestion = suggest_stream_profiles(events)[0]
+        self.assertEqual(suggestion["profile"]["entity_fields"], ["x7_field_a"])
+
+        advised = apply_profile_advice([suggestion], [{
+            "stream_id": "custom_vendor",
+            "field_roles": {"x7_field_a": "reject"},
+            "reason": "scoped advisor pass",
+        }])[0]
+
+        self.assertIn("x7_field_a", advised["profile"]["entity_fields"])
+
     def test_missing_llm_advice_keeps_heuristic_profile(self):
         suggestion = suggest_stream_profiles([
             parse_log_line("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login")
@@ -286,6 +319,64 @@ class ProfileSuggestionTests(unittest.TestCase):
         self.assertEqual(advised["profile_advisor"]["status"], "heuristic")
         self.assertEqual(advised["profile"], suggestion["profile"])
 
+    def test_unfamiliar_field_name_selected_as_entity_from_value_shape_alone(self):
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index} "
+                f"x7_field_b=some-{index % 5} timestamp=2026-06-29T10:00:{index:02d}Z"
+            )
+            for index in range(1, 25)
+        ]
+
+        suggestion = suggest_stream_profiles(events)[0]
+
+        self.assertIn("x7_field_a", suggestion["profile"]["entity_fields"])
+        self.assertEqual(suggestion["field_shapes"]["x7_field_a"], "ip")
+        self.assertGreaterEqual(suggestion["field_confidence"]["x7_field_a"], 60)
+
+    def test_ambiguous_ip_fields_disambiguated_by_private_ratio(self):
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor field_alpha=10.0.0.{index} "
+                f"field_beta=8.8.8.{index} timestamp=2026-06-29T10:00:{index:02d}Z"
+            )
+            for index in range(1, 25)
+        ]
+
+        suggestion = suggest_stream_profiles(events)[0]
+        relationships = suggestion["profile"]["relationship_fields"]
+
+        pairs = {(item["left"], item["right"]) for item in relationships}
+        self.assertIn(("field_alpha", "field_beta"), pairs)
+
+    def test_deny_action_detector_suggested_from_value_pattern_alone(self):
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor whatever_field="
+                f"{'Denied' if index % 3 else 'Blocked'} src=10.0.0.{index} timestamp=2026-06-29T10:00:{index:02d}Z"
+            )
+            for index in range(1, 25)
+        ]
+
+        suggestion = suggest_stream_profiles(events)[0]
+        detectors = suggestion["profile"]["detectors"]
+
+        self.assertIn("deny_action", detectors)
+        self.assertEqual(detectors["deny_action"].get("source"), "value_shape")
+
+    def test_every_suggestion_carries_field_confidence_and_shapes(self):
+        events = [
+            parse_log_line(f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index} dstip=8.8.8.8 dstport=443 action=deny timestamp=2026-06-29T10:00:{index:02d}Z hitcount={index}")
+            for index in range(1, 25)
+        ]
+
+        suggestion = suggest_stream_profiles(events)[0]
+
+        self.assertIn("field_confidence", suggestion)
+        self.assertIn("field_shapes", suggestion)
+        self.assertIn("srcip", suggestion["field_shapes"])
+        self.assertEqual(suggestion["field_shapes"]["srcip"], "ip")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_stream_profiles.py

@@ -35,3 +35,34 @@ class StreamProfileTests(unittest.TestCase):
         self.assertEqual(relationships[0].left, "username")
         self.assertEqual(relationships[0].right, "srcip")
         self.assertEqual(relationships[0].name, "user source IP")
+
+    def test_field_origin_defaults_missing_fields_to_manual(self):
+        profiles = parse_profiles([{
+            "stream_id": "windows",
+            "entity_field": "username",
+            "entity_fields": ["username", "hostname"],
+            "categorical_fields": ["action"],
+            "field_origin": {"hostname": "auto"},
+        }])
+
+        origin = profiles["windows"].field_origin
+        self.assertEqual(origin["username"], "manual")
+        self.assertEqual(origin["hostname"], "auto")
+        self.assertEqual(origin["action"], "manual")
+
+    def test_field_origin_round_trips_and_rejects_unknown_values(self):
+        profiles = parse_profiles([{
+            "stream_id": "windows",
+            "entity_field": "username",
+            "field_origin": {"username": "bogus"},
+            "updated_at": 1757000000,
+        }])
+
+        self.assertEqual(profiles["windows"].field_origin["username"], "manual")
+        self.assertEqual(profiles["windows"].updated_at, 1757000000)
+
+    def test_missing_field_origin_and_updated_at_default_safely(self):
+        profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username"}])
+
+        self.assertEqual(profiles["windows"].field_origin, {"username": "manual"})
+        self.assertEqual(profiles["windows"].updated_at, 0)