from __future__ import annotations

from collections import Counter, defaultdict

from .detectors import event_detector_categories
from .entities import ENTITY_FIELDS
from .models import LogEvent
from .normalization import FIELD_ALIASES

ALIAS_CANONICAL_BY_FIELD = {
    alias: canonical
    for canonical, aliases in FIELD_ALIASES.items()
    for alias in aliases
}

ENTITY_PRIORITY = (
    "srcip",
    "source_ip",
    "client_ip",
    "username",
    "actor_id",
    "actor",
    "principal",
    "user",
    "user_name",
    "account",
    "account_name",
    "targetusername",
    "subjectusername",
    "winlog_event_data_targetusername",
    "winlog_event_data_subjectusername",
    "hostname",
    "host",
    "computer",
    "computer_name",
    "winlog_computer_name",
    "source",
    "ip",
    "winlog_event_data_ipaddress",
)
TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at", "winlog_timecreated", "event_created")
CATEGORICAL_PRIORITY = (
    "action",
    "severity",
    "type",
    "subtype",
    "service",
    "dstport",
    "srcport",
    "policyid",
    "eventid",
    "event_id",
    "winlog_event_id",
    "event_code",
    "channel",
    "provider_name",
    "logon_type",
    "dns_query",
    "query_domain",
    "url",
    "username",
    "hostname",
    "eventid",
    "context",
)
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
WINDOWS_ENTITY_PRIORITY = (
    "username",
    "user.name",
    "user.id",
    "hostname",
    "host.name",
    "srcip",
    "source.ip",
    "client.ip",
    "source",
    "targetusername",
    "winlog_event_data_targetusername",
    "winlog.event_data.targetusername",
    "winlog.event_data.target_user_name",
    "subjectusername",
    "winlog_event_data_subjectusername",
    "winlog.event_data.subjectusername",
    "winlog.event_data.subject_user_name",
    "user",
    "account_name",
    "computer",
    "computer_name",
    "winlog_computer_name",
    "winlog.computer_name",
    "agent.hostname",
    "source_ip",
    "client_ip",
    "winlog_event_data_ipaddress",
    "winlog.event_data.ipaddress",
    "winlog.event_data.ip_address",
    "ipaddress",
)
WINDOWS_CATEGORICAL_PRIORITY = (
    "eventid",
    "event_id",
    "winlog_event_id",
    "winlog.event_id",
    "event_code",
    "event.code",
    "action",
    "event.action",
    "event.outcome",
    "severity",
    "channel",
    "winlog.channel",
    "provider_name",
    "event_provider",
    "event.provider",
    "winlog_provider_name",
    "winlog.provider_name",
    "logon_type",
    "winlog.event_data.logontype",
    "winlog.event_data.logon_type",
    "event.category",
    "event.type",
    "event.kind",
    "event.module",
    "event.dataset",
    "task",
    "winlog.task",
    "opcode",
    "winlog.opcode",
    "keywords",
    "winlog.keywords",
    "process_name",
    "process.name",
    "process.executable",
    "image",
    "service_name",
    "service.name",
    "authentication_package_name",
    "winlog.event_data.authenticationpackagename",
    "logon_process_name",
    "winlog.event_data.logonprocessname",
    "failure_reason",
    "winlog.event_data.failurereason",
    "sub_status",
    "winlog.event_data.substatus",
    "status",
    "winlog.event_data.status",
)
WINDOWS_NUMERIC_PRIORITY = (
    "event_record_id",
    "record_id",
    "winlog_record_id",
    "winlog.record_id",
    "process_id",
    "process.pid",
    "winlog_process_pid",
    "winlog.process.pid",
    "thread_id",
    "winlog.process.thread.id",
)
ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"}
IGNORED_DISCOVERY_FIELDS = {
    "message",
    "msg",
    "full_message",
    "raw",
    "answer",
    "gl2_message_id",
    "gl2_source_node",
    "gl2_source_input",
    "fgai_stream",
    "fgai_stream_id",
    "fgai_stream_name",
}
EMPTY_DISCOVERY_VALUES = {"", "-", "--", "unknown", "n/a", "none", "null", "nil", "undefined", "[]", "{}"}


def _stream_id(event: LogEvent) -> str:
    return event.fields.get("fgai_stream_id") or event.fields.get("fgai_stream") or "local"


def _stream_name(event: LogEvent, stream_id: str) -> str:
    return event.fields.get("fgai_stream") or event.fields.get("fgai_stream_name") or stream_id


def _is_number(value: str) -> bool:
    try:
        float(value)
        return True
    except (TypeError, ValueError):
        return False


def _has_discovery_value(value: object) -> bool:
    text = str(value).strip()
    return text.lower() not in EMPTY_DISCOVERY_VALUES


def _pick_present(priority: tuple[str, ...], coverage: Counter[str], total: int, *, min_ratio: float = 0.05, limit: int = 6) -> list[str]:
    selected = []
    for field in priority:
        if coverage[field] and coverage[field] / max(1, total) >= min_ratio:
            selected.append(field)
        if len(selected) >= limit:
            break
    return selected


def _looks_like_entity_field(field: str) -> bool:
    field = field.lower()
    if any(field in fields for fields in ENTITY_FIELDS.values()):
        return True
    tokens = ("user", "account", "actor", "principal", "login", "identity", "subject", "target", "host", "computer", "workstation", "client", "source", "src", "remote", "ipaddress", "ip")
    return any(token in field for token in tokens)


def _looks_like_time_field(field: str) -> bool:
    field = field.lower()
    return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))


def _semantic_field_group(field: str) -> str:
    return _semantic_field_groups(field)[0]


def _semantic_field_groups(field: str) -> list[str]:
    normalized = field.lower()
    if normalized in ALIAS_CANONICAL_BY_FIELD:
        return [ALIAS_CANONICAL_BY_FIELD[normalized]]
    groups = []
    if normalized.startswith("lcs_"):
        groups.append("lcs_*")
    compact = normalized.replace(".", "_").replace("-", "_")
    parts = [part for part in compact.split("_") if part]
    if not parts:
        return [normalized]
    if parts[-1] in {"ip", "addr", "address"}:
        groups.append("*_ip")
    if parts[-1] in {"user", "username", "account", "principal", "actor", "name"} and any(part in {"user", "account", "principal", "actor"} for part in parts):
        groups.append("*_user")
    if parts[-1] in {"host", "hostname", "computer", "node", "name"} and any(part in {"host", "hostname", "computer", "node"} for part in parts):
        groups.append("*_host")
    if parts[-1] in {"id", "uuid", "guid"}:
        groups.append("*_id")
    if parts[-1] in {"name", "label", "title"}:
        groups.append("*_name")
    if parts[-1] in {"status", "state", "result", "outcome", "disposition"}:
        groups.append("*_status")
    if parts[-1] in {"action", "operation", "event"}:
        groups.append("*_action")
    if parts[-1] in {"type", "category", "class", "kind"}:
        groups.append("*_type")
    if parts[-1] in {"code", "reason"}:
        groups.append("*_code")
    if parts[-1] in {"port"}:
        groups.append("*_port")
    if parts[-1] in {"domain", "fqdn"}:
        groups.append("*_domain")
    if parts[-1] in {"url", "uri", "path"}:
        groups.append("*_url")
    groups.append(normalized)
    return list(dict.fromkeys(groups))


def _shared_field_kind(field: str, *, cardinality: int, total: int, numeric_ratio: float) -> str:
    if _looks_like_entity_field(field):
        return "entity"
    if _looks_like_time_field(field):
        return "time"
    if numeric_ratio >= 0.95:
        return "numeric"
    if cardinality > min(200, max(8, int(total * 0.7))):
        return "high_cardinality"
    return "categorical"


def _looks_like_windows_stream(stream_id: str, stream_name: str, coverage: Counter[str]) -> bool:
    name = f"{stream_id} {stream_name}".lower()
    if any(token in name for token in ("windows", "winlog", "event log", "security event", "sysmon", "powershell")):
        return True
    windows_fields = {
        "winlog_event_id",
        "winlog.event_id",
        "winlog_computer_name",
        "winlog.computer_name",
        "winlog_event_data_targetusername",
        "winlog.event_data.targetusername",
        "winlog_event_data_subjectusername",
        "winlog.event_data.subjectusername",
        "eventid",
        "event_code",
        "event.code",
        "logon_type",
        "winlog.channel",
    }
    return sum(1 for field in windows_fields if coverage[field]) >= 2


def _compact_windows_fields(fields: list[str]) -> list[str]:
    output = list(dict.fromkeys(fields))
    alias_groups = (
        ("username", "user.name", "user.id", "targetusername", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog.event_data.target_user_name", "subjectusername", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername", "winlog.event_data.subject_user_name", "user", "account_name"),
        ("hostname", "host.name", "computer", "computer_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname"),
        ("srcip", "source.ip", "source_ip", "client.ip", "client_ip", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "ipaddress"),
        ("eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code"),
        ("action", "event.action", "event.outcome"),
    )
    for canonical, *aliases in alias_groups:
        if canonical in output:
            output = [field for field in output if field == canonical or field not in aliases]
    return output


def _detected_log_type(stream_id: str, stream_name: str, coverage: Counter[str]) -> str:
    name = f"{stream_id} {stream_name}".lower()
    if _looks_like_windows_stream(stream_id, stream_name, coverage):
        return "windows"
    if any(token in name for token in ("fortigate", "firewall", "pan-os", "checkpoint", "iptables", "ufw")) or (
        coverage["policyid"] and (coverage["dstip"] or coverage["dstport"]) and coverage["action"]
    ):
        return "firewall"
    if any(token in name for token in ("dns", "adguard", "bind", "unbound")) or coverage["dns_query"] or coverage["query_domain"] or coverage["qh"]:
        return "dns"
    if any(token in name for token in ("squid", "proxy", "nginx", "apache", "iis", "web")) or coverage["url"] or coverage["request_uri"] or coverage["http_method"]:
        return "web_proxy"
    if any(token in name for token in ("audit", "app", "application", "serverlog", "prod")):
        return "application"
    return "unknown"


def _field_stats(coverage: Counter[str], unique_values: dict[str, set[str]], numeric_counts: Counter[str], total: int) -> list[dict[str, object]]:
    rows = []
    for field, count in coverage.most_common():
        if field in IGNORED_DISCOVERY_FIELDS:
            continue
        numeric_ratio = numeric_counts[field] / max(1, count)
        rows.append({
            "field": field,
            "coverage": round(count / max(1, total), 2),
            "unique_values": len(unique_values[field]),
            "numeric_ratio": round(numeric_ratio, 2),
        })
    return rows


def _rejected_fields(
    coverage: Counter[str],
    unique_values: dict[str, set[str]],
    numeric_counts: Counter[str],
    total: int,
    selected_fields: set[str],
    *,
    limit: int = 12,
) -> list[dict[str, object]]:
    rejected = []
    for field, count in coverage.most_common():
        if len(rejected) >= limit:
            break
        coverage_ratio = count / max(1, total)
        cardinality = len(unique_values[field])
        reason = ""
        if field in selected_fields:
            continue
        if field in IGNORED_DISCOVERY_FIELDS:
            reason = "raw/internal field"
        elif coverage_ratio < 0.1:
            reason = "low coverage"
        elif cardinality <= 1:
            reason = "constant value"
        elif cardinality > min(200, max(8, int(total * 0.7))):
            reason = "too high cardinality"
        elif numeric_counts[field] and numeric_counts[field] / max(1, count) < 0.95:
            reason = "mixed numeric/text values"
        else:
            reason = "lower signal than selected fields"
        rejected.append({
            "field": field,
            "reason": reason,
            "coverage": round(coverage_ratio, 2),
            "unique_values": cardinality,
        })
    return rejected


def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
    output = list(selected)
    for field, count in coverage.most_common():
        if len(output) >= 4:
            break
        if field in output or field in IGNORED_DISCOVERY_FIELDS or not _looks_like_entity_field(field):
            continue
        coverage_ratio = count / max(1, total)
        cardinality = len(unique_values[field])
        if coverage_ratio >= 0.05 and 1 < cardinality <= min(500, max(10, total)):
            output.append(field)
    return output


def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
    output = list(selected)
    for field, count in coverage.most_common():
        if len(output) >= 12:
            break
        if field in output or field in IGNORED_DISCOVERY_FIELDS:
            continue
        coverage_ratio = count / max(1, total)
        cardinality = len(unique_values[field])
        if coverage_ratio >= 0.1 and 1 < cardinality <= min(200, max(8, int(total * 0.7))):
            output.append(field)
    return output


def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str], total: int, selected: list[str]) -> list[str]:
    output = list(selected)
    for field, count in coverage.most_common():
        if len(output) >= 8:
            break
        if field in output or field in IGNORED_DISCOVERY_FIELDS:
            continue
        if count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95:
            output.append(field)
    return output


def _relationship_name(left: str, right: str) -> str:
    return f"{left} to {right}"


def _field_matches(field: str, groups: set[str], tokens: set[str]) -> bool:
    lower = field.lower()
    semantic = set(_semantic_field_groups(lower))
    return bool(semantic & groups) or any(token in lower for token in tokens)


def _relationship_candidates(
    coverage: Counter[str],
    unique_values: dict[str, set[str]],
    total: int,
    entity_fields: list[str],
    categorical_fields: list[str],
    numeric_fields: list[str],
    *,
    limit: int = 6,
) -> list[dict[str, str]]:
    def usable(field: str) -> bool:
        return (
            bool(field)
            and field in coverage
            and field not in IGNORED_DISCOVERY_FIELDS
            and coverage[field] / max(1, total) >= 0.02
            and len(unique_values[field]) > 1
        )

    candidates: list[tuple[str, str]] = []
    entities = [field for field in entity_fields if usable(field)]
    categorical = [field for field in categorical_fields if usable(field)]
    numeric = [field for field in numeric_fields if usable(field)]

    identity_fields = [
        field for field in entities
        if _field_matches(field, {"username", "*_user"}, {"user", "account", "actor", "principal", "login", "identity"})
    ]
    source_fields = [
        field for field in entities
        if _field_matches(field, {"srcip", "*_ip"}, {"src", "source", "client", "remote", "ipaddress", "clientip"})
    ]
    host_fields = [
        field for field in entities
        if _field_matches(field, {"*_host"}, {"host", "hostname", "computer", "workstation", "device"})
    ]
    destination_fields = [
        field for field in [*entities, *categorical]
        if _field_matches(field, {"dstip", "*_ip"}, {"dst", "dest", "destination", "server"})
    ]
    destination_fields.extend(field for field in categorical if field.lower() in {"dstport", "destination_port", "dest_port", "service", "application", "app", "url", "query_domain", "dns_query"})

    for left in identity_fields:
        for right in [*source_fields, *host_fields]:
            if left != right:
                candidates.append((left, right))
    for left in source_fields:
        for right in destination_fields:
            if left != right:
                candidates.append((left, right))
    for left in host_fields:
        for right in [field for field in categorical if any(token in field.lower() for token in ("process", "service", "app", "action", "status", "event"))]:
            if left != right:
                candidates.append((left, right))

    behavior_fields = [
        field for field in [*categorical, *numeric]
        if field not in entities
        and not _looks_like_time_field(field)
        and not any(token in field.lower() for token in ("gl2_", "message_id", "sort_field", "accounted_message_size", "processing_timestamp"))
    ]
    for left in entities:
        for right in behavior_fields[:4]:
            if left != right:
                candidates.append((left, right))

    output = []
    seen = set()
    for left, right in candidates:
        key = (left.lower(), right.lower())
        if key in seen:
            continue
        seen.add(key)
        output.append({"left": left, "right": right, "name": _relationship_name(left, right)})
        if len(output) >= limit:
            break
    return output


def _shared_profile_fields(
    coverage: Counter[str],
    unique_values: dict[str, set[str]],
    numeric_counts: Counter[str],
    total: int,
    stream_field_counts: Counter[str],
    stream_semantic_counts: Counter[str],
    stream_semantic_fields: dict[str, set[str]],
) -> tuple[list[str], list[str], list[dict[str, object]]]:
    categorical: list[str] = []
    numeric: list[str] = []
    stats_by_key: dict[str, dict[str, object]] = {}
    for field, stream_count in stream_field_counts.most_common():
        if stream_count < 2 or field in IGNORED_DISCOVERY_FIELDS or not coverage[field]:
            continue
        coverage_ratio = coverage[field] / max(1, total)
        cardinality = len(unique_values[field])
        if coverage_ratio < 0.1 or cardinality <= 1:
            continue
        numeric_ratio = numeric_counts[field] / max(1, coverage[field])
        kind = _shared_field_kind(field, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio)
        candidate = kind in {"categorical", "numeric"}
        stats_by_key.setdefault(field, {
            "field": field,
            "streams": stream_count,
            "coverage": round(coverage_ratio, 2),
            "unique_values": cardinality,
            "kind": kind,
            "candidate": candidate,
            "semantic_group": _semantic_field_group(field),
        })
        if kind == "entity":
            pass
        elif kind == "numeric":
            numeric.append(field)
        elif kind == "categorical":
            categorical.append(field)
    for group, stream_count in stream_semantic_counts.most_common():
        if stream_count < 2 or group in IGNORED_DISCOVERY_FIELDS:
            continue
        aliases = sorted(stream_semantic_fields.get(group, set()))
        present_aliases = [field for field in aliases if coverage[field]]
        if not present_aliases:
            continue
        best = max(present_aliases, key=lambda field: coverage[field])
        coverage_ratio = coverage[best] / max(1, total)
        cardinality = len(unique_values[best])
        numeric_ratio = numeric_counts[best] / max(1, coverage[best])
        kind = _shared_field_kind(group, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio)
        if group not in stats_by_key:
            stats_by_key[group] = {
                "field": group,
                "streams": stream_count,
                "coverage": round(coverage_ratio, 2),
                "unique_values": cardinality,
                "kind": kind,
                "candidate": kind in {"categorical", "numeric"},
                "semantic_group": group,
                "aliases": aliases[:10],
            }
        else:
            stats_by_key[group].setdefault("aliases", aliases[:10])
            stats_by_key[group]["streams"] = max(int(stats_by_key[group].get("streams", 0) or 0), stream_count)
        if kind == "numeric":
            for field in present_aliases:
                if field not in numeric:
                    numeric.append(field)
        elif kind == "categorical":
            for field in present_aliases:
                if field not in categorical:
                    categorical.append(field)
    stats = sorted(
        stats_by_key.values(),
        key=lambda item: (
            0 if item.get("kind") in {"entity", "time"} else 1,
            -int(item.get("streams", 0) or 0),
            -float(item.get("coverage", 0) or 0),
            str(item.get("field", "")),
        ),
    )
    return list(dict.fromkeys(categorical))[:8], list(dict.fromkeys(numeric))[:5], stats[:16]


def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
    profile = suggestion.get("profile", {}) if isinstance(suggestion.get("profile"), dict) else {}
    fields = {str(item.get("field", "")) for item in suggestion.get("common_fields", []) if isinstance(item, dict)}
    discovery = suggestion.get("discovery", {}) if isinstance(suggestion.get("discovery"), dict) else {}
    for item in discovery.get("top_fields", []) if isinstance(discovery.get("top_fields"), list) else []:
        if isinstance(item, dict):
            fields.add(str(item.get("field", "")))
    for item in suggestion.get("shared_fields", []) if isinstance(suggestion.get("shared_fields"), list) else []:
        if not isinstance(item, dict):
            continue
        fields.add(str(item.get("field", "")))
        aliases = item.get("aliases", [])
        if isinstance(aliases, list):
            fields.update(str(alias) for alias in aliases if alias)
    selected = discovery.get("selected_fields", {}) if isinstance(discovery.get("selected_fields"), dict) else {}
    for value in selected.values():
        if isinstance(value, list):
            fields.update(str(field) for field in value if field)
    for key in ("entity_fields", "categorical_fields", "numeric_fields"):
        value = profile.get(key, [])
        if isinstance(value, list):
            fields.update(str(item) for item in value if item)
    if profile.get("timestamp_field"):
        fields.add(str(profile["timestamp_field"]))
    return {field for field in fields if field and field not in IGNORED_DISCOVERY_FIELDS}


def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict[str, object]]) -> list[dict[str, object]]:
    advice_by_stream = {str(item.get("stream_id", "")): item for item in advice if isinstance(item, dict)}
    output = []
    for suggestion in suggestions:
        stream_id = str(suggestion.get("stream_id", ""))
        item = dict(suggestion)
        advisor = advice_by_stream.get(stream_id)
        if not advisor:
            item["profile_advisor"] = {"status": "heuristic", "reason": "advisor returned no profile for this stream"}
            output.append(item)
            continue
        allowed_fields = _allowed_fields(item)
        def valid_fields(key: str, fallback: list[str]) -> list[str]:
            value = advisor.get(key, [])
            fields = [str(field) for field in value if str(field) in allowed_fields] if isinstance(value, list) else []
            return list(dict.fromkeys(fields or fallback))
        heuristic = item.get("profile", {}) if isinstance(item.get("profile"), dict) else {}
        entity_fields = valid_fields("entity_fields", list(heuristic.get("entity_fields", [])) if isinstance(heuristic.get("entity_fields"), list) else [])
        timestamp = str(advisor.get("timestamp_field", ""))
        if timestamp not in allowed_fields:
            timestamp = str(heuristic.get("timestamp_field", "timestamp"))
        categorical = valid_fields("categorical_fields", list(heuristic.get("categorical_fields", [])) if isinstance(heuristic.get("categorical_fields"), list) else [])
        numeric = valid_fields("numeric_fields", list(heuristic.get("numeric_fields", [])) if isinstance(heuristic.get("numeric_fields"), list) else [])
        relationships = []
        raw_relationships = advisor.get("relationship_fields", [])
        if isinstance(raw_relationships, list):
            for relation in raw_relationships:
                if not isinstance(relation, dict):
                    continue
                left = str(relation.get("left", "")).strip()
                right = str(relation.get("right", "")).strip()
                if left not in allowed_fields or right not in allowed_fields or left == right:
                    continue
                relationships.append({"left": left, "right": right, "name": str(relation.get("name", "") or f"{left}->{right}")})
        if not relationships and isinstance(heuristic.get("relationship_fields"), list):
            relationships = [item for item in heuristic.get("relationship_fields", []) if isinstance(item, dict)]
        detectors_raw = advisor.get("detectors", {})
        detectors = {}
        if isinstance(detectors_raw, dict):
            for name, settings in detectors_raw.items():
                if str(name) not in ALLOWED_DETECTORS or not isinstance(settings, dict):
                    continue
                try:
                    detectors[str(name)] = {"enabled": bool(settings.get("enabled", True)), "minimum": max(1, int(settings.get("minimum", 1))), "z_threshold": max(1.0, float(settings.get("z_threshold", 3.0)))}
                except (TypeError, ValueError):
                    continue
        if not detectors:
            detectors = dict(heuristic.get("detectors", {})) if isinstance(heuristic.get("detectors"), dict) else {}
        if entity_fields:
            raw_field_roles = advisor.get("field_roles", {})
            field_roles = {}
            if isinstance(raw_field_roles, dict):
                field_roles = {
                    str(field): str(role)
                    for field, role in raw_field_roles.items()
                    if str(field) in allowed_fields and str(role)
                }
            raw_correlation_roles = advisor.get("correlation_roles", {})
            correlation_roles = {}
            if isinstance(raw_correlation_roles, dict):
                for role, fields in raw_correlation_roles.items():
                    if not isinstance(fields, list):
                        continue
                    valid = [str(field) for field in fields if str(field) in allowed_fields]
                    if valid:
                        correlation_roles[str(role)] = list(dict.fromkeys(valid))
            profile = {
                **heuristic,
                "entity_field": entity_fields[0],
                "entity_fields": entity_fields,
                "timestamp_field": timestamp,
                "categorical_fields": categorical,
                "numeric_fields": numeric,
                "detectors": detectors,
                "relationship_fields": relationships,
            }
            item.update({"profile": profile, "entity_fields": entity_fields, "timestamp_field": timestamp, "categorical_fields": categorical, "numeric_fields": numeric, "detectors": detectors, "relationship_fields": relationships})
            item["profile_advisor"] = {
                "status": "ok",
                "model": str(advisor.get("model", "")),
                "reason": str(advisor.get("reason", "")),
                "field_roles": field_roles,
                "correlation_roles": correlation_roles,
            }
        else:
            item["profile_advisor"] = {"status": "invalid", "reason": "advisor returned no valid entity fields"}
        output.append(item)
    return output


def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]:
    existing_profiles = existing_profiles or {}
    grouped: dict[str, list[LogEvent]] = defaultdict(list)
    names: dict[str, str] = {}
    for event in events:
        stream_id = _stream_id(event)
        grouped[stream_id].append(event)
        names.setdefault(stream_id, _stream_name(event, stream_id))
    stream_field_counts: Counter[str] = Counter()
    stream_semantic_counts: Counter[str] = Counter()
    stream_semantic_fields: dict[str, set[str]] = defaultdict(set)
    for stream_events in grouped.values():
        fields = {
            field
            for event in stream_events
            for field, value in event.fields.items()
            if field not in IGNORED_DISCOVERY_FIELDS and _has_discovery_value(value)
        }
        stream_field_counts.update(fields)
        semantic_groups = {group for field in fields for group in _semantic_field_groups(field)}
        stream_semantic_counts.update(semantic_groups)
        for field in fields:
            for group in _semantic_field_groups(field):
                stream_semantic_fields[group].add(field)

    suggestions: list[dict[str, object]] = []
    for stream_id, stream_events in grouped.items():
        total = len(stream_events)
        coverage: Counter[str] = Counter()
        unique_values: dict[str, set[str]] = defaultdict(set)
        numeric_counts: Counter[str] = Counter()
        detector_counts: Counter[str] = Counter()
        for event in stream_events:
            for field, value in event.fields.items():
                if not _has_discovery_value(value):
                    continue
                coverage[field] += 1
                if len(unique_values[field]) < 200:
                    unique_values[field].add(value)
                if _is_number(value):
                    numeric_counts[field] += 1
            detector_counts.update(event_detector_categories(event))

        stream_name = names.get(stream_id, stream_id)
        is_windows = _looks_like_windows_stream(stream_id, stream_name, coverage)
        log_type = _detected_log_type(stream_id, stream_name, coverage)
        entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
        categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
        numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
        shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields)
        entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3))
        timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
        if not timestamp:
            timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
        categorical_seed = list(dict.fromkeys([*_pick_present(categorical_priority, coverage, total, min_ratio=0.02, limit=10 if is_windows else 8), *shared_categorical]))
        categorical = _generic_categorical_fields(coverage, unique_values, total, categorical_seed)
        numeric_seed = [
            field for field in numeric_priority
            if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
        ][:5]
        numeric_seed = list(dict.fromkeys([*numeric_seed, *shared_numeric]))
        numeric = numeric_seed if is_windows else _generic_numeric_fields(coverage, numeric_counts, total, numeric_seed)
        if not entity_fields:
            entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
        if not entity_fields:
            entity_fields = ["source"]
        if is_windows:
            entity_fields = _compact_windows_fields(entity_fields)[:4]
            categorical = _compact_windows_fields(categorical)
            entity_set = set(entity_fields)
            categorical = [
                field for field in categorical
                if field not in entity_set and field not in {
                    "srcip", "source.ip", "source_ip", "client.ip", "client_ip",
                    "hostname", "host.name", "computer", "username", "user.name",
                    "source", "context",
                }
            ][:10]
            for field in WINDOWS_CATEGORICAL_PRIORITY:
                if len(categorical) >= 10:
                    break
                compacted = _compact_windows_fields([*categorical, field])
                if field not in compacted or field in categorical:
                    continue
                if not coverage[field] or len(unique_values[field]) <= 1:
                    continue
                if field in entity_set or field in {"srcip", "source.ip", "client.ip", "hostname", "host.name", "username", "user.name"}:
                    continue
                categorical = compacted
            numeric = [
                field for field in _compact_windows_fields(numeric)
                if field not in set(categorical) and field not in {"eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code", "logon_type"}
            ][:5]
        detectors = {
            name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
            for name, count in detector_counts.items()
            if count > 0
        }
        if is_windows and coverage["eventid"]:
            detectors.setdefault("auth_failure", {"enabled": True, "minimum": 5, "z_threshold": 3.0})
        if not detectors:
            detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
        relationship_fields = _relationship_candidates(coverage, unique_values, total, entity_fields, categorical, numeric)
        high_coverage = [
            {"field": field, "coverage": round(count / max(1, total), 2), "unique_values": len(unique_values[field])}
            for field, count in coverage.most_common(12)
        ]
        selected_fields = {timestamp, *entity_fields, *categorical, *numeric, *(item["left"] for item in relationship_fields), *(item["right"] for item in relationship_fields)}
        discovery = {
            "log_type": log_type,
            "field_count": len([field for field in coverage if field not in IGNORED_DISCOVERY_FIELDS]),
            "selected_fields": {
                "entity": entity_fields,
                "time": [timestamp] if timestamp else [],
                "categorical": categorical,
                "numeric": numeric,
                "relationships": [f"{item['left']}->{item['right']}" for item in relationship_fields],
            },
            "top_fields": _field_stats(coverage, unique_values, numeric_counts, total)[:16],
            "shared_fields": shared_stats,
            "rejected_fields": _rejected_fields(coverage, unique_values, numeric_counts, total, selected_fields),
            "reasons": [
                f"detected {log_type} log pattern",
                f"selected {len(entity_fields)} entity field(s), {len(categorical)} categorical field(s), {len(numeric)} numeric field(s), and {len(relationship_fields)} relationship(s)",
                f"found {len(shared_stats)} field(s) that also appear in other enabled streams",
                "ignored raw/internal, constant, sparse, and very high-cardinality fields",
            ],
        }
        existing = existing_profiles.get(stream_id)
        score = min(100, 30 + len(entity_fields) * 15 + min(20, len(categorical) * 3) + min(15, len(detectors) * 5))
        suggestions.append({
            "stream_id": stream_id,
            "stream_name": names.get(stream_id, stream_id),
            "events": total,
            "profile_exists": bool(existing),
            "confidence": "high" if score >= 75 and total >= 100 else "medium" if score >= 55 and total >= 20 else "low",
            "score": score,
            "entity_fields": entity_fields,
            "timestamp_field": timestamp,
            "categorical_fields": categorical,
            "numeric_fields": numeric,
            "relationship_fields": relationship_fields,
            "detectors": detectors,
            "common_fields": high_coverage,
            "shared_fields": shared_stats,
            "discovery": discovery,
            "detected_log_type": log_type,
            "profile": {
                "stream_id": stream_id,
                "name": f"{names.get(stream_id, stream_id)} recommended profile",
                "entity_field": entity_fields[0],
                "entity_fields": entity_fields,
                "timestamp_field": timestamp,
                "categorical_fields": categorical,
                "numeric_fields": numeric,
                "detectors": detectors,
                "field_weights": {},
                "relationship_fields": relationship_fields,
            },
            "reason": "selected common entity, time, categorical, and numeric fields from observed events; non-priority fields are included when coverage and cardinality look useful",
        })
    return sorted(suggestions, key=lambda item: (bool(item["profile_exists"]), -int(item["score"]), str(item["stream_name"])))
