fgAI/src/fgai/replay.py

from __future__ import annotations

import tempfile
from collections import Counter, defaultdict
from pathlib import Path

from .anomaly import anomaly_summary, detect_source_anomalies
from .baseline import BaselineStore
from .config import ConfigStore
from .field_shapes import parse_iso_timestamp
from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource
from .logs import read_events
from .models import LogEvent
from .stream_profiles import parse_profiles


def replay_comparison(current: dict[str, object], candidate: dict[str, object]) -> dict[str, object]:
    """Summarize how two replay outputs differ."""
    current_counts = current.get("field_detector_counts", {})
    candidate_counts = candidate.get("field_detector_counts", {})
    detectors = sorted(set(current_counts if isinstance(current_counts, dict) else {}) | set(candidate_counts if isinstance(candidate_counts, dict) else {}))
    detector_deltas = {
        detector: int((candidate_counts if isinstance(candidate_counts, dict) else {}).get(detector, 0)) - int((current_counts if isinstance(current_counts, dict) else {}).get(detector, 0))
        for detector in detectors
    }
    return {
        "events_delta": int(candidate.get("events", 0)) - int(current.get("events", 0)),
        "field_findings_delta": len(candidate.get("field_findings", [])) - len(current.get("field_findings", [])),
        "source_anomalies_delta": len(candidate.get("source_anomalies", [])) - len(current.get("source_anomalies", [])),
        "detector_count_delta": detector_deltas,
    }


def _timestamp(event: LogEvent, fallback: int) -> int:
    value = event.fields.get("eventtime", event.fields.get("timestamp", ""))
    try:
        return int(float(value))
    except (TypeError, ValueError):
        parsed = parse_iso_timestamp(value)
        return int(parsed) if parsed is not None else fallback


def _with_stream(events: list[LogEvent], stream_id: str, stream_name: str) -> list[LogEvent]:
    if not stream_id:
        return events
    return [LogEvent(event.raw, {**event.fields, "fgai_stream_id": stream_id, "fgai_stream": stream_name or stream_id}) for event in events]


def replay_events(events: list[LogEvent], profiles: dict[str, object], *, stream_id: str = "", stream_name: str = "", bucket_seconds: int = 300) -> dict[str, object]:
    """Evaluate historical events using a temporary baseline without touching runtime state."""
    normalized = _with_stream(events, stream_id, stream_name)
    buckets: dict[int, list[LogEvent]] = defaultdict(list)
    for index, event in enumerate(normalized):
        timestamp = _timestamp(event, index * bucket_seconds)
        buckets[timestamp - (timestamp % bucket_seconds)].append(event)

    field_findings: list[dict[str, object]] = []
    source_findings = []
    with tempfile.TemporaryDirectory(prefix="signalscope-replay-") as directory:
        baseline = BaselineStore(str(Path(directory) / "baseline.sqlite3"), bucket_seconds=bucket_seconds)
        for bucket, batch in sorted(buckets.items()):
            profiles_by_source = baseline.profiles({event.src_ip for event in batch if event.src_ip})
            deviations = baseline.profile_deviations(batch, profiles)
            anomalies = detect_source_anomalies(batch, baselines=profiles_by_source, field_deviations=deviations)
            for entity, findings in deviations.items():
                for finding in findings:
                    field_findings.append({"bucket_start": bucket, "entity": entity, **finding})
            source_findings.extend(anomalies)
            baseline.ingest(batch, observed_at=bucket)
            baseline.ingest_profile_fields(batch, profiles, observed_at=bucket)

    detector_counts = Counter(str(item.get("detector", "unknown")) for item in field_findings)
    return {
        "events": len(normalized),
        "buckets": len(buckets),
        "field_findings": field_findings,
        "field_detector_counts": dict(sorted(detector_counts.items())),
        "source_anomaly_summary": anomaly_summary(source_findings),
        "source_anomalies": [
            {"subject": item.subject, "score": item.score, "severity": item.severity, "confidence": item.confidence, "reasons": item.reasons, "evidence": item.evidence}
            for item in source_findings
        ],
    }


def _stream_name(config: dict[str, object], stream_id: str) -> str:
    return next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == stream_id), stream_id)


def _profile_fields(profile: object | None) -> tuple[str, ...]:
    if not profile:
        return ()
    return tuple(
        field for field in (
            str(getattr(profile, "entity_field", "")),
            *tuple(str(item) for item in getattr(profile, "entity_fields", ())),
            str(getattr(profile, "timestamp_field", "")),
            *tuple(str(item) for item in getattr(profile, "categorical_fields", ())),
            *tuple(str(item) for item in getattr(profile, "numeric_fields", ())),
            *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())),
            *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())),
        )
        if field
    )


def _configured_streams(config: dict[str, object], selected: list[str] | None = None) -> list[dict[str, str]]:
    selected_order = list(dict.fromkeys(item for item in selected or [] if item))
    selected_set = set(selected_order)
    configured = [
        {"id": str(item.get("id", "")), "title": str(item.get("title", ""))}
        for item in config.get("graylog_streams", [])
        if isinstance(item, dict) and item.get("id") and (item.get("enabled") or selected_set)
    ]
    if selected_set:
        configured = [item for item in configured if item["id"] in selected_set]
        by_id = {item["id"]: item for item in configured}
        configured = [by_id.get(stream_id, {"id": stream_id, "title": _stream_name(config, stream_id)}) for stream_id in selected_order]
    if not configured and str(config.get("graylog_stream", "")):
        stream_id = str(config.get("graylog_stream", ""))
        configured = [{"id": stream_id, "title": _stream_name(config, stream_id)}]
    return configured


def run_local_replay(config_path: str, logs_path: str, *, stream_id: str = "", bucket_seconds: int = 300, compare_config_file: str = "") -> dict[str, object]:
    """Replay a local JSONL/key-value log export against a temporary baseline.

    Safe to call from the dashboard: replay_events() never touches the live
    baseline or monitor state, so this can run concurrently with a live poll.
    """
    config = ConfigStore(config_path).read()
    profiles = parse_profiles(config.get("graylog_stream_profiles", []))
    stream_name = _stream_name(config, stream_id)
    events = read_events(logs_path)
    result = replay_events(events, profiles, stream_id=stream_id, stream_name=stream_name, bucket_seconds=bucket_seconds)
    if compare_config_file:
        candidate_config = ConfigStore(compare_config_file).read()
        candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", []))
        candidate = replay_events(events, candidate_profiles, stream_id=stream_id, stream_name=_stream_name(candidate_config, stream_id), bucket_seconds=bucket_seconds)
        result = {"current": result, "candidate": candidate, "comparison": replay_comparison(result, candidate), "candidate_config_file": compare_config_file}
    return result


def run_graylog_replay(config_path: str, *, stream_ids: list[str] | None = None, range_seconds: int = 24 * 3600, max_events_per_stream: int = 5_000, bucket_seconds: int = 300, compare_config_file: str = "") -> dict[str, object]:
    """Replay a Graylog MCP time window against a temporary baseline.

    Safe to call from the dashboard for the same reason as run_local_replay —
    a temporary baseline only, and this only reads from Graylog MCP.
    """
    config = ConfigStore(config_path).read()
    url, token = str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", ""))
    if not url or not token:
        raise RuntimeError("graylog_mcp_not_configured")
    profiles = parse_profiles(config.get("graylog_stream_profiles", []))
    candidate_profiles = {}
    if compare_config_file:
        candidate_config = ConfigStore(compare_config_file).read()
        candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", []))
    streams = _configured_streams(config, stream_ids)
    if not streams:
        raise RuntimeError("no_graylog_streams_selected")
    events: list[LogEvent] = []
    stream_statuses = []
    for stream in streams:
        stream_id = stream["id"]
        profile_fields = tuple(dict.fromkeys([*_profile_fields(profiles.get(stream_id)), *_profile_fields(candidate_profiles.get(stream_id))]))
        stream_events, status = GraylogStreamSource(
            GraylogMcpClient(url, token),
            stream_id,
            str(config.get("graylog_query", "*")),
            str(config.get("graylog_field_mapping", "")),
            stream.get("title") or stream_id,
            profile_fields,
        ).fetch(max_events=max_events_per_stream, range_seconds=range_seconds)
        events.extend(stream_events)
        stream_statuses.append({"stream_id": stream_id, "stream_name": stream.get("title") or stream_id, **status})
    result = {
        "source": "graylog_mcp",
        "range_seconds": range_seconds,
        "streams": stream_statuses,
        "replay": replay_events(events, profiles, bucket_seconds=bucket_seconds),
    }
    if compare_config_file:
        candidate = replay_events(events, candidate_profiles, bucket_seconds=bucket_seconds)
        current = result["replay"]
        result["candidate_config_file"] = compare_config_file
        result["candidate_replay"] = candidate
        result["comparison"] = replay_comparison(current, candidate)
    return result