from __future__ import annotations import tempfile from collections import Counter, defaultdict from pathlib import Path from .anomaly import anomaly_summary, detect_source_anomalies from .baseline import BaselineStore from .config import ConfigStore from .field_shapes import parse_iso_timestamp from .graylog_mcp import GraylogMcpClient from .graylog_source import GraylogStreamSource from .logs import read_events from .models import LogEvent from .stream_profiles import parse_profiles def replay_comparison(current: dict[str, object], candidate: dict[str, object]) -> dict[str, object]: """Summarize how two replay outputs differ.""" current_counts = current.get("field_detector_counts", {}) candidate_counts = candidate.get("field_detector_counts", {}) detectors = sorted(set(current_counts if isinstance(current_counts, dict) else {}) | set(candidate_counts if isinstance(candidate_counts, dict) else {})) detector_deltas = { detector: int((candidate_counts if isinstance(candidate_counts, dict) else {}).get(detector, 0)) - int((current_counts if isinstance(current_counts, dict) else {}).get(detector, 0)) for detector in detectors } return { "events_delta": int(candidate.get("events", 0)) - int(current.get("events", 0)), "field_findings_delta": len(candidate.get("field_findings", [])) - len(current.get("field_findings", [])), "source_anomalies_delta": len(candidate.get("source_anomalies", [])) - len(current.get("source_anomalies", [])), "detector_count_delta": detector_deltas, } def _timestamp(event: LogEvent, fallback: int) -> int: value = event.fields.get("eventtime", event.fields.get("timestamp", "")) try: return int(float(value)) except (TypeError, ValueError): parsed = parse_iso_timestamp(value) return int(parsed) if parsed is not None else fallback def _with_stream(events: list[LogEvent], stream_id: str, stream_name: str) -> list[LogEvent]: if not stream_id: return events return [LogEvent(event.raw, {**event.fields, "fgai_stream_id": stream_id, "fgai_stream": stream_name or stream_id}) for event in events] def replay_events(events: list[LogEvent], profiles: dict[str, object], *, stream_id: str = "", stream_name: str = "", bucket_seconds: int = 300) -> dict[str, object]: """Evaluate historical events using a temporary baseline without touching runtime state.""" normalized = _with_stream(events, stream_id, stream_name) buckets: dict[int, list[LogEvent]] = defaultdict(list) for index, event in enumerate(normalized): timestamp = _timestamp(event, index * bucket_seconds) buckets[timestamp - (timestamp % bucket_seconds)].append(event) field_findings: list[dict[str, object]] = [] source_findings = [] with tempfile.TemporaryDirectory(prefix="signalscope-replay-") as directory: baseline = BaselineStore(str(Path(directory) / "baseline.sqlite3"), bucket_seconds=bucket_seconds) for bucket, batch in sorted(buckets.items()): profiles_by_source = baseline.profiles({event.src_ip for event in batch if event.src_ip}) deviations = baseline.profile_deviations(batch, profiles) anomalies = detect_source_anomalies(batch, baselines=profiles_by_source, field_deviations=deviations) for entity, findings in deviations.items(): for finding in findings: field_findings.append({"bucket_start": bucket, "entity": entity, **finding}) source_findings.extend(anomalies) baseline.ingest(batch, observed_at=bucket) baseline.ingest_profile_fields(batch, profiles, observed_at=bucket) detector_counts = Counter(str(item.get("detector", "unknown")) for item in field_findings) return { "events": len(normalized), "buckets": len(buckets), "field_findings": field_findings, "field_detector_counts": dict(sorted(detector_counts.items())), "source_anomaly_summary": anomaly_summary(source_findings), "source_anomalies": [ {"subject": item.subject, "score": item.score, "severity": item.severity, "confidence": item.confidence, "reasons": item.reasons, "evidence": item.evidence} for item in source_findings ], } def _stream_name(config: dict[str, object], stream_id: str) -> str: return next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == stream_id), stream_id) def _profile_fields(profile: object | None) -> tuple[str, ...]: if not profile: return () return tuple( field for field in ( str(getattr(profile, "entity_field", "")), *tuple(str(item) for item in getattr(profile, "entity_fields", ())), str(getattr(profile, "timestamp_field", "")), *tuple(str(item) for item in getattr(profile, "categorical_fields", ())), *tuple(str(item) for item in getattr(profile, "numeric_fields", ())), *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())), *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())), ) if field ) def _configured_streams(config: dict[str, object], selected: list[str] | None = None) -> list[dict[str, str]]: selected_order = list(dict.fromkeys(item for item in selected or [] if item)) selected_set = set(selected_order) configured = [ {"id": str(item.get("id", "")), "title": str(item.get("title", ""))} for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") and (item.get("enabled") or selected_set) ] if selected_set: configured = [item for item in configured if item["id"] in selected_set] by_id = {item["id"]: item for item in configured} configured = [by_id.get(stream_id, {"id": stream_id, "title": _stream_name(config, stream_id)}) for stream_id in selected_order] if not configured and str(config.get("graylog_stream", "")): stream_id = str(config.get("graylog_stream", "")) configured = [{"id": stream_id, "title": _stream_name(config, stream_id)}] return configured def run_local_replay(config_path: str, logs_path: str, *, stream_id: str = "", bucket_seconds: int = 300, compare_config_file: str = "") -> dict[str, object]: """Replay a local JSONL/key-value log export against a temporary baseline. Safe to call from the dashboard: replay_events() never touches the live baseline or monitor state, so this can run concurrently with a live poll. """ config = ConfigStore(config_path).read() profiles = parse_profiles(config.get("graylog_stream_profiles", [])) stream_name = _stream_name(config, stream_id) events = read_events(logs_path) result = replay_events(events, profiles, stream_id=stream_id, stream_name=stream_name, bucket_seconds=bucket_seconds) if compare_config_file: candidate_config = ConfigStore(compare_config_file).read() candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", [])) candidate = replay_events(events, candidate_profiles, stream_id=stream_id, stream_name=_stream_name(candidate_config, stream_id), bucket_seconds=bucket_seconds) result = {"current": result, "candidate": candidate, "comparison": replay_comparison(result, candidate), "candidate_config_file": compare_config_file} return result def run_graylog_replay(config_path: str, *, stream_ids: list[str] | None = None, range_seconds: int = 24 * 3600, max_events_per_stream: int = 5_000, bucket_seconds: int = 300, compare_config_file: str = "") -> dict[str, object]: """Replay a Graylog MCP time window against a temporary baseline. Safe to call from the dashboard for the same reason as run_local_replay — a temporary baseline only, and this only reads from Graylog MCP. """ config = ConfigStore(config_path).read() url, token = str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")) if not url or not token: raise RuntimeError("graylog_mcp_not_configured") profiles = parse_profiles(config.get("graylog_stream_profiles", [])) candidate_profiles = {} if compare_config_file: candidate_config = ConfigStore(compare_config_file).read() candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", [])) streams = _configured_streams(config, stream_ids) if not streams: raise RuntimeError("no_graylog_streams_selected") events: list[LogEvent] = [] stream_statuses = [] for stream in streams: stream_id = stream["id"] profile_fields = tuple(dict.fromkeys([*_profile_fields(profiles.get(stream_id)), *_profile_fields(candidate_profiles.get(stream_id))])) stream_events, status = GraylogStreamSource( GraylogMcpClient(url, token), stream_id, str(config.get("graylog_query", "*")), str(config.get("graylog_field_mapping", "")), stream.get("title") or stream_id, profile_fields, ).fetch(max_events=max_events_per_stream, range_seconds=range_seconds) events.extend(stream_events) stream_statuses.append({"stream_id": stream_id, "stream_name": stream.get("title") or stream_id, **status}) result = { "source": "graylog_mcp", "range_seconds": range_seconds, "streams": stream_statuses, "replay": replay_events(events, profiles, bucket_seconds=bucket_seconds), } if compare_config_file: candidate = replay_events(events, candidate_profiles, bucket_seconds=bucket_seconds) current = result["replay"] result["candidate_config_file"] = compare_config_file result["candidate_replay"] = candidate result["comparison"] = replay_comparison(current, candidate) return result