Implemented the next roadmap step: direct Graylog MCP replay with temporary baselines.

src/fgai/cli.py

@@ -16,7 +16,9 @@ from .syslog_server import listen_udp_syslog
 from .monitor import monitor_loop
 from .threat_intel import enrich_ips, is_public_ip
 from .config import ConfigStore
-from .replay import replay_events
+from .graylog_mcp import GraylogMcpClient
+from .graylog_source import GraylogStreamSource
+from .replay import replay_comparison, replay_events
 from .stream_profiles import parse_profiles
 
 
@@ -218,11 +220,97 @@ def run_dashboard(args: argparse.Namespace) -> int:
     return 0
 
 
+def _stream_name(config: dict[str, object], stream_id: str) -> str:
+    return next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == stream_id), stream_id)
+
+
+def _profile_fields(profile: object | None) -> tuple[str, ...]:
+    if not profile:
+        return ()
+    return tuple(
+        field for field in (
+            str(getattr(profile, "entity_field", "")),
+            str(getattr(profile, "timestamp_field", "")),
+            *tuple(str(item) for item in getattr(profile, "categorical_fields", ())),
+            *tuple(str(item) for item in getattr(profile, "numeric_fields", ())),
+        )
+        if field
+    )
+
+
+def _configured_streams(config: dict[str, object], selected: list[str] | None = None) -> list[dict[str, str]]:
+    selected_order = list(dict.fromkeys(item for item in selected or [] if item))
+    selected_set = set(selected_order)
+    configured = [
+        {"id": str(item.get("id", "")), "title": str(item.get("title", ""))}
+        for item in config.get("graylog_streams", [])
+        if isinstance(item, dict) and item.get("id") and (item.get("enabled") or selected_set)
+    ]
+    if selected_set:
+        configured = [item for item in configured if item["id"] in selected_set]
+        by_id = {item["id"]: item for item in configured}
+        configured = [by_id.get(stream_id, {"id": stream_id, "title": _stream_name(config, stream_id)}) for stream_id in selected_order]
+    if not configured and str(config.get("graylog_stream", "")):
+        stream_id = str(config.get("graylog_stream", ""))
+        configured = [{"id": stream_id, "title": _stream_name(config, stream_id)}]
+    return configured
+
+
 def replay_history(args: argparse.Namespace) -> int:
     config = ConfigStore(args.config_file).read()
     profiles = parse_profiles(config.get("graylog_stream_profiles", []))
-    stream_name = next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == args.stream_id), args.stream_id)
-    result = replay_events(read_events(args.logs), profiles, stream_id=args.stream_id, stream_name=stream_name, bucket_seconds=args.bucket_seconds)
+    stream_name = _stream_name(config, args.stream_id)
+    events = read_events(args.logs)
+    result = replay_events(events, profiles, stream_id=args.stream_id, stream_name=stream_name, bucket_seconds=args.bucket_seconds)
+    if args.compare_config_file:
+        candidate_config = ConfigStore(args.compare_config_file).read()
+        candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", []))
+        candidate = replay_events(events, candidate_profiles, stream_id=args.stream_id, stream_name=_stream_name(candidate_config, args.stream_id), bucket_seconds=args.bucket_seconds)
+        result = {"current": result, "candidate": candidate, "comparison": replay_comparison(result, candidate), "candidate_config_file": args.compare_config_file}
+    _print_json(result)
+    return 0
+
+
+def replay_graylog(args: argparse.Namespace) -> int:
+    config = ConfigStore(args.config_file).read()
+    url, token = str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", ""))
+    if not url or not token:
+        raise RuntimeError("graylog_mcp_not_configured")
+    profiles = parse_profiles(config.get("graylog_stream_profiles", []))
+    candidate_profiles = {}
+    if args.compare_config_file:
+        candidate_config = ConfigStore(args.compare_config_file).read()
+        candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", []))
+    streams = _configured_streams(config, args.stream_id)
+    if not streams:
+        raise RuntimeError("no_graylog_streams_selected")
+    events = []
+    stream_statuses = []
+    for stream in streams:
+        stream_id = stream["id"]
+        profile_fields = tuple(dict.fromkeys([*_profile_fields(profiles.get(stream_id)), *_profile_fields(candidate_profiles.get(stream_id))]))
+        stream_events, status = GraylogStreamSource(
+            GraylogMcpClient(url, token),
+            stream_id,
+            str(config.get("graylog_query", "*")),
+            str(config.get("graylog_field_mapping", "")),
+            stream.get("title") or stream_id,
+            profile_fields,
+        ).fetch(max_events=args.max_events_per_stream, range_seconds=args.range_seconds)
+        events.extend(stream_events)
+        stream_statuses.append({"stream_id": stream_id, "stream_name": stream.get("title") or stream_id, **status})
+    result = {
+        "source": "graylog_mcp",
+        "range_seconds": args.range_seconds,
+        "streams": stream_statuses,
+        "replay": replay_events(events, profiles, bucket_seconds=args.bucket_seconds),
+    }
+    if args.compare_config_file:
+        candidate = replay_events(events, candidate_profiles, bucket_seconds=args.bucket_seconds)
+        current = result["replay"]
+        result["candidate_config_file"] = args.compare_config_file
+        result["candidate_replay"] = candidate
+        result["comparison"] = replay_comparison(current, candidate)
     _print_json(result)
     return 0
 
@@ -322,8 +410,18 @@ def build_parser() -> argparse.ArgumentParser:
     replay.add_argument("--config-file", default="state/fgai-config.json", help="Stream profile configuration")
     replay.add_argument("--stream-id", default="", help="Apply this configured Graylog stream profile to exported events")
     replay.add_argument("--bucket-seconds", type=int, default=300, help="Replay baseline bucket size")
+    replay.add_argument("--compare-config-file", default="", help="Replay the same export against another saved config and report detector deltas")
     replay.set_defaults(func=replay_history)
 
+    replay_mcp = subparsers.add_parser("replay-graylog", help="Replay a Graylog MCP time window against temporary baselines")
+    replay_mcp.add_argument("--config-file", default="state/fgai-config.json", help="Runtime configuration with Graylog MCP URL, token, streams and profiles")
+    replay_mcp.add_argument("--stream-id", action="append", default=[], help="Replay only this stream ID. Repeat for multiple streams. Defaults to enabled streams.")
+    replay_mcp.add_argument("--range-seconds", type=int, default=24 * 3600, help="Graylog relative time window to fetch")
+    replay_mcp.add_argument("--max-events-per-stream", type=int, default=5_000, help="Maximum events fetched from each selected stream")
+    replay_mcp.add_argument("--bucket-seconds", type=int, default=300, help="Replay baseline bucket size")
+    replay_mcp.add_argument("--compare-config-file", default="", help="Replay the same Graylog events against another saved config and report detector deltas")
+    replay_mcp.set_defaults(func=replay_graylog)
+
     return parser
 
 

src/fgai/graylog_source.py

@@ -54,12 +54,12 @@ class GraylogStreamSource:
         if not isinstance(self.mapping, dict):
             raise RuntimeError("invalid_graylog_field_mapping")
 
-    def fetch(self, *, max_events: int = 5_000) -> tuple[list[LogEvent], dict[str, object]]:
+    def fetch(self, *, max_events: int = 5_000, range_seconds: int = 300) -> tuple[list[LogEvent], dict[str, object]]:
         status = self.client.probe()
         mapping_fields = [str(value) for value in self.mapping.values() if isinstance(value, str)]
         arguments: dict[str, object] = {
             "query": self.query,
-            "range_seconds": 300,
+            "range_seconds": max(1, int(range_seconds)),
             "fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields, *self.profile_fields])),
         }
         if self.stream:

src/fgai/replay.py

@@ -10,6 +10,23 @@ from .baseline import BaselineStore
 from .models import LogEvent
 
 
+def replay_comparison(current: dict[str, object], candidate: dict[str, object]) -> dict[str, object]:
+    """Summarize how two replay outputs differ."""
+    current_counts = current.get("field_detector_counts", {})
+    candidate_counts = candidate.get("field_detector_counts", {})
+    detectors = sorted(set(current_counts if isinstance(current_counts, dict) else {}) | set(candidate_counts if isinstance(candidate_counts, dict) else {}))
+    detector_deltas = {
+        detector: int((candidate_counts if isinstance(candidate_counts, dict) else {}).get(detector, 0)) - int((current_counts if isinstance(current_counts, dict) else {}).get(detector, 0))
+        for detector in detectors
+    }
+    return {
+        "events_delta": int(candidate.get("events", 0)) - int(current.get("events", 0)),
+        "field_findings_delta": len(candidate.get("field_findings", [])) - len(current.get("field_findings", [])),
+        "source_anomalies_delta": len(candidate.get("source_anomalies", [])) - len(current.get("source_anomalies", [])),
+        "detector_count_delta": detector_deltas,
+    }
+
+
 def _timestamp(event: LogEvent, fallback: int) -> int:
     value = event.fields.get("eventtime", event.fields.get("timestamp", ""))
     try: