From b6560d64f7b81f4071a382b6e909dfb3bdc4cfb9cfd5efe139326c0809e7d238 Mon Sep 17 00:00:00 2001 From: larssand Date: Thu, 25 Jun 2026 18:34:51 +0200 Subject: [PATCH] Implemented the next roadmap step: direct Graylog MCP replay with temporary baselines. --- README.md | 18 ++++++ ROADMAP.md | 3 +- src/fgai/cli.py | 104 ++++++++++++++++++++++++++++++++++- src/fgai/graylog_source.py | 4 +- src/fgai/replay.py | 17 ++++++ tests/test_cli.py | 39 +++++++++++++ tests/test_graylog_source.py | 5 ++ tests/test_replay.py | 12 +++- 8 files changed, 195 insertions(+), 7 deletions(-) create mode 100644 tests/test_cli.py diff --git a/README.md b/README.md index 3801f27..0ce0879 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,24 @@ Replay uses a temporary SQLite baseline and evaluates events in timestamp order. It reports detector counts and the findings that would have been generated. Use the configured stream ID so the export is evaluated with that stream's profile. +Replay directly from Graylog MCP without touching the live baseline: + +```bash +signalscope replay-graylog --range-seconds 86400 +``` + +By default this uses the enabled streams from the dashboard configuration. Limit +the run to one or more streams with repeated `--stream-id` flags. To test a +candidate detector/profile configuration before applying it, compare it against +the current runtime config: + +```bash +signalscope replay-graylog --range-seconds 86400 --compare-config-file exports/candidate-config.json +``` + +The comparison reports detector-count, field-finding, and source-anomaly deltas +using the same fetched event window. + The current MCP endpoint is `http://:9000/api/mcp`. Enable it in Graylog under `System -> Configurations -> MCP` and use stream IDs internally; the fgAI stream picker resolves titles in the UI. diff --git a/ROADMAP.md b/ROADMAP.md index bada1b3..f4cdb31 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -30,7 +30,8 @@ Goal: make findings more accurate before adding more integrations. - [ ] Add sequence detection, for example DNS lookup -> outbound connection -> authentication event. - [x] Add per-stream detector enablement and thresholds in the UI. - [x] Add a dry-run replay command for historic JSONL or Graylog exports using temporary baselines. -- [ ] Add direct Graylog MCP time-range replay and result comparison against saved detector configurations. +- [x] Add direct Graylog MCP time-range replay and result comparison against saved detector configurations. +- [ ] Add dashboard controls for launching safe replay jobs and viewing detector deltas. Acceptance: each finding shows its detector, confidence, baseline sample count, current value, expected value, and a bounded set of raw-event references. diff --git a/src/fgai/cli.py b/src/fgai/cli.py index cdaaf19..87d1253 100644 --- a/src/fgai/cli.py +++ b/src/fgai/cli.py @@ -16,7 +16,9 @@ from .syslog_server import listen_udp_syslog from .monitor import monitor_loop from .threat_intel import enrich_ips, is_public_ip from .config import ConfigStore -from .replay import replay_events +from .graylog_mcp import GraylogMcpClient +from .graylog_source import GraylogStreamSource +from .replay import replay_comparison, replay_events from .stream_profiles import parse_profiles @@ -218,11 +220,97 @@ def run_dashboard(args: argparse.Namespace) -> int: return 0 +def _stream_name(config: dict[str, object], stream_id: str) -> str: + return next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == stream_id), stream_id) + + +def _profile_fields(profile: object | None) -> tuple[str, ...]: + if not profile: + return () + return tuple( + field for field in ( + str(getattr(profile, "entity_field", "")), + str(getattr(profile, "timestamp_field", "")), + *tuple(str(item) for item in getattr(profile, "categorical_fields", ())), + *tuple(str(item) for item in getattr(profile, "numeric_fields", ())), + ) + if field + ) + + +def _configured_streams(config: dict[str, object], selected: list[str] | None = None) -> list[dict[str, str]]: + selected_order = list(dict.fromkeys(item for item in selected or [] if item)) + selected_set = set(selected_order) + configured = [ + {"id": str(item.get("id", "")), "title": str(item.get("title", ""))} + for item in config.get("graylog_streams", []) + if isinstance(item, dict) and item.get("id") and (item.get("enabled") or selected_set) + ] + if selected_set: + configured = [item for item in configured if item["id"] in selected_set] + by_id = {item["id"]: item for item in configured} + configured = [by_id.get(stream_id, {"id": stream_id, "title": _stream_name(config, stream_id)}) for stream_id in selected_order] + if not configured and str(config.get("graylog_stream", "")): + stream_id = str(config.get("graylog_stream", "")) + configured = [{"id": stream_id, "title": _stream_name(config, stream_id)}] + return configured + + def replay_history(args: argparse.Namespace) -> int: config = ConfigStore(args.config_file).read() profiles = parse_profiles(config.get("graylog_stream_profiles", [])) - stream_name = next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == args.stream_id), args.stream_id) - result = replay_events(read_events(args.logs), profiles, stream_id=args.stream_id, stream_name=stream_name, bucket_seconds=args.bucket_seconds) + stream_name = _stream_name(config, args.stream_id) + events = read_events(args.logs) + result = replay_events(events, profiles, stream_id=args.stream_id, stream_name=stream_name, bucket_seconds=args.bucket_seconds) + if args.compare_config_file: + candidate_config = ConfigStore(args.compare_config_file).read() + candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", [])) + candidate = replay_events(events, candidate_profiles, stream_id=args.stream_id, stream_name=_stream_name(candidate_config, args.stream_id), bucket_seconds=args.bucket_seconds) + result = {"current": result, "candidate": candidate, "comparison": replay_comparison(result, candidate), "candidate_config_file": args.compare_config_file} + _print_json(result) + return 0 + + +def replay_graylog(args: argparse.Namespace) -> int: + config = ConfigStore(args.config_file).read() + url, token = str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")) + if not url or not token: + raise RuntimeError("graylog_mcp_not_configured") + profiles = parse_profiles(config.get("graylog_stream_profiles", [])) + candidate_profiles = {} + if args.compare_config_file: + candidate_config = ConfigStore(args.compare_config_file).read() + candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", [])) + streams = _configured_streams(config, args.stream_id) + if not streams: + raise RuntimeError("no_graylog_streams_selected") + events = [] + stream_statuses = [] + for stream in streams: + stream_id = stream["id"] + profile_fields = tuple(dict.fromkeys([*_profile_fields(profiles.get(stream_id)), *_profile_fields(candidate_profiles.get(stream_id))])) + stream_events, status = GraylogStreamSource( + GraylogMcpClient(url, token), + stream_id, + str(config.get("graylog_query", "*")), + str(config.get("graylog_field_mapping", "")), + stream.get("title") or stream_id, + profile_fields, + ).fetch(max_events=args.max_events_per_stream, range_seconds=args.range_seconds) + events.extend(stream_events) + stream_statuses.append({"stream_id": stream_id, "stream_name": stream.get("title") or stream_id, **status}) + result = { + "source": "graylog_mcp", + "range_seconds": args.range_seconds, + "streams": stream_statuses, + "replay": replay_events(events, profiles, bucket_seconds=args.bucket_seconds), + } + if args.compare_config_file: + candidate = replay_events(events, candidate_profiles, bucket_seconds=args.bucket_seconds) + current = result["replay"] + result["candidate_config_file"] = args.compare_config_file + result["candidate_replay"] = candidate + result["comparison"] = replay_comparison(current, candidate) _print_json(result) return 0 @@ -322,8 +410,18 @@ def build_parser() -> argparse.ArgumentParser: replay.add_argument("--config-file", default="state/fgai-config.json", help="Stream profile configuration") replay.add_argument("--stream-id", default="", help="Apply this configured Graylog stream profile to exported events") replay.add_argument("--bucket-seconds", type=int, default=300, help="Replay baseline bucket size") + replay.add_argument("--compare-config-file", default="", help="Replay the same export against another saved config and report detector deltas") replay.set_defaults(func=replay_history) + replay_mcp = subparsers.add_parser("replay-graylog", help="Replay a Graylog MCP time window against temporary baselines") + replay_mcp.add_argument("--config-file", default="state/fgai-config.json", help="Runtime configuration with Graylog MCP URL, token, streams and profiles") + replay_mcp.add_argument("--stream-id", action="append", default=[], help="Replay only this stream ID. Repeat for multiple streams. Defaults to enabled streams.") + replay_mcp.add_argument("--range-seconds", type=int, default=24 * 3600, help="Graylog relative time window to fetch") + replay_mcp.add_argument("--max-events-per-stream", type=int, default=5_000, help="Maximum events fetched from each selected stream") + replay_mcp.add_argument("--bucket-seconds", type=int, default=300, help="Replay baseline bucket size") + replay_mcp.add_argument("--compare-config-file", default="", help="Replay the same Graylog events against another saved config and report detector deltas") + replay_mcp.set_defaults(func=replay_graylog) + return parser diff --git a/src/fgai/graylog_source.py b/src/fgai/graylog_source.py index 5e4c414..12f6f41 100644 --- a/src/fgai/graylog_source.py +++ b/src/fgai/graylog_source.py @@ -54,12 +54,12 @@ class GraylogStreamSource: if not isinstance(self.mapping, dict): raise RuntimeError("invalid_graylog_field_mapping") - def fetch(self, *, max_events: int = 5_000) -> tuple[list[LogEvent], dict[str, object]]: + def fetch(self, *, max_events: int = 5_000, range_seconds: int = 300) -> tuple[list[LogEvent], dict[str, object]]: status = self.client.probe() mapping_fields = [str(value) for value in self.mapping.values() if isinstance(value, str)] arguments: dict[str, object] = { "query": self.query, - "range_seconds": 300, + "range_seconds": max(1, int(range_seconds)), "fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields, *self.profile_fields])), } if self.stream: diff --git a/src/fgai/replay.py b/src/fgai/replay.py index 119c701..fac9f55 100644 --- a/src/fgai/replay.py +++ b/src/fgai/replay.py @@ -10,6 +10,23 @@ from .baseline import BaselineStore from .models import LogEvent +def replay_comparison(current: dict[str, object], candidate: dict[str, object]) -> dict[str, object]: + """Summarize how two replay outputs differ.""" + current_counts = current.get("field_detector_counts", {}) + candidate_counts = candidate.get("field_detector_counts", {}) + detectors = sorted(set(current_counts if isinstance(current_counts, dict) else {}) | set(candidate_counts if isinstance(candidate_counts, dict) else {})) + detector_deltas = { + detector: int((candidate_counts if isinstance(candidate_counts, dict) else {}).get(detector, 0)) - int((current_counts if isinstance(current_counts, dict) else {}).get(detector, 0)) + for detector in detectors + } + return { + "events_delta": int(candidate.get("events", 0)) - int(current.get("events", 0)), + "field_findings_delta": len(candidate.get("field_findings", [])) - len(current.get("field_findings", [])), + "source_anomalies_delta": len(candidate.get("source_anomalies", [])) - len(current.get("source_anomalies", [])), + "detector_count_delta": detector_deltas, + } + + def _timestamp(event: LogEvent, fallback: int) -> int: value = event.fields.get("eventtime", event.fields.get("timestamp", "")) try: diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ab663ef --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,39 @@ +import unittest + +from fgai.cli import _configured_streams + + +class CliTests(unittest.TestCase): + def test_configured_streams_uses_enabled_streams_by_default(self): + config = { + "graylog_streams": [ + {"id": "fortigate", "title": "Fortigate", "enabled": True}, + {"id": "adguard", "title": "Adguard", "enabled": False}, + ] + } + self.assertEqual(_configured_streams(config), [{"id": "fortigate", "title": "Fortigate"}]) + + def test_configured_streams_can_select_disabled_stream(self): + config = { + "graylog_streams": [ + {"id": "fortigate", "title": "Fortigate", "enabled": True}, + {"id": "adguard", "title": "Adguard", "enabled": False}, + ] + } + self.assertEqual(_configured_streams(config, ["adguard"]), [{"id": "adguard", "title": "Adguard"}]) + + def test_configured_streams_preserves_selected_order(self): + config = { + "graylog_streams": [ + {"id": "fortigate", "title": "Fortigate", "enabled": True}, + {"id": "adguard", "title": "Adguard", "enabled": False}, + ] + } + self.assertEqual( + _configured_streams(config, ["adguard", "fortigate"]), + [{"id": "adguard", "title": "Adguard"}, {"id": "fortigate", "title": "Fortigate"}], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_graylog_source.py b/tests/test_graylog_source.py index 660ed63..35bf31e 100644 --- a/tests/test_graylog_source.py +++ b/tests/test_graylog_source.py @@ -30,6 +30,11 @@ class GraylogSourceTests(unittest.TestCase): self.assertIn("client", client.arguments["fields"]) self.assertEqual(client.arguments["offset"], 0) + def test_uses_requested_range_seconds(self): + client = _Client() + GraylogStreamSource(client, "vpn").fetch(range_seconds=86_400) + self.assertEqual(client.arguments["range_seconds"], 86_400) + def test_requests_selected_profile_fields(self): client = _Client() GraylogStreamSource(client, "windows", profile_fields=("TargetUserName", "EventID")).fetch() diff --git a/tests/test_replay.py b/tests/test_replay.py index 9511c28..5c0401e 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -1,7 +1,7 @@ import unittest from fgai.logs import parse_log_line -from fgai.replay import replay_events +from fgai.replay import replay_comparison, replay_events from fgai.stream_profiles import parse_profiles @@ -19,3 +19,13 @@ class ReplayTests(unittest.TestCase): result = replay_events(events, profiles, stream_id="windows") self.assertEqual(result["events"], 17) self.assertGreaterEqual(result["field_detector_counts"].get("auth_failure_burst", 0), 1) + + def test_replay_comparison_reports_detector_deltas(self): + result = replay_comparison( + {"events": 10, "field_findings": [{"a": 1}], "source_anomalies": [], "field_detector_counts": {"auth_failure_burst": 2}}, + {"events": 10, "field_findings": [{"a": 1}, {"a": 2}], "source_anomalies": [{"a": 1}], "field_detector_counts": {"auth_failure_burst": 1, "rare_value": 3}}, + ) + self.assertEqual(result["field_findings_delta"], 1) + self.assertEqual(result["source_anomalies_delta"], 1) + self.assertEqual(result["detector_count_delta"]["auth_failure_burst"], -1) + self.assertEqual(result["detector_count_delta"]["rare_value"], 3)