3e01ac977d
Closes the last ROADMAP §1 item. The replay/replay-graylog CLI backend already existed and was already safe (a temporary, auto-cleaned SQLite baseline, never touching live state) — the gap was purely that a human had to SSH in and run it by hand. Extracts the core logic from cli.py into run_local_replay()/run_graylog_replay() in replay.py, reused by both the CLI (now thin wrappers) and a new dashboard panel (Diagnostics -> Replay & Detector Deltas). Runs as a background job via an in-memory job store, since the dashboard already runs ThreadingHTTPServer and this session directly observed live MCP polling take anywhere from 9 minutes to over an hour — a synchronous request/response was never viable. Deliberately serialized to one job at a time given this box's GPU/CPU constraints. Also removed a redundant "Detector Precision" panel that duplicated (with a weaker, current-poll-cycle-only computation) what the new Detector Health panel already does correctly with real historical data. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
119 lines
6.1 KiB
Python
119 lines
6.1 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from fgai.logs import parse_log_line
|
|
from fgai.replay import _configured_streams, replay_comparison, replay_events, run_graylog_replay, run_local_replay
|
|
from fgai.stream_profiles import parse_profiles
|
|
|
|
|
|
class ReplayTests(unittest.TestCase):
|
|
def test_replay_uses_temporary_profile_baseline(self):
|
|
profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}])
|
|
events = [
|
|
parse_log_line(f"timestamp={1_700_000_000 + index * 300} username=alice action=failed event={index}")
|
|
for index in range(12)
|
|
]
|
|
events.extend(
|
|
parse_log_line(f"timestamp={1_700_003_600} username=alice action=failed burst={index}")
|
|
for index in range(5)
|
|
)
|
|
result = replay_events(events, profiles, stream_id="windows")
|
|
self.assertEqual(result["events"], 17)
|
|
self.assertGreaterEqual(result["field_detector_counts"].get("auth_failure_burst", 0), 1)
|
|
|
|
def test_replay_comparison_reports_detector_deltas(self):
|
|
result = replay_comparison(
|
|
{"events": 10, "field_findings": [{"a": 1}], "source_anomalies": [], "field_detector_counts": {"auth_failure_burst": 2}},
|
|
{"events": 10, "field_findings": [{"a": 1}, {"a": 2}], "source_anomalies": [{"a": 1}], "field_detector_counts": {"auth_failure_burst": 1, "rare_value": 3}},
|
|
)
|
|
self.assertEqual(result["field_findings_delta"], 1)
|
|
self.assertEqual(result["source_anomalies_delta"], 1)
|
|
self.assertEqual(result["detector_count_delta"]["auth_failure_burst"], -1)
|
|
self.assertEqual(result["detector_count_delta"]["rare_value"], 3)
|
|
|
|
def test_configured_streams_uses_enabled_streams_by_default(self):
|
|
config = {
|
|
"graylog_streams": [
|
|
{"id": "fortigate", "title": "Fortigate", "enabled": True},
|
|
{"id": "adguard", "title": "Adguard", "enabled": False},
|
|
]
|
|
}
|
|
self.assertEqual(_configured_streams(config), [{"id": "fortigate", "title": "Fortigate"}])
|
|
|
|
def test_configured_streams_can_select_disabled_stream(self):
|
|
config = {
|
|
"graylog_streams": [
|
|
{"id": "fortigate", "title": "Fortigate", "enabled": True},
|
|
{"id": "adguard", "title": "Adguard", "enabled": False},
|
|
]
|
|
}
|
|
self.assertEqual(_configured_streams(config, ["adguard"]), [{"id": "adguard", "title": "Adguard"}])
|
|
|
|
def test_configured_streams_preserves_selected_order(self):
|
|
config = {
|
|
"graylog_streams": [
|
|
{"id": "fortigate", "title": "Fortigate", "enabled": True},
|
|
{"id": "adguard", "title": "Adguard", "enabled": False},
|
|
]
|
|
}
|
|
self.assertEqual(
|
|
_configured_streams(config, ["adguard", "fortigate"]),
|
|
[{"id": "adguard", "title": "Adguard"}, {"id": "fortigate", "title": "Fortigate"}],
|
|
)
|
|
|
|
def test_run_local_replay_matches_replay_events_output(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config_path = Path(directory) / "config.json"
|
|
config_path.write_text(json.dumps({"graylog_stream_profiles": [{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}]}), encoding="utf-8")
|
|
logs_path = Path(directory) / "events.log"
|
|
lines = [
|
|
f"timestamp={1_700_000_000 + index * 300} username=alice action=failed event={index}"
|
|
for index in range(12)
|
|
]
|
|
lines.extend(f"timestamp={1_700_003_600} username=alice action=failed burst={index}" for index in range(5))
|
|
logs_path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
result = run_local_replay(str(config_path), str(logs_path), stream_id="windows")
|
|
|
|
self.assertEqual(result["events"], 17)
|
|
self.assertGreaterEqual(result["field_detector_counts"].get("auth_failure_burst", 0), 1)
|
|
|
|
def test_run_local_replay_with_compare_config_reports_comparison(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config_path = Path(directory) / "config.json"
|
|
config_path.write_text(json.dumps({"graylog_stream_profiles": [{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}]}), encoding="utf-8")
|
|
candidate_path = Path(directory) / "candidate.json"
|
|
candidate_path.write_text(json.dumps({"graylog_stream_profiles": [{"stream_id": "windows", "entity_field": "username", "categorical_fields": []}]}), encoding="utf-8")
|
|
logs_path = Path(directory) / "events.log"
|
|
lines = [
|
|
f"timestamp={1_700_000_000 + index * 300} username=alice action=failed event={index}"
|
|
for index in range(12)
|
|
]
|
|
lines.extend(f"timestamp={1_700_003_600} username=alice action=failed burst={index}" for index in range(5))
|
|
logs_path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
result = run_local_replay(str(config_path), str(logs_path), stream_id="windows", compare_config_file=str(candidate_path))
|
|
|
|
self.assertIn("comparison", result)
|
|
self.assertIn("current", result)
|
|
self.assertIn("candidate", result)
|
|
self.assertEqual(result["candidate_config_file"], str(candidate_path))
|
|
|
|
def test_run_graylog_replay_requires_mcp_configuration(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config_path = Path(directory) / "config.json"
|
|
config_path.write_text(json.dumps({}), encoding="utf-8")
|
|
|
|
with self.assertRaises(RuntimeError):
|
|
run_graylog_replay(str(config_path))
|
|
|
|
def test_run_graylog_replay_requires_selected_streams(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
config_path = Path(directory) / "config.json"
|
|
config_path.write_text(json.dumps({"graylog_mcp_url": "https://graylog.example", "graylog_mcp_token": "token"}), encoding="utf-8")
|
|
|
|
with self.assertRaises(RuntimeError):
|
|
run_graylog_replay(str(config_path))
|