Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dashboard-triggered replay jobs and detector deltas

Browse Source 
Closes the last ROADMAP §1 item. The replay/replay-graylog CLI backend
already existed and was already safe (a temporary, auto-cleaned SQLite
baseline, never touching live state) — the gap was purely that a human had
to SSH in and run it by hand. Extracts the core logic from cli.py into
run_local_replay()/run_graylog_replay() in replay.py, reused by both the
CLI (now thin wrappers) and a new dashboard panel (Diagnostics -> Replay &
Detector Deltas). Runs as a background job via an in-memory job store,
since the dashboard already runs ThreadingHTTPServer and this session
directly observed live MCP polling take anywhere from 9 minutes to over an
hour — a synchronous request/response was never viable. Deliberately
serialized to one job at a time given this box's GPU/CPU constraints.

Also removed a redundant "Detector Precision" panel that duplicated (with
a weaker, current-poll-cycle-only computation) what the new Detector
Health panel already does correctly with real historical data.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-24 18:39:54 +02:00
parent bc4d2f465f
commit 3e01ac977d
8 changed files with 474 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
tests/test_cli.py
View File

@@ -1,39 +0,0 @@
import unittest
from fgai.cli import _configured_streams
class CliTests(unittest.TestCase):
def test_configured_streams_uses_enabled_streams_by_default(self):
config = {
"graylog_streams": [
{"id": "fortigate", "title": "Fortigate", "enabled": True},
{"id": "adguard", "title": "Adguard", "enabled": False},
]
}
self.assertEqual(_configured_streams(config), [{"id": "fortigate", "title": "Fortigate"}])
def test_configured_streams_can_select_disabled_stream(self):
config = {
"graylog_streams": [
{"id": "fortigate", "title": "Fortigate", "enabled": True},
{"id": "adguard", "title": "Adguard", "enabled": False},
]
}
self.assertEqual(_configured_streams(config, ["adguard"]), [{"id": "adguard", "title": "Adguard"}])
def test_configured_streams_preserves_selected_order(self):
config = {
"graylog_streams": [
{"id": "fortigate", "title": "Fortigate", "enabled": True},
{"id": "adguard", "title": "Adguard", "enabled": False},
]
}
self.assertEqual(
_configured_streams(config, ["adguard", "fortigate"]),
[{"id": "adguard", "title": "Adguard"}, {"id": "fortigate", "title": "Fortigate"}],
)
if __name__ == "__main__":
unittest.main()

80
tests/test_dashboard.py
View File

@@ -1,5 +1,6 @@
import json
import inspect
import time
import unittest
from unittest.mock import patch
@@ -255,6 +256,85 @@ class DashboardTests(unittest.TestCase):
self.assertIn('name="auto_apply_profile_suggestions"', HTML)
self.assertIn("field_confidence_threshold: config.field_confidence_threshold ?? 60", HTML)
def test_dashboard_exposes_replay_controls(self):
self.assertIn("/api/replay/start", HTML)
self.assertIn("/api/replay/status", HTML)
self.assertIn("Replay", HTML)
class ReplayJobTests(unittest.TestCase):
def setUp(self):
dashboard._replay_jobs.clear()
def tearDown(self):
dashboard._replay_jobs.clear()
def test_run_replay_job_records_success(self):
with patch("fgai.dashboard.run_local_replay", return_value={"events": 5}):
dashboard._replay_jobs["job-1"] = {"status": "running", "started_at": 100}
dashboard._run_replay_job("job-1", {"mode": "local"})
job = dashboard._replay_jobs["job-1"]
self.assertEqual(job["status"], "done")
self.assertEqual(job["result"], {"events": 5})
self.assertIn("finished_at", job)
def test_run_replay_job_records_failure_without_crashing(self):
with patch("fgai.dashboard.run_local_replay", side_effect=RuntimeError("boom")):
dashboard._replay_jobs["job-1"] = {"status": "running", "started_at": 100}
dashboard._run_replay_job("job-1", {"mode": "local"})
job = dashboard._replay_jobs["job-1"]
self.assertEqual(job["status"], "error")
self.assertEqual(job["error"], "boom")
def test_run_replay_job_dispatches_graylog_mode(self):
with patch("fgai.dashboard.run_graylog_replay", return_value={"events": 9}) as mocked:
dashboard._replay_jobs["job-1"] = {"status": "running", "started_at": 100}
dashboard._run_replay_job("job-1", {"mode": "graylog", "stream_ids": ["a", "b"], "range_seconds": 600})
mocked.assert_called_once()
self.assertEqual(dashboard._replay_jobs["job-1"]["result"], {"events": 9})
def test_start_replay_job_rejects_a_second_concurrent_job(self):
dashboard._replay_jobs["existing"] = {"status": "running", "started_at": 100}
result = dashboard._start_replay_job({"mode": "local"})
self.assertIn("error", result)
self.assertEqual(len(dashboard._replay_jobs), 1)
def test_start_replay_job_starts_a_thread_when_idle(self):
with patch("fgai.dashboard.run_local_replay", return_value={"events": 1}):
result = dashboard._start_replay_job({"mode": "local"})
self.assertIn("job_id", result)
job_id = result["job_id"]
for _ in range(50):
if dashboard._replay_jobs.get(job_id, {}).get("status") != "running":
break
time.sleep(0.02)
self.assertEqual(dashboard._replay_jobs[job_id]["status"], "done")
def test_replay_job_status_returns_job_by_id(self):
dashboard._replay_jobs["job-1"] = {"status": "done", "started_at": 100, "result": {"events": 1}}
status = dashboard._replay_job_status("job-1")
self.assertEqual(status["status"], "done")
def test_replay_job_status_returns_latest_job_when_no_id_given(self):
dashboard._replay_jobs["older"] = {"status": "done", "started_at": 100}
dashboard._replay_jobs["newer"] = {"status": "running", "started_at": 200}
status = dashboard._replay_job_status()
self.assertEqual(status["job_id"], "newer")
def test_replay_job_status_returns_none_when_no_jobs_exist(self):
self.assertIsNone(dashboard._replay_job_status())
self.assertIsNone(dashboard._replay_job_status("missing"))
if __name__ == "__main__":
unittest.main()

89
tests/test_replay.py
View File

@@ -1,7 +1,10 @@
import json
import tempfile
import unittest
from pathlib import Path
from fgai.logs import parse_log_line
from fgai.replay import replay_comparison, replay_events
from fgai.replay import _configured_streams, replay_comparison, replay_events, run_graylog_replay, run_local_replay
from fgai.stream_profiles import parse_profiles
@@ -29,3 +32,87 @@ class ReplayTests(unittest.TestCase):
self.assertEqual(result["source_anomalies_delta"], 1)
self.assertEqual(result["detector_count_delta"]["auth_failure_burst"], -1)
self.assertEqual(result["detector_count_delta"]["rare_value"], 3)
def test_configured_streams_uses_enabled_streams_by_default(self):
config = {
"graylog_streams": [
{"id": "fortigate", "title": "Fortigate", "enabled": True},
{"id": "adguard", "title": "Adguard", "enabled": False},
]
}
self.assertEqual(_configured_streams(config), [{"id": "fortigate", "title": "Fortigate"}])
def test_configured_streams_can_select_disabled_stream(self):
config = {
"graylog_streams": [
{"id": "fortigate", "title": "Fortigate", "enabled": True},
{"id": "adguard", "title": "Adguard", "enabled": False},
]
}
self.assertEqual(_configured_streams(config, ["adguard"]), [{"id": "adguard", "title": "Adguard"}])
def test_configured_streams_preserves_selected_order(self):
config = {
"graylog_streams": [
{"id": "fortigate", "title": "Fortigate", "enabled": True},
{"id": "adguard", "title": "Adguard", "enabled": False},
]
}
self.assertEqual(
_configured_streams(config, ["adguard", "fortigate"]),
[{"id": "adguard", "title": "Adguard"}, {"id": "fortigate", "title": "Fortigate"}],
)
def test_run_local_replay_matches_replay_events_output(self):
with tempfile.TemporaryDirectory() as directory:
config_path = Path(directory) / "config.json"
config_path.write_text(json.dumps({"graylog_stream_profiles": [{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}]}), encoding="utf-8")
logs_path = Path(directory) / "events.log"
lines = [
f"timestamp={1_700_000_000 + index * 300} username=alice action=failed event={index}"
for index in range(12)
]
lines.extend(f"timestamp={1_700_003_600} username=alice action=failed burst={index}" for index in range(5))
logs_path.write_text("\n".join(lines), encoding="utf-8")
result = run_local_replay(str(config_path), str(logs_path), stream_id="windows")
self.assertEqual(result["events"], 17)
self.assertGreaterEqual(result["field_detector_counts"].get("auth_failure_burst", 0), 1)
def test_run_local_replay_with_compare_config_reports_comparison(self):
with tempfile.TemporaryDirectory() as directory:
config_path = Path(directory) / "config.json"
config_path.write_text(json.dumps({"graylog_stream_profiles": [{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}]}), encoding="utf-8")
candidate_path = Path(directory) / "candidate.json"
candidate_path.write_text(json.dumps({"graylog_stream_profiles": [{"stream_id": "windows", "entity_field": "username", "categorical_fields": []}]}), encoding="utf-8")
logs_path = Path(directory) / "events.log"
lines = [
f"timestamp={1_700_000_000 + index * 300} username=alice action=failed event={index}"
for index in range(12)
]
lines.extend(f"timestamp={1_700_003_600} username=alice action=failed burst={index}" for index in range(5))
logs_path.write_text("\n".join(lines), encoding="utf-8")
result = run_local_replay(str(config_path), str(logs_path), stream_id="windows", compare_config_file=str(candidate_path))
self.assertIn("comparison", result)
self.assertIn("current", result)
self.assertIn("candidate", result)
self.assertEqual(result["candidate_config_file"], str(candidate_path))
def test_run_graylog_replay_requires_mcp_configuration(self):
with tempfile.TemporaryDirectory() as directory:
config_path = Path(directory) / "config.json"
config_path.write_text(json.dumps({}), encoding="utf-8")
with self.assertRaises(RuntimeError):
run_graylog_replay(str(config_path))
def test_run_graylog_replay_requires_selected_streams(self):
with tempfile.TemporaryDirectory() as directory:
config_path = Path(directory) / "config.json"
config_path.write_text(json.dumps({"graylog_mcp_url": "https://graylog.example", "graylog_mcp_token": "token"}), encoding="utf-8")
with self.assertRaises(RuntimeError):
run_graylog_replay(str(config_path))