From 6ea8bfd714e84771b9c764cbdbec6a73657f3bd3979b60f7201690071626fc8d Mon Sep 17 00:00:00 2001 From: larssand Date: Mon, 29 Jun 2026 19:48:29 +0200 Subject: [PATCH] investigation export. --- README.md | 13 ++++ ROADMAP.md | 2 +- src/fgai/cli.py | 24 ++++++++ src/fgai/dashboard.py | 17 ++++++ src/fgai/exports.py | 135 ++++++++++++++++++++++++++++++++++++++++++ tests/test_exports.py | 75 +++++++++++++++++++++++ 6 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 src/fgai/exports.py create mode 100644 tests/test_exports.py diff --git a/README.md b/README.md index dfca30a..b1489c0 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,19 @@ Use the dashboard incident actions to acknowledge, resolve, or reopen an inciden and attach a note. The state is keyed to a stable incident fingerprint so it can survive monitor refreshes even when the current detection window changes. +Export the current investigation view when you need to share or archive an +incident outside the dashboard: + +```bash +signalscope export-investigation --format markdown --output exports/investigation.md +signalscope export-investigation --incident-id --format json +``` + +The report is built from `state/fgai-status.json` by default and includes +summary counters, stream coverage, incident state, analyst notes, evidence, +timeline rows, and Graylog query details. The dashboard exposes the same data at +`/api/export/incidents?format=markdown` or `format=json`. + With a stream profile in place, SignalScope also builds independent burst baselines for authentication failures, DNS queries, and deny/block actions when those events are present. These are evaluated per configured entity, so a Windows diff --git a/ROADMAP.md b/ROADMAP.md index cda4840..b5bf1c5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -45,7 +45,7 @@ Goal: make one incident answer what happened, to whom, and across which sources. - [x] Add incident lifecycle: open, acknowledged, resolved, reopened. - [x] Persist incident state and analyst notes separately from transient detection output. - [x] Add direct Graylog query links or query details for each timeline event. -- [ ] Add investigation export as JSON and Markdown report. +- [x] Add investigation export as JSON and Markdown report. Acceptance: an analyst can open an incident, see an ordered multi-stream timeline, review evidence, and record an outcome without losing it after the next monitor poll. diff --git a/src/fgai/cli.py b/src/fgai/cli.py index fd259f6..1204666 100644 --- a/src/fgai/cli.py +++ b/src/fgai/cli.py @@ -16,6 +16,7 @@ from .syslog_server import listen_udp_syslog from .monitor import monitor_loop from .threat_intel import enrich_ips, is_public_ip from .config import ConfigStore +from .exports import investigation_report, investigation_report_markdown from .graylog_mcp import GraylogMcpClient from .graylog_source import GraylogStreamSource from .replay import replay_comparison, replay_events @@ -220,6 +221,22 @@ def run_dashboard(args: argparse.Namespace) -> int: return 0 +def export_investigation(args: argparse.Namespace) -> int: + status_path = Path(args.status_file) + status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {} + report = investigation_report(status, incident_id=args.incident_id or None) + fmt = "markdown" if args.format == "md" else args.format + output = investigation_report_markdown(report) if fmt == "markdown" else json.dumps(report, indent=2, sort_keys=True) + if args.output: + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(output, encoding="utf-8") + print(f"Wrote {path}") + else: + print(output, end="" if output.endswith("\n") else "\n") + return 0 + + def _stream_name(config: dict[str, object], stream_id: str) -> str: return next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == stream_id), stream_id) @@ -406,6 +423,13 @@ def build_parser() -> argparse.ArgumentParser: dashboard.add_argument("--config-file", default="state/fgai-config.json", help="Local runtime configuration JSON") dashboard.set_defaults(func=run_dashboard) + export = subparsers.add_parser("export-investigation", help="Export current investigation incidents as Markdown or JSON") + export.add_argument("--status-file", default="state/fgai-status.json", help="Status JSON produced by monitor") + export.add_argument("--incident-id", default="", help="Export only one incident ID") + export.add_argument("--format", choices=["markdown", "md", "json"], default="markdown", help="Report format") + export.add_argument("--output", default="", help="Write the report to this file instead of stdout") + export.set_defaults(func=export_investigation) + replay = subparsers.add_parser("replay", help="Replay a historical log export against temporary baselines") replay.add_argument("--logs", required=True, help="Historic JSONL or key/value log export") replay.add_argument("--config-file", default="state/fgai-config.json", help="Stream profile configuration") diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index c1b3008..251a13e 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -3,8 +3,10 @@ from __future__ import annotations import json from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from urllib.parse import parse_qs, urlparse from .config import ConfigStore +from .exports import investigation_report, investigation_report_markdown from .graylog_mcp import GraylogMcpClient from .metrics import prometheus_metrics from .feedback import FeedbackStore @@ -388,6 +390,21 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | body = json.dumps({"summary": {}, "anomalies": [], "block_candidates": []}).encode("utf-8") self._send(200, "application/json", body) return + if self.path.startswith("/api/export/incidents"): + parsed = urlparse(self.path) + params = parse_qs(parsed.query) + fmt = params.get("format", ["markdown"])[0] + incident_id = params.get("incident_id", [""])[0] or None + try: + status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {} + except json.JSONDecodeError: + status = {} + report = investigation_report(status, incident_id=incident_id) + if fmt == "json": + self._send(200, "application/json", json.dumps(report, indent=2, sort_keys=True).encode("utf-8")) + else: + self._send(200, "text/markdown; charset=utf-8", investigation_report_markdown(report).encode("utf-8")) + return if self.path == "/metrics": try: status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {} diff --git a/src/fgai/exports.py b/src/fgai/exports.py new file mode 100644 index 0000000..3cc876f --- /dev/null +++ b/src/fgai/exports.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + + +def _list(value: object) -> list[object]: + return value if isinstance(value, list) else [] + + +def _dict(value: object) -> dict[str, object]: + return value if isinstance(value, dict) else {} + + +def _text(value: object, default: str = "-") -> str: + text = str(value or "").strip() + return text or default + + +def _incident_matches(incident: dict[str, object], incident_id: str | None) -> bool: + return not incident_id or str(incident.get("id", "")) == incident_id + + +def investigation_report(status: dict[str, object], incident_id: str | None = None) -> dict[str, object]: + incidents = [item for item in _list(status.get("incidents")) if isinstance(item, dict) and _incident_matches(item, incident_id)] + recommendations = [item for item in _list(status.get("recommendations")) if isinstance(item, dict)] + entities = {str(item.get("entity", "")) for item in incidents} + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "incident_id": incident_id or "", + "summary": _dict(status.get("summary")), + "anomaly_summary": _dict(status.get("anomaly_summary")), + "baseline": _dict(status.get("baseline")), + "data_quality": _dict(status.get("data_quality")), + "stream_coverage": _list(status.get("stream_coverage")), + "incidents": incidents, + "recommendations": [ + item for item in recommendations if not entities or str(item.get("subject", "")) in entities + ], + } + + +def investigation_report_markdown(report: dict[str, object]) -> str: + lines = [ + "# SignalScope Investigation Report", + "", + f"Generated: {_text(report.get('generated_at'))}", + ] + if report.get("incident_id"): + lines.append(f"Incident filter: `{_text(report.get('incident_id'))}`") + lines += ["", "## Summary"] + summary = _dict(report.get("summary")) + anomaly_summary = _dict(report.get("anomaly_summary")) + baseline = _dict(report.get("baseline")) + lines += [ + f"- Events: {_text(summary.get('total'), '0')}", + f"- High anomalies: {_text(anomaly_summary.get('high'), '0')}", + f"- Critical anomalies: {_text(anomaly_summary.get('critical'), '0')}", + f"- Baseline sources ready: {_text(baseline.get('sources_ready'), '0')}", + ] + + coverage = [item for item in _list(report.get("stream_coverage")) if isinstance(item, dict)] + if coverage: + lines += ["", "## Stream Coverage", "", "| Stream | Enabled | Profile | Events | Health |", "| --- | --- | --- | ---: | --- |"] + for item in coverage[:20]: + lines.append( + "| " + + " | ".join( + [ + _text(item.get("stream_name") or item.get("stream_id")), + _text(item.get("enabled")), + _text(item.get("profile_name") or item.get("profile")), + _text(item.get("events_fetched"), "0"), + _text(item.get("health")), + ] + ) + + " |" + ) + + incidents = [item for item in _list(report.get("incidents")) if isinstance(item, dict)] + lines += ["", "## Incidents"] + if not incidents: + lines.append("No incidents matched the report scope.") + for incident in incidents: + lines += [ + "", + f"### {_text(incident.get('entity'))} ({_text(incident.get('severity'))})", + "", + f"- ID: `{_text(incident.get('id'))}`", + f"- Type: {_text(incident.get('entity_type'))}", + f"- Score: {_text(incident.get('score'), '0')}", + f"- State: {_text(incident.get('lifecycle_status'), 'open')}", + f"- Streams: {_text(', '.join(str(item) for item in _list(incident.get('correlated_streams'))))}", + f"- First seen: {_text(incident.get('first_seen'))}", + f"- Last seen: {_text(incident.get('last_seen'))}", + ] + if incident.get("note"): + lines.append(f"- Analyst note: {_text(incident.get('note'))}") + evidence = [str(item) for item in _list(incident.get("evidence")) if item] + if evidence: + lines += ["", "Evidence:"] + lines.extend(f"- {item}" for item in evidence[:10]) + timeline = [item for item in _list(incident.get("timeline")) if isinstance(item, dict)] + if timeline: + lines += ["", "Timeline:", "", "| Time | Stream | Action | Destination | Service | Context | Graylog query |", "| --- | --- | --- | --- | --- | --- | --- |"] + for item in timeline[:20]: + query = _dict(item.get("query_details")).get("query") if isinstance(item.get("query_details"), dict) else item.get("graylog_query") + lines.append( + "| " + + " | ".join( + _text(value).replace("|", "\\|") + for value in ( + item.get("timestamp"), + item.get("stream_name") or item.get("stream_title") or item.get("stream_id"), + item.get("action"), + item.get("destination"), + item.get("service"), + item.get("context") or item.get("message"), + query, + ) + ) + + " |" + ) + + recommendations = [item for item in _list(report.get("recommendations")) if isinstance(item, dict)] + if recommendations: + lines += ["", "## Recommendations"] + for item in recommendations[:20]: + lines += [ + "", + f"- {_text(item.get('subject'))}: {_text(item.get('title'))} ({_text(item.get('severity'))}, score {_text(item.get('score'), '0')})", + f" Recommendation: {_text(item.get('recommendation'))}", + ] + + return "\n".join(lines).rstrip() + "\n" diff --git a/tests/test_exports.py b/tests/test_exports.py new file mode 100644 index 0000000..ddb07ef --- /dev/null +++ b/tests/test_exports.py @@ -0,0 +1,75 @@ +import unittest + +from fgai.exports import investigation_report, investigation_report_markdown + + +class InvestigationExportTests(unittest.TestCase): + def test_report_filters_incident_and_related_recommendations(self): + status = { + "summary": {"total": 42}, + "anomaly_summary": {"critical": 1}, + "incidents": [ + {"id": "one", "entity": "alice", "severity": "high", "score": 70, "timeline": []}, + {"id": "two", "entity": "10.0.0.5", "severity": "medium", "score": 35, "timeline": []}, + ], + "recommendations": [ + {"subject": "alice", "title": "Review login", "severity": "high", "score": 70}, + {"subject": "other", "title": "Ignore", "severity": "low", "score": 10}, + ], + } + + report = investigation_report(status, incident_id="one") + + self.assertEqual([item["id"] for item in report["incidents"]], ["one"]) + self.assertEqual([item["subject"] for item in report["recommendations"]], ["alice"]) + self.assertEqual(report["summary"]["total"], 42) + + def test_markdown_includes_evidence_timeline_and_query_details(self): + report = investigation_report( + { + "summary": {"total": 10}, + "anomaly_summary": {"high": 1}, + "baseline": {"sources_ready": 2}, + "stream_coverage": [ + {"stream_name": "Windows", "enabled": True, "profile_name": "Windows profile", "events_fetched": 9, "health": "ready"} + ], + "incidents": [ + { + "id": "abc123", + "entity": "alice", + "entity_type": "user", + "score": 88, + "severity": "critical", + "lifecycle_status": "acknowledged", + "note": "Known test account.", + "correlated_streams": ["Windows", "Firewall"], + "first_seen": "2026-06-29T10:00:00Z", + "last_seen": "2026-06-29T10:05:00Z", + "evidence": ["failed logins above baseline"], + "timeline": [ + { + "timestamp": "2026-06-29T10:00:00Z", + "stream_name": "Windows", + "action": "failure", + "destination": "host01", + "service": "logon", + "context": "4625", + "query_details": {"query": 'username:"alice" AND event_id:4625'}, + } + ], + } + ], + } + ) + + markdown = investigation_report_markdown(report) + + self.assertIn("SignalScope Investigation Report", markdown) + self.assertIn("alice (critical)", markdown) + self.assertIn("Known test account.", markdown) + self.assertIn('username:"alice" AND event_id:4625', markdown) + self.assertIn("Windows profile", markdown) + + +if __name__ == "__main__": + unittest.main()