Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multi-entity stream profiles.

Browse Source
This commit is contained in:
larssand
2026-06-29 19:18:11 +02:00
parent 68370217da
commit aab9f15c6d
6 changed files with 101 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@@ -105,6 +105,11 @@ Graylog event is not learned repeatedly. Related anomalies, profile deviations,
and multi-stream correlations are grouped into investigation incidents with a and multi-stream correlations are grouped into investigation incidents with a
compact evidence timeline. compact evidence timeline.
Incident lifecycle state is stored locally in `state/signalscope-incidents.json`.
Use the dashboard incident actions to acknowledge, resolve, or reopen an incident
and attach a note. The state is keyed to a stable incident fingerprint so it can
survive monitor refreshes even when the current detection window changes.
With a stream profile in place, SignalScope also builds independent burst With a stream profile in place, SignalScope also builds independent burst
baselines for authentication failures, DNS queries, and deny/block actions when baselines for authentication failures, DNS queries, and deny/block actions when
those events are present. These are evaluated per configured entity, so a Windows those events are present. These are evaluated per configured entity, so a Windows

4
ROADMAP.md
View File

@@ -42,8 +42,8 @@ Goal: make one incident answer what happened, to whom, and across which sources.
- [x] Allow multiple entity fields per stream, such as user plus source IP plus hostname. - [x] Allow multiple entity fields per stream, such as user plus source IP plus hostname.
- [ ] Add entity aliasing: map DHCP, VPN, DNS, and endpoint identities to the same host where evidence supports it. - [ ] Add entity aliasing: map DHCP, VPN, DNS, and endpoint identities to the same host where evidence supports it.
- [ ] Add configurable incident grouping windows and incident lifecycle: open, acknowledged, resolved, reopened. - [x] Add incident lifecycle: open, acknowledged, resolved, reopened.
- [ ] Persist incident state and analyst notes separately from transient detection output. - [x] Persist incident state and analyst notes separately from transient detection output.
- [ ] Add direct Graylog query links or query details for each timeline event. - [ ] Add direct Graylog query links or query details for each timeline event.
- [ ] Add investigation export as JSON and Markdown report. - [ ] Add investigation export as JSON and Markdown report.

21
src/fgai/dashboard.py
View File

@@ -8,6 +8,7 @@ from .config import ConfigStore
from .graylog_mcp import GraylogMcpClient from .graylog_mcp import GraylogMcpClient
from .metrics import prometheus_metrics from .metrics import prometheus_metrics
from .feedback import FeedbackStore from .feedback import FeedbackStore
from .incidents import IncidentStore
HTML = """<!doctype html> HTML = """<!doctype html>
@@ -175,10 +176,18 @@ async function refresh() {
{label:'Entity', key:'entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)}, {label:'Entity', key:'entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`}, {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'State', render:r => esc(r.lifecycle_status || 'open')},
{label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')}, {label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')},
{label:'Evidence', render:r => esc((r.evidence || []).join('; '))}, {label:'Evidence', render:r => esc((r.evidence || []).join('; '))},
{label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('<br>'); const id=`incident:${r.entity}:${r.first_seen || ''}`; return rows ? `<details data-detail-id="${esc(id)}"><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }} {label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('<br>'); const id=`incident:${r.id || r.entity}:${r.first_seen || ''}`; return rows ? `<details data-detail-id="${esc(id)}"><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }},
{label:'Action', render:r => `<div class="review-actions"><button class="incident-action" data-id="${esc(r.id)}" data-status="acknowledged">Ack</button><button class="incident-action" data-id="${esc(r.id)}" data-status="resolved">Resolve</button><button class="incident-action" data-id="${esc(r.id)}" data-status="open">Reopen</button></div>${r.note ? `<div class="muted">${esc(r.note)}</div>` : ''}`}
], 'incidents'); ], 'incidents');
document.querySelectorAll('.incident-action').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Incident note (optional):') || '';
const response = await fetch('/api/incidents', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({id:button.dataset.id, status:button.dataset.status, note})});
document.getElementById('feedbackNotice').textContent = response.ok ? 'Incident state saved.' : 'Could not save incident state.';
refresh();
}));
document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [ document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
{label:'Source', key:'src_ip'}, {label:'Source', key:'src_ip'},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
@@ -330,6 +339,9 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
if self.path == "/api/feedback": if self.path == "/api/feedback":
self._send(200, "application/json", json.dumps(FeedbackStore().entries()).encode("utf-8")) self._send(200, "application/json", json.dumps(FeedbackStore().entries()).encode("utf-8"))
return return
if self.path == "/api/incidents":
self._send(200, "application/json", json.dumps(IncidentStore().entries()).encode("utf-8"))
return
if self.path == "/api/graylog/streams": if self.path == "/api/graylog/streams":
config = config_store.read() config = config_store.read()
try: try:
@@ -399,6 +411,13 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
except (ValueError, json.JSONDecodeError) as exc: except (ValueError, json.JSONDecodeError) as exc:
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
return return
if self.path == "/api/incidents" and self._is_loopback_client():
try:
payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8"))
self._send(200, "application/json", json.dumps(IncidentStore().update(str(payload.get("id", "")), str(payload.get("status", "")), str(payload.get("note", "")))).encode("utf-8"))
except (ValueError, json.JSONDecodeError) as exc:
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
return
if self.path != "/api/config" or not self._is_loopback_client(): if self.path != "/api/config" or not self._is_loopback_client():
self._send(403, "application/json", b'{"error":"configuration is local-only"}') self._send(403, "application/json", b'{"error":"configuration is local-only"}')
return return

53
src/fgai/incidents.py
View File

@@ -1,6 +1,10 @@
from __future__ import annotations from __future__ import annotations
import hashlib
import json
import time
from collections import defaultdict from collections import defaultdict
from pathlib import Path
from .entities import entity_type from .entities import entity_type
from .models import AnomalyFinding from .models import AnomalyFinding
@@ -39,7 +43,9 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str,
severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low" severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id") or item.get("stream_name") or item.get("stream_title")}) streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id") or item.get("stream_name") or item.get("stream_title")})
timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20] timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
incident_id = hashlib.sha256(json.dumps({"entity": entity, "streams": streams, "evidence": list(dict.fromkeys(str(item) for item in group["evidence"] if item))[:4]}, sort_keys=True).encode("utf-8")).hexdigest()[:16]
incidents.append({ incidents.append({
"id": incident_id,
"entity": entity, "entity": entity,
"entity_type": entity_type(entity), "entity_type": entity_type(entity),
"score": score, "score": score,
@@ -52,3 +58,50 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str,
"last_seen": timeline[-1].get("timestamp", "") if timeline else "", "last_seen": timeline[-1].get("timestamp", "") if timeline else "",
}) })
return sorted(incidents, key=lambda item: int(item["score"]), reverse=True) return sorted(incidents, key=lambda item: int(item["score"]), reverse=True)
class IncidentStore:
def __init__(self, path: str = "state/signalscope-incidents.json") -> None:
self.path = Path(path)
def entries(self) -> dict[str, dict[str, object]]:
try:
items = json.loads(self.path.read_text(encoding="utf-8"))
except (FileNotFoundError, json.JSONDecodeError):
return {}
return {str(key): value for key, value in items.items() if isinstance(value, dict)} if isinstance(items, dict) else {}
def apply(self, incidents: list[dict[str, object]]) -> list[dict[str, object]]:
states = self.entries()
now = int(time.time())
changed = False
for incident in incidents:
incident_id = str(incident.get("id", ""))
if not incident_id:
continue
state = states.get(incident_id)
if not state:
state = {"status": "open", "note": "", "created_at": now, "updated_at": now}
states[incident_id] = state
changed = True
incident["lifecycle_status"] = str(state.get("status", "open"))
incident["note"] = str(state.get("note", ""))
incident["updated_at"] = int(state.get("updated_at", 0) or 0)
if changed:
self._write(states)
return incidents
def update(self, incident_id: str, status: str, note: str = "") -> dict[str, object]:
status = status.lower()
if status not in {"open", "acknowledged", "resolved"}:
raise ValueError("invalid incident status")
states = self.entries()
current = states.get(incident_id, {"created_at": int(time.time())})
entry = {**current, "status": status, "note": note, "updated_at": int(time.time())}
states[incident_id] = entry
self._write(states)
return {"id": incident_id, **entry}
def _write(self, states: dict[str, dict[str, object]]) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(states, indent=2, sort_keys=True), encoding="utf-8")

6
src/fgai/monitor.py
View File

@@ -13,7 +13,7 @@ from .feedback import FeedbackStore
from .graylog_mcp import GraylogMcpClient from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource from .graylog_source import GraylogStreamSource
from .history import HistoryStore from .history import HistoryStore
from .incidents import build_incidents from .incidents import IncidentStore, build_incidents
from .data_quality import assess_data_quality from .data_quality import assess_data_quality
from .llm import ollama_dashboard_assessment from .llm import ollama_dashboard_assessment
from .logs import local_in_failures, read_events, summarize_events, top_field_values from .logs import local_in_failures, read_events, summarize_events, top_field_values
@@ -102,6 +102,7 @@ def build_status(
baseline_path: str | None = None, baseline_path: str | None = None,
config_path: str | None = None, config_path: str | None = None,
history_path: str | None = None, history_path: str | None = None,
incident_path: str | None = None,
) -> dict[str, object]: ) -> dict[str, object]:
config_store = ConfigStore(config_path) if config_path else None config_store = ConfigStore(config_path) if config_path else None
config_exists = bool(config_store and config_store.path.exists()) config_exists = bool(config_store and config_store.path.exists())
@@ -202,6 +203,7 @@ def build_status(
threat_intel_status = ThreatIntelClient(enabled=threat_enabled).status() threat_intel_status = ThreatIntelClient(enabled=threat_enabled).status()
recommendations = build_recommendations(events, anomalies, reputation) recommendations = build_recommendations(events, anomalies, reputation)
correlations = correlate_source_ips(events) correlations = correlate_source_ips(events)
incidents = IncidentStore(incident_path or "state/signalscope-incidents.json").apply(build_incidents(anomalies, field_deviations, correlations))
block_candidates = suggest_block_candidates( block_candidates = suggest_block_candidates(
events, events,
min_events=min_block_events, min_events=min_block_events,
@@ -245,7 +247,7 @@ def build_status(
"sequence_findings": sequence_findings, "sequence_findings": sequence_findings,
"feedback": feedback, "feedback": feedback,
"cross_source_correlations": correlations, "cross_source_correlations": correlations,
"incidents": build_incidents(anomalies, field_deviations, correlations), "incidents": incidents,
"data_quality": assess_data_quality(events, mcp_status), "data_quality": assess_data_quality(events, mcp_status),
"anomalies": [ "anomalies": [
{ {

18
tests/test_incidents.py
View File

@@ -1,5 +1,8 @@
import unittest import unittest
from fgai.incidents import build_incidents import tempfile
from pathlib import Path
from fgai.incidents import IncidentStore, build_incidents
from fgai.models import AnomalyFinding from fgai.models import AnomalyFinding
class IncidentTests(unittest.TestCase): class IncidentTests(unittest.TestCase):
@@ -16,3 +19,16 @@ class IncidentTests(unittest.TestCase):
def test_incident_uses_stream_name_for_field_deviation(self): def test_incident_uses_stream_name_for_field_deviation(self):
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "6a3993", "stream_name": "Windows"}]}, []) result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "6a3993", "stream_name": "Windows"}]}, [])
self.assertEqual(result[0]["correlated_streams"], ["Windows"]) self.assertEqual(result[0]["correlated_streams"], ["Windows"])
def test_incident_store_persists_lifecycle_state(self):
with tempfile.TemporaryDirectory() as directory:
store = IncidentStore(str(Path(directory) / "incidents.json"))
incident = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, [])[0]
applied = store.apply([incident])[0]
self.assertEqual(applied["lifecycle_status"], "open")
store.update(str(applied["id"]), "acknowledged", "checking vpn logs")
applied_again = store.apply([incident])[0]
self.assertEqual(applied_again["lifecycle_status"], "acknowledged")
self.assertEqual(applied_again["note"], "checking vpn logs")