From 16b5bbdd631f66fb417e14e5b1addd5986a27c46be5941a7fd7838ed4b3f0569 Mon Sep 17 00:00:00 2001 From: larssand Date: Mon, 29 Jun 2026 19:26:16 +0200 Subject: [PATCH] Graylog query details --- README.md | 5 +++++ ROADMAP.md | 2 +- src/fgai/baseline.py | 6 +++++- src/fgai/dashboard.py | 4 ++-- src/fgai/entities.py | 35 ++++++++++++++++++++--------------- src/fgai/query_details.py | 37 +++++++++++++++++++++++++++++++++++++ src/fgai/sequences.py | 6 +++++- tests/test_query_details.py | 28 ++++++++++++++++++++++++++++ 8 files changed, 103 insertions(+), 20 deletions(-) create mode 100644 src/fgai/query_details.py create mode 100644 tests/test_query_details.py diff --git a/README.md b/README.md index 998c9f1..dfca30a 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,11 @@ Graylog event is not learned repeatedly. Related anomalies, profile deviations, and multi-stream correlations are grouped into investigation incidents with a compact evidence timeline. +Timeline and related-activity rows include copyable Graylog query details built +from normalized source, destination, action, and DNS fields. These are query +details rather than hard-coded web links, so they work with MCP and with Graylog +deployments behind different URLs or reverse proxies. + Incident lifecycle state is stored locally in `state/signalscope-incidents.json`. Use the dashboard incident actions to acknowledge, resolve, or reopen an incident and attach a note. The state is keyed to a stable incident fingerprint so it can diff --git a/ROADMAP.md b/ROADMAP.md index e4a8d16..cda4840 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -44,7 +44,7 @@ Goal: make one incident answer what happened, to whom, and across which sources. - [ ] Add entity aliasing: map DHCP, VPN, DNS, and endpoint identities to the same host where evidence supports it. - [x] Add incident lifecycle: open, acknowledged, resolved, reopened. - [x] Persist incident state and analyst notes separately from transient detection output. -- [ ] Add direct Graylog query links or query details for each timeline event. +- [x] Add direct Graylog query links or query details for each timeline event. - [ ] Add investigation export as JSON and Markdown report. Acceptance: an analyst can open an incident, see an ordered multi-stream timeline, review evidence, and record an outcome without losing it after the next monitor poll. diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 4106d56..501139e 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -13,6 +13,7 @@ from .models import LogEvent from .entities import profile_entities from .detectors import DETECTOR_MINIMUMS, event_detector_categories from .normalization import canonical_value +from .query_details import event_query_details def _number(value: str | None) -> int: @@ -63,7 +64,8 @@ def _weighted_score(base: int, profile: object | None, field: str, detector: str return min(100, max(0, int(round(base * multiplier)))), round(multiplier, 2) -def _sample_event(event: LogEvent, value: str = "") -> dict[str, str]: +def _sample_event(event: LogEvent, value: str = "") -> dict[str, object]: + query_details = event_query_details(event) return { "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), @@ -73,6 +75,8 @@ def _sample_event(event: LogEvent, value: str = "") -> dict[str, str]: "service": canonical_value(event.fields, "service"), "value": value, "message": canonical_value(event.fields, "context")[:240], + "query_details": query_details, + "graylog_query": query_details["query"], } diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 4e2343d..c1b3008 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -179,7 +179,7 @@ async function refresh() { {label:'State', render:r => esc(r.lifecycle_status || 'open')}, {label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')}, {label:'Evidence', render:r => esc((r.evidence || []).join('; '))}, - {label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('
'); const id=`incident:${r.id || r.entity}:${r.first_seen || ''}`; return rows ? `
${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}

${rows}

` : '-'; }}, + {label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => `${esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)}${item.graylog_query ? `
${esc(item.graylog_query)}` : ''}`).join('
'); const id=`incident:${r.id || r.entity}:${r.first_seen || ''}`; return rows ? `
${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}

${rows}

` : '-'; }}, {label:'Action', render:r => `
${r.note ? `
${esc(r.note)}
` : ''}`} ], 'incidents'); document.querySelectorAll('.incident-action').forEach(button => button.addEventListener('click', async () => { @@ -213,7 +213,7 @@ async function refresh() { document.getElementById('relatedActivity').innerHTML = table(relatedRows, [ {label:'Entity', key:'entity', render:r => esc(r.entity || r.source_ip)}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, - {label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'} + {label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'}, {label:'Graylog Query', render:r => r.graylog_query ? `${esc(r.graylog_query)}` : '-'} ], 'related-activity'); document.getElementById('reputation').innerHTML = table(reputationRows, [ {label:'IP', key:'ip'}, diff --git a/src/fgai/entities.py b/src/fgai/entities.py index 024af20..f802376 100644 --- a/src/fgai/entities.py +++ b/src/fgai/entities.py @@ -5,6 +5,7 @@ from collections.abc import Iterable from .models import LogEvent from .normalization import canonical_value +from .query_details import event_query_details ENTITY_FIELDS: dict[str, tuple[str, ...]] = { @@ -52,19 +53,23 @@ def profile_entities(event: LogEvent, profile: object) -> tuple[str, ...]: return tuple(values) -def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]: - samples = [ - { - "stream": event.fields.get("fgai_stream", "local_syslog"), - "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), - "type": canonical_value(event.fields, "type"), - "subtype": event.subtype, - "action": event.action, - "severity": event.severity, - "destination": event.dst_ip or canonical_value(event.fields, "context"), - "service": canonical_value(event.fields, "service"), - "context": canonical_value(event.fields, "context")[:240], - } - for event in events - ] +def _timeline_sample(event: LogEvent) -> dict[str, object]: + query_details = event_query_details(event) + return { + "stream": event.fields.get("fgai_stream", "local_syslog"), + "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), + "type": canonical_value(event.fields, "type"), + "subtype": event.subtype, + "action": event.action, + "severity": event.severity, + "destination": event.dst_ip or canonical_value(event.fields, "context"), + "service": canonical_value(event.fields, "service"), + "context": canonical_value(event.fields, "context")[:240], + "query_details": query_details, + "graylog_query": query_details["query"], + } + + +def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, object]]: + samples = [_timeline_sample(event) for event in events] return sorted(samples, key=lambda item: item["timestamp"])[-limit:] diff --git a/src/fgai/query_details.py b/src/fgai/query_details.py new file mode 100644 index 0000000..efff7d2 --- /dev/null +++ b/src/fgai/query_details.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import json + +from .models import LogEvent +from .normalization import FIELD_ALIASES, canonical_value + + +def _quote(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _field_group(canonical: str, value: str, *, limit: int = 6) -> str: + aliases = FIELD_ALIASES.get(canonical, (canonical,))[:limit] + return "(" + " OR ".join(f"{field}:{_quote(value)}" for field in aliases) + ")" + + +def event_query_details(event: LogEvent) -> dict[str, str]: + """Build a copyable Graylog query for an event without assuming one vendor schema.""" + terms: list[str] = [] + for canonical in ("srcip", "dstip", "action", "dns_query"): + value = canonical_value(event.fields, canonical) + if value: + terms.append(_field_group(canonical, value)) + timestamp = event.fields.get("eventtime", event.fields.get("timestamp", "")) + if not terms: + context = canonical_value(event.fields, "context") + if context: + terms.append(_field_group("context", context, limit=4)) + return { + "stream": event.fields.get("fgai_stream", ""), + "stream_id": event.fields.get("fgai_stream_id", ""), + "timestamp": timestamp, + "query": " AND ".join(terms) if terms else "*", + "raw": event.raw if event.raw.startswith("{") else json.dumps(event.fields, sort_keys=True), + } diff --git a/src/fgai/sequences.py b/src/fgai/sequences.py index a4b32a3..c368d9f 100644 --- a/src/fgai/sequences.py +++ b/src/fgai/sequences.py @@ -8,6 +8,7 @@ from .entities import event_entities from .logs import THREAT_ACTIONS from .models import LogEvent from .normalization import canonical_value +from .query_details import event_query_details DEFAULT_SEQUENCE_PATTERNS = { @@ -46,7 +47,8 @@ def _is_network_event(event: LogEvent) -> bool: return event.action in {"accept", "pass", "allowed", "allow", "close", "client-rst", "server-rst"} | THREAT_ACTIONS -def _sample(event: LogEvent, value: str) -> dict[str, str]: +def _sample(event: LogEvent, value: str) -> dict[str, object]: + query_details = event_query_details(event) return { "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "stream": event.fields.get("fgai_stream", event.fields.get("fgai_stream_id", "")), @@ -57,6 +59,8 @@ def _sample(event: LogEvent, value: str) -> dict[str, str]: "service": canonical_value(event.fields, "service"), "value": value, "message": canonical_value(event.fields, "context")[:240], + "query_details": query_details, + "graylog_query": query_details["query"], } diff --git a/tests/test_query_details.py b/tests/test_query_details.py new file mode 100644 index 0000000..3e8c367 --- /dev/null +++ b/tests/test_query_details.py @@ -0,0 +1,28 @@ +import unittest + +from fgai.entities import sample_timeline +from fgai.logs import parse_log_line +from fgai.query_details import event_query_details + + +class QueryDetailsTests(unittest.TestCase): + def test_builds_generic_graylog_query_from_aliases(self): + event = parse_log_line("src_addr=10.0.0.5 dst_addr=198.51.100.10 fw_action=allow") + + details = event_query_details(event) + + self.assertIn('srcip:"10.0.0.5"', details["query"]) + self.assertIn('dstip:"198.51.100.10"', details["query"]) + self.assertIn('action:"allow"', details["query"]) + + def test_timeline_contains_query_details(self): + event = parse_log_line('timestamp=2026-06-29T16:45:52Z fgai_stream=Firewall srcip=10.0.0.5 dstip=198.51.100.10 action=deny') + + row = sample_timeline([event])[0] + + self.assertEqual(row["query_details"]["stream"], "Firewall") + self.assertIn('srcip:"10.0.0.5"', row["graylog_query"]) + + +if __name__ == "__main__": + unittest.main()