Graylog query details
This commit is contained in:
@@ -105,6 +105,11 @@ Graylog event is not learned repeatedly. Related anomalies, profile deviations,
|
||||
and multi-stream correlations are grouped into investigation incidents with a
|
||||
compact evidence timeline.
|
||||
|
||||
Timeline and related-activity rows include copyable Graylog query details built
|
||||
from normalized source, destination, action, and DNS fields. These are query
|
||||
details rather than hard-coded web links, so they work with MCP and with Graylog
|
||||
deployments behind different URLs or reverse proxies.
|
||||
|
||||
Incident lifecycle state is stored locally in `state/signalscope-incidents.json`.
|
||||
Use the dashboard incident actions to acknowledge, resolve, or reopen an incident
|
||||
and attach a note. The state is keyed to a stable incident fingerprint so it can
|
||||
|
||||
@@ -44,7 +44,7 @@ Goal: make one incident answer what happened, to whom, and across which sources.
|
||||
- [ ] Add entity aliasing: map DHCP, VPN, DNS, and endpoint identities to the same host where evidence supports it.
|
||||
- [x] Add incident lifecycle: open, acknowledged, resolved, reopened.
|
||||
- [x] Persist incident state and analyst notes separately from transient detection output.
|
||||
- [ ] Add direct Graylog query links or query details for each timeline event.
|
||||
- [x] Add direct Graylog query links or query details for each timeline event.
|
||||
- [ ] Add investigation export as JSON and Markdown report.
|
||||
|
||||
Acceptance: an analyst can open an incident, see an ordered multi-stream timeline, review evidence, and record an outcome without losing it after the next monitor poll.
|
||||
|
||||
@@ -13,6 +13,7 @@ from .models import LogEvent
|
||||
from .entities import profile_entities
|
||||
from .detectors import DETECTOR_MINIMUMS, event_detector_categories
|
||||
from .normalization import canonical_value
|
||||
from .query_details import event_query_details
|
||||
|
||||
|
||||
def _number(value: str | None) -> int:
|
||||
@@ -63,7 +64,8 @@ def _weighted_score(base: int, profile: object | None, field: str, detector: str
|
||||
return min(100, max(0, int(round(base * multiplier)))), round(multiplier, 2)
|
||||
|
||||
|
||||
def _sample_event(event: LogEvent, value: str = "") -> dict[str, str]:
|
||||
def _sample_event(event: LogEvent, value: str = "") -> dict[str, object]:
|
||||
query_details = event_query_details(event)
|
||||
return {
|
||||
"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
|
||||
"source": event.src_ip or event.fields.get("source", ""),
|
||||
@@ -73,6 +75,8 @@ def _sample_event(event: LogEvent, value: str = "") -> dict[str, str]:
|
||||
"service": canonical_value(event.fields, "service"),
|
||||
"value": value,
|
||||
"message": canonical_value(event.fields, "context")[:240],
|
||||
"query_details": query_details,
|
||||
"graylog_query": query_details["query"],
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -179,7 +179,7 @@ async function refresh() {
|
||||
{label:'State', render:r => esc(r.lifecycle_status || 'open')},
|
||||
{label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')},
|
||||
{label:'Evidence', render:r => esc((r.evidence || []).join('; '))},
|
||||
{label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('<br>'); const id=`incident:${r.id || r.entity}:${r.first_seen || ''}`; return rows ? `<details data-detail-id="${esc(id)}"><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }},
|
||||
{label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => `${esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>` : ''}`).join('<br>'); const id=`incident:${r.id || r.entity}:${r.first_seen || ''}`; return rows ? `<details data-detail-id="${esc(id)}"><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }},
|
||||
{label:'Action', render:r => `<div class="review-actions"><button class="incident-action" data-id="${esc(r.id)}" data-status="acknowledged">Ack</button><button class="incident-action" data-id="${esc(r.id)}" data-status="resolved">Resolve</button><button class="incident-action" data-id="${esc(r.id)}" data-status="open">Reopen</button></div>${r.note ? `<div class="muted">${esc(r.note)}</div>` : ''}`}
|
||||
], 'incidents');
|
||||
document.querySelectorAll('.incident-action').forEach(button => button.addEventListener('click', async () => {
|
||||
@@ -213,7 +213,7 @@ async function refresh() {
|
||||
document.getElementById('relatedActivity').innerHTML = table(relatedRows, [
|
||||
{label:'Entity', key:'entity', render:r => esc(r.entity || r.source_ip)}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
|
||||
{label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'},
|
||||
{label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'}
|
||||
{label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'}, {label:'Graylog Query', render:r => r.graylog_query ? `<code>${esc(r.graylog_query)}</code>` : '-'}
|
||||
], 'related-activity');
|
||||
document.getElementById('reputation').innerHTML = table(reputationRows, [
|
||||
{label:'IP', key:'ip'},
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Iterable
|
||||
|
||||
from .models import LogEvent
|
||||
from .normalization import canonical_value
|
||||
from .query_details import event_query_details
|
||||
|
||||
|
||||
ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
|
||||
@@ -52,9 +53,9 @@ def profile_entities(event: LogEvent, profile: object) -> tuple[str, ...]:
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]:
|
||||
samples = [
|
||||
{
|
||||
def _timeline_sample(event: LogEvent) -> dict[str, object]:
|
||||
query_details = event_query_details(event)
|
||||
return {
|
||||
"stream": event.fields.get("fgai_stream", "local_syslog"),
|
||||
"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
|
||||
"type": canonical_value(event.fields, "type"),
|
||||
@@ -64,7 +65,11 @@ def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict
|
||||
"destination": event.dst_ip or canonical_value(event.fields, "context"),
|
||||
"service": canonical_value(event.fields, "service"),
|
||||
"context": canonical_value(event.fields, "context")[:240],
|
||||
"query_details": query_details,
|
||||
"graylog_query": query_details["query"],
|
||||
}
|
||||
for event in events
|
||||
]
|
||||
|
||||
|
||||
def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
|
||||
samples = [_timeline_sample(event) for event in events]
|
||||
return sorted(samples, key=lambda item: item["timestamp"])[-limit:]
|
||||
|
||||
37
src/fgai/query_details.py
Normal file
37
src/fgai/query_details.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from .models import LogEvent
|
||||
from .normalization import FIELD_ALIASES, canonical_value
|
||||
|
||||
|
||||
def _quote(value: str) -> str:
|
||||
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
def _field_group(canonical: str, value: str, *, limit: int = 6) -> str:
|
||||
aliases = FIELD_ALIASES.get(canonical, (canonical,))[:limit]
|
||||
return "(" + " OR ".join(f"{field}:{_quote(value)}" for field in aliases) + ")"
|
||||
|
||||
|
||||
def event_query_details(event: LogEvent) -> dict[str, str]:
|
||||
"""Build a copyable Graylog query for an event without assuming one vendor schema."""
|
||||
terms: list[str] = []
|
||||
for canonical in ("srcip", "dstip", "action", "dns_query"):
|
||||
value = canonical_value(event.fields, canonical)
|
||||
if value:
|
||||
terms.append(_field_group(canonical, value))
|
||||
timestamp = event.fields.get("eventtime", event.fields.get("timestamp", ""))
|
||||
if not terms:
|
||||
context = canonical_value(event.fields, "context")
|
||||
if context:
|
||||
terms.append(_field_group("context", context, limit=4))
|
||||
return {
|
||||
"stream": event.fields.get("fgai_stream", ""),
|
||||
"stream_id": event.fields.get("fgai_stream_id", ""),
|
||||
"timestamp": timestamp,
|
||||
"query": " AND ".join(terms) if terms else "*",
|
||||
"raw": event.raw if event.raw.startswith("{") else json.dumps(event.fields, sort_keys=True),
|
||||
}
|
||||
@@ -8,6 +8,7 @@ from .entities import event_entities
|
||||
from .logs import THREAT_ACTIONS
|
||||
from .models import LogEvent
|
||||
from .normalization import canonical_value
|
||||
from .query_details import event_query_details
|
||||
|
||||
|
||||
DEFAULT_SEQUENCE_PATTERNS = {
|
||||
@@ -46,7 +47,8 @@ def _is_network_event(event: LogEvent) -> bool:
|
||||
return event.action in {"accept", "pass", "allowed", "allow", "close", "client-rst", "server-rst"} | THREAT_ACTIONS
|
||||
|
||||
|
||||
def _sample(event: LogEvent, value: str) -> dict[str, str]:
|
||||
def _sample(event: LogEvent, value: str) -> dict[str, object]:
|
||||
query_details = event_query_details(event)
|
||||
return {
|
||||
"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
|
||||
"stream": event.fields.get("fgai_stream", event.fields.get("fgai_stream_id", "")),
|
||||
@@ -57,6 +59,8 @@ def _sample(event: LogEvent, value: str) -> dict[str, str]:
|
||||
"service": canonical_value(event.fields, "service"),
|
||||
"value": value,
|
||||
"message": canonical_value(event.fields, "context")[:240],
|
||||
"query_details": query_details,
|
||||
"graylog_query": query_details["query"],
|
||||
}
|
||||
|
||||
|
||||
|
||||
28
tests/test_query_details.py
Normal file
28
tests/test_query_details.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import unittest
|
||||
|
||||
from fgai.entities import sample_timeline
|
||||
from fgai.logs import parse_log_line
|
||||
from fgai.query_details import event_query_details
|
||||
|
||||
|
||||
class QueryDetailsTests(unittest.TestCase):
|
||||
def test_builds_generic_graylog_query_from_aliases(self):
|
||||
event = parse_log_line("src_addr=10.0.0.5 dst_addr=198.51.100.10 fw_action=allow")
|
||||
|
||||
details = event_query_details(event)
|
||||
|
||||
self.assertIn('srcip:"10.0.0.5"', details["query"])
|
||||
self.assertIn('dstip:"198.51.100.10"', details["query"])
|
||||
self.assertIn('action:"allow"', details["query"])
|
||||
|
||||
def test_timeline_contains_query_details(self):
|
||||
event = parse_log_line('timestamp=2026-06-29T16:45:52Z fgai_stream=Firewall srcip=10.0.0.5 dstip=198.51.100.10 action=deny')
|
||||
|
||||
row = sample_timeline([event])[0]
|
||||
|
||||
self.assertEqual(row["query_details"]["stream"], "Firewall")
|
||||
self.assertIn('srcip:"10.0.0.5"', row["graylog_query"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user