diff --git a/README.md b/README.md index 8b0144c..bd22a77 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,12 @@ baselines for authentication failures, DNS queries, and deny/block actions when those events are present. These are evaluated per configured entity, so a Windows account, DNS client, or firewall source is compared to its own history. +SignalScope also detects ordered behavior sequences across any streams that share +an entity. The built-in sequence is category-based, not source-specific: +`dns_query -> network_connection -> auth_failure`. Those categories can come from +AdGuard, Windows DNS, a proxy, firewall, VPN, endpoint, or any other Graylog +stream as long as the fields normalize into the same generic event model. + Stream profiles can also carry `field_weights` to tune scoring without changing the baseline itself. Weights are multipliers from `0` to `5` and can target a field, a detector, or a field+detector pair: diff --git a/ROADMAP.md b/ROADMAP.md index c8842df..584f0f5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -27,7 +27,7 @@ Goal: make findings more accurate before adding more integrations. - [x] Add rare-value detection with a minimum historical observation threshold. - [x] Add detector-specific authentication failure, DNS volume, and denied-traffic burst thresholds. - [x] Add configurable per-field detector weights. -- [ ] Add sequence detection, for example DNS lookup -> outbound connection -> authentication event. +- [x] Add sequence detection, for example DNS lookup -> outbound connection -> authentication event. - [x] Add per-stream detector enablement and thresholds in the UI. - [x] Add a dry-run replay command for historic JSONL or Graylog exports using temporary baselines. - [x] Add direct Graylog MCP time-range replay and result comparison against saved detector configurations. diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index e03534f..7d34766 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -20,6 +20,7 @@ from .logs import local_in_failures, read_events, summarize_events, top_field_va from .mitigation import parse_allowlist, suggest_block_candidates from .policies import audit_policies, read_policies from .recommendations import build_recommendations +from .sequences import detect_sequences from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip from .stream_profiles import parse_profiles @@ -104,6 +105,9 @@ def build_status( baseline = BaselineStore(baseline_path) if baseline_path else None profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {} field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {} + sequence_findings = detect_sequences(events) + for entity, findings in sequence_findings.items(): + field_deviations.setdefault(entity, []).extend(findings) for deviations in field_deviations.values(): for deviation in deviations: stream_id = str(deviation.get("stream_id", "")) @@ -194,6 +198,7 @@ def build_status( }, "event_context": build_event_context(events), "field_deviations": field_deviations, + "sequence_findings": sequence_findings, "feedback": feedback, "cross_source_correlations": correlations, "incidents": build_incidents(anomalies, field_deviations, correlations), diff --git a/src/fgai/sequences.py b/src/fgai/sequences.py new file mode 100644 index 0000000..64846d9 --- /dev/null +++ b/src/fgai/sequences.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from collections import defaultdict +from datetime import datetime + +from .detectors import event_detector_categories +from .entities import event_entities +from .logs import THREAT_ACTIONS +from .models import LogEvent + + +DEFAULT_SEQUENCE_PATTERNS = { + "dns_network_auth_sequence": ("dns_query", "network_connection", "auth_failure"), +} + + +def _timestamp(event: LogEvent, fallback: int) -> int: + value = event.fields.get("eventtime", event.fields.get("timestamp", "")) + if value: + try: + parsed = float(value) + while parsed > 10_000_000_000: + parsed /= 1_000 + return int(parsed) + except ValueError: + try: + return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()) + except ValueError: + pass + date = event.fields.get("date") + clock = event.fields.get("time") + if date and clock: + try: + return int(datetime.fromisoformat(f"{date}T{clock}").timestamp()) + except ValueError: + pass + return fallback + + +def _is_network_event(event: LogEvent) -> bool: + if event.fields.get("dstip") or event.fields.get("destination_ip") or event.fields.get("dst_ip"): + return True + if event.fields.get("dstport") or event.fields.get("destination_port") or event.fields.get("service"): + return True + return event.action in {"accept", "pass", "allowed", "allow", "close", "client-rst", "server-rst"} | THREAT_ACTIONS + + +def _sample(event: LogEvent, value: str) -> dict[str, str]: + return { + "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), + "stream": event.fields.get("fgai_stream", event.fields.get("fgai_stream_id", "")), + "source": event.src_ip or event.fields.get("source", ""), + "destination": event.dst_ip or event.fields.get("query_domain", event.fields.get("qh", "")), + "action": event.action, + "severity": event.severity, + "service": event.fields.get("service", event.fields.get("query_type", "")), + "value": value, + "message": event.fields.get("message", event.fields.get("msg", ""))[:240], + } + + +def detect_sequences(events: list[LogEvent], *, window_seconds: int = 900, limit: int = 50, patterns: dict[str, tuple[str, ...]] | None = None) -> dict[str, list[dict[str, object]]]: + """Detect ordered multi-event behavior for any entity shared across streams.""" + patterns = patterns or DEFAULT_SEQUENCE_PATTERNS + by_entity: dict[str, list[tuple[int, LogEvent, set[str]]]] = defaultdict(list) + for index, event in enumerate(events): + categories = set(event_detector_categories(event)) + if _is_network_event(event): + categories.add("network_connection") + if not categories: + continue + timestamp = _timestamp(event, index) + for identity in event_entities(event): + by_entity[identity["entity"]].append((timestamp, event, categories)) + + findings: dict[str, list[dict[str, object]]] = defaultdict(list) + for entity, items in by_entity.items(): + ordered = sorted(items, key=lambda item: item[0]) + for detector, sequence in patterns.items(): + match: list[tuple[int, LogEvent, str]] = [] + start_at = 0 + for category in sequence: + found = next( + ( + (timestamp, event, category) + for timestamp, event, categories in ordered + if timestamp >= start_at and (not match or timestamp <= match[0][0] + window_seconds) and category in categories + ), + None, + ) + if not found: + match = [] + break + match.append(found) + start_at = found[0] + if len(match) != len(sequence): + continue + duration = max(0, match[-1][0] - match[0][0]) + streams = sorted({event.fields.get("fgai_stream", "") for _, event, _ in match} - {""}) + score = 28 if len(streams) >= 2 else 20 + findings[entity].append({ + "detector": detector, + "field": "sequence", + "stream_id": "multi_stream", + "stream_name": "Multi-stream", + "stream_title": "Multi-stream", + "score": score, + "base_score": score, + "weight": 1.0, + "confidence": "medium" if len(streams) >= 2 else "low", + "baseline_samples": 0, + "baseline_scope": f"{window_seconds}s sequence window", + "reason": f"ordered {' -> '.join(sequence)} sequence observed in {duration}s", + "current": len(sequence), + "baseline": 0, + "value": detector, + "sample_values": streams, + "sample_events": [_sample(event, category) for _, event, category in match], + }) + if sum(len(values) for values in findings.values()) >= limit: + return findings + break + return findings diff --git a/tests/test_sequences.py b/tests/test_sequences.py new file mode 100644 index 0000000..74ef055 --- /dev/null +++ b/tests/test_sequences.py @@ -0,0 +1,44 @@ +import unittest + +from fgai.logs import parse_log_line +from fgai.sequences import detect_sequences + + +class SequenceTests(unittest.TestCase): + def test_detects_generic_dns_network_auth_sequence(self): + events = [ + parse_log_line("timestamp=2026-06-25T10:00:00Z fgai_stream=Resolver srcip=10.0.0.5 query_domain=example.test"), + parse_log_line("timestamp=2026-06-25T10:02:00Z fgai_stream=Proxy srcip=10.0.0.5 dstip=203.0.113.10 dstport=443 action=accept"), + parse_log_line("timestamp=2026-06-25T10:05:00Z fgai_stream=Identity srcip=10.0.0.5 action=failed eventid=4625"), + ] + + result = detect_sequences(events) + + self.assertIn("10.0.0.5", result) + finding = result["10.0.0.5"][0] + self.assertEqual(finding["detector"], "dns_network_auth_sequence") + self.assertEqual(finding["sample_values"], ["Identity", "Proxy", "Resolver"]) + self.assertEqual([item["value"] for item in finding["sample_events"]], ["dns_query", "network_connection", "auth_failure"]) + + def test_does_not_match_sequence_outside_window(self): + events = [ + parse_log_line("timestamp=2026-06-25T10:00:00Z fgai_stream=Resolver srcip=10.0.0.5 query_domain=example.test"), + parse_log_line("timestamp=2026-06-25T10:02:00Z fgai_stream=Proxy srcip=10.0.0.5 dstip=203.0.113.10 dstport=443 action=accept"), + parse_log_line("timestamp=2026-06-25T11:00:00Z fgai_stream=Identity srcip=10.0.0.5 action=failed eventid=4625"), + ] + + self.assertEqual(detect_sequences(events, window_seconds=900), {}) + + def test_supports_custom_generic_patterns(self): + events = [ + parse_log_line("timestamp=2026-06-25T10:00:00Z fgai_stream=Proxy srcip=10.0.0.5 dstip=203.0.113.10 action=accept"), + parse_log_line("timestamp=2026-06-25T10:01:00Z fgai_stream=Firewall srcip=10.0.0.5 action=deny"), + ] + + result = detect_sequences(events, patterns={"connection_then_deny": ("network_connection", "deny_action")}) + + self.assertEqual(result["10.0.0.5"][0]["detector"], "connection_then_deny") + + +if __name__ == "__main__": + unittest.main()