From 868022008a27ad2b371ef64488511c17bddbf9d17c9355ec97d118956e3c583d Mon Sep 17 00:00:00 2001 From: larssand Date: Wed, 24 Jun 2026 19:16:16 +0200 Subject: [PATCH] Implemented the next multi-source detection layer in this repository. --- README.md | 16 +++++- src/fgai/baseline.py | 103 ++++++++++++++++++++++++----------- src/fgai/correlation.py | 32 ++++++----- src/fgai/dashboard.py | 20 +++++-- src/fgai/entities.py | 59 ++++++++++++++++++++ src/fgai/feedback.py | 4 +- src/fgai/graylog_source.py | 5 +- src/fgai/incidents.py | 55 ++++++++++++++++--- src/fgai/llm.py | 1 + src/fgai/metrics.py | 4 ++ src/fgai/monitor.py | 17 +++++- tests/test_baseline.py | 12 ++++ tests/test_correlation.py | 9 +++ tests/test_graylog_source.py | 6 ++ tests/test_incidents.py | 5 ++ 15 files changed, 281 insertions(+), 67 deletions(-) create mode 100644 src/fgai/entities.py diff --git a/README.md b/README.md index 9627b33..9d1ae56 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,20 @@ The dashboard and Ollama then correlate behavior across sources, for example a client IP appearing in FortiGate, AdGuard/DNS, Windows Security, Nginx, Squid, VPN, or Proxmox. +Correlation is entity-aware rather than FortiGate-specific. SignalScope recognizes +common IP fields such as `srcip`, `source_ip`, `remote_addr`, and Windows event +IP fields; account fields such as `username`, `user`, and `TargetUserName`; and +host fields such as `hostname`, `computer`, and `winlog_computer_name`. Configure +the exact entity field per stream in the profile when your Graylog schema differs. + +Each profile baseline is stored per stream, entity, selected field, and five-minute +bucket. Once enough history exists, SignalScope compares the current rate or +numeric value to the same UTC weekday/hour where possible, then falls back to the +stream's overall history. Repeated MCP pages are fingerprinted so the same +Graylog event is not learned repeatedly. Related anomalies, profile deviations, +and multi-stream correlations are grouped into investigation incidents with a +compact evidence timeline. + The current MCP endpoint is `http://:9000/api/mcp`. Enable it in Graylog under `System -> Configurations -> MCP` and use stream IDs internally; the fgAI stream picker resolves titles in the UI. @@ -77,7 +91,7 @@ http://127.0.0.1:8088/metrics ``` This endpoint is passive and has no Prometheus or Grafana dependency. It reports -low-cardinality event counts, anomaly severities, baseline readiness, and Graylog +low-cardinality event counts, anomaly severities, incident counts, baseline readiness, and Graylog MCP health. Use it later as a Prometheus scrape target or as input for a Checkmk local check. Do not use source IPs, domains, or raw event IDs as metric labels. diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index fde084a..97123c2 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -4,11 +4,13 @@ import hashlib import sqlite3 import time from collections import Counter, defaultdict +from datetime import datetime, timezone from pathlib import Path from statistics import mean, pstdev from .logs import THREAT_ACTIONS, is_utm_event from .models import LogEvent +from .entities import profile_entity def _number(value: str | None) -> int: @@ -18,6 +20,19 @@ def _number(value: str | None) -> int: return 0 +def _event_epoch(event: LogEvent, fallback: int) -> int: + value = event.fields.get("eventtime", event.fields.get("timestamp", "")) + if not value: + return fallback + try: + return int(float(value)) + except ValueError: + try: + return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()) + except ValueError: + return fallback + + class BaselineStore: """Persistent five-minute behavior baseline, implemented with stdlib SQLite.""" @@ -48,6 +63,13 @@ class BaselineStore: stream_id text not null, entity text not null, field text not null, value text not null, seen_count integer not null, primary key (stream_id, entity, field, value) ); + create table if not exists profile_seen_events (fingerprint text primary key); + create table if not exists profile_temporal_buckets ( + stream_id text not null, entity text not null, field text not null, + weekday integer not null, hour integer not null, bucket_start integer not null, + events integer not null, numeric_sum real not null, numeric_sum_squares real not null, + primary key (stream_id, entity, field, weekday, hour, bucket_start) + ); """ ) @@ -92,34 +114,46 @@ class BaselineStore: def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int: observed_at = observed_at or int(time.time()) - bucket = observed_at - (observed_at % self.bucket_seconds) - pending: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0, 0.0]) + pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0]) + temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0]) pending_values: Counter[tuple[str, str, str, str]] = Counter() - for event in events: - stream_id = event.fields.get("fgai_stream_id", "") - profile = profiles.get(stream_id) - if not profile: - continue - entity_field = str(getattr(profile, "entity_field", "")).lower() - entity = event.fields.get(entity_field) - if not entity: - continue - fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())] - numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())} - for field in fields: - key = (stream_id, entity, str(field).lower()) - value = _number(event.fields.get(key[2])) if key[2] in numeric else 0 - pending[key][0] += 1 - pending[key][1] += value - pending[key][2] += value * value - if key[2] not in numeric: - raw_value = event.fields.get(key[2]) - if raw_value: - pending_values[(*key, raw_value)] += 1 with self._connect() as connection: - for (stream_id, entity, field), values in pending.items(): + for event in events: + stream_id = event.fields.get("fgai_stream_id", "") + profile = profiles.get(stream_id) + if not profile: + continue + fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest() + if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1: + continue + entity = profile_entity(event, str(getattr(profile, "entity_field", ""))) + if not entity: + continue + timestamp = _event_epoch(event, observed_at) + bucket = timestamp - (timestamp % self.bucket_seconds) + moment = datetime.fromtimestamp(timestamp, tz=timezone.utc) + fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())] + numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())} + for field in fields: + key = (stream_id, entity, str(field).lower(), bucket) + value = _number(event.fields.get(key[2])) if key[2] in numeric else 0 + pending[key][0] += 1 + pending[key][1] += value + pending[key][2] += value * value + temporal = (*key[:3], moment.weekday(), moment.hour, bucket) + temporal_pending[temporal][0] += 1 + temporal_pending[temporal][1] += value + temporal_pending[temporal][2] += value * value + if key[2] not in numeric: + raw_value = event.fields.get(key[2]) + if raw_value: + pending_values[(*key[:3], raw_value)] += 1 + for (stream_id, entity, field, bucket), values in pending.items(): connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?) on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values)) + for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items(): + connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values)) for key, count in pending_values.items(): connection.execute("""insert into profile_values values (?, ?, ?, ?, ?) on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count""", (*key, count)) @@ -131,7 +165,7 @@ class BaselineStore: profile = profiles.get(event.fields.get("fgai_stream_id", "")) if not profile: continue - entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower()) + entity = profile_entity(event, str(getattr(profile, "entity_field", ""))) if not entity: continue numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())} @@ -143,20 +177,26 @@ class BaselineStore: output: dict[str, list[dict[str, object]]] = defaultdict(list) with self._connect() as connection: for (stream, entity, field), values in current.items(): - rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall() + matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)] + current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time()) + moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc) + rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall() + baseline_scope = "same weekday/hour" + if len(rows) < 12: + rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall() + baseline_scope = "all observed periods" if len(rows) < 12: continue if values[1] == 0: history = [row[0] for row in rows] current_value = values[0] - reason = f"{field} event rate deviates from its stream baseline" + reason = f"{field} event rate deviates from its {baseline_scope} baseline" else: history = [row[1] / row[0] if row[0] else 0 for row in rows] current_value = values[1] / values[0] if values[0] else 0 - reason = f"{field} value deviates from its stream baseline" + reason = f"{field} value deviates from its {baseline_scope} baseline" deviation = abs(current_value - mean(history)) if deviation > (pstdev(history) or 1.0) * 3: - matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and event.fields.get(str(getattr(profiles.get(stream), "entity_field", "")).lower()) == entity and event.fields.get(field)] samples = sorted({event.fields.get(field, "") for event in matching})[:5] evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]] output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": samples, "sample_events": evidence_events}) @@ -165,7 +205,7 @@ class BaselineStore: profile = profiles.get(event.fields.get("fgai_stream_id", "")) if not profile: continue - entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower()) + entity = profile_entity(event, str(getattr(profile, "entity_field", ""))) if not entity: continue stream = event.fields.get("fgai_stream_id", "") @@ -177,7 +217,8 @@ class BaselineStore: known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone() known_total = connection.execute("select count(*) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0] if known is None and known_total >= 10: - evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value} + samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value] + evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]} if evidence not in output[entity]: output[entity].append(evidence) return output diff --git a/src/fgai/correlation.py b/src/fgai/correlation.py index 4b39fba..6a1a476 100644 --- a/src/fgai/correlation.py +++ b/src/fgai/correlation.py @@ -2,29 +2,33 @@ from __future__ import annotations from collections import defaultdict +from .entities import event_entities, sample_timeline from .logs import THREAT_ACTIONS, is_utm_event from .models import LogEvent def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]: - grouped: dict[str, list[LogEvent]] = defaultdict(list) + """Correlate IPs, users, and hosts across independently configured Graylog streams.""" + grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list) for event in events: - if event.src_ip: - grouped[event.src_ip].append(event) + for identity in event_entities(event): + grouped[(identity["entity_type"], identity["entity"])].append(event) correlations = [] - for source_ip, source_events in grouped.items(): + for (kind, entity), source_events in grouped.items(): streams = sorted({event.fields.get("fgai_stream", "local_syslog") for event in source_events}) if len(streams) < 2: continue threat_events = sum(event.action in THREAT_ACTIONS or is_utm_event(event) for event in source_events) - samples = [ - { - "stream": event.fields.get("fgai_stream", "local_syslog"), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), - "type": event.fields.get("type", ""), "subtype": event.subtype, "action": event.action, - "severity": event.severity, "destination": event.dst_ip or "", "service": event.fields.get("service", ""), - "context": event.fields.get("query_domain", event.fields.get("qh", event.fields.get("message", ""))), - } - for event in source_events[:20] - ] - correlations.append({"source_ip": source_ip, "streams": streams, "events": len(source_events), "security_events": threat_events, "samples": samples}) + samples = sample_timeline(source_events) + correlations.append({ + "entity": entity, + "entity_type": kind, + "source_ip": entity if kind == "ip" else "", + "streams": streams, + "events": len(source_events), + "security_events": threat_events, + "samples": samples, + "first_seen": samples[0]["timestamp"] if samples else "", + "last_seen": samples[-1]["timestamp"] if samples else "", + }) return sorted(correlations, key=lambda item: (int(item["security_events"]), int(item["events"])), reverse=True)[:limit] diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 5499549..6e46931 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -74,7 +74,7 @@ HTML = """ -

Events and Anomalies

Baseline and Stream Health

AI Assessment

LLM assessment disabled.

Anomalies

Recommendations

+

Events and Anomalies

Baseline and Stream Health

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

Runtime Configuration

@@ -151,27 +151,35 @@ async function refresh() { {label:'Policies', render:r => esc((r.related_policy_ids || []).join(', '))}, {label:'Services', render:r => esc((r.related_services || []).join(', '))} ]); + document.getElementById('incidents').innerHTML = table(data.incidents || [], [ + {label:'Entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)}, + {label:'Score', key:'score'}, + {label:'Severity', render:r => `${esc(r.severity)}`}, + {label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')}, + {label:'Evidence', render:r => esc((r.evidence || []).join('; '))}, + {label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('
'); return rows ? `
${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}

${rows}

` : '-'; }} + ]); document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [ {label:'Source', key:'src_ip'}, {label:'Score', key:'score'}, {label:'Reasons', render:r => esc((r.reasons || []).join('; '))} ]); const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel})); - const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({source_ip: correlation.source_ip, ...sample}))); + const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample}))); const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id])); const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item}))); document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [ - {label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('
'); return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} + {label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('
'); return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} ]); document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => { const note = prompt('Review note (optional):') || ''; const days = prompt('Expiry in days (0 = no expiry):', '0') || '0'; - const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})}); + const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, value:button.dataset.value, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})}); document.getElementById('feedbackNotice').textContent = response.ok ? 'Review saved. The matching pattern will be labeled on the next refresh.' : 'Could not save review.'; refresh(); })); document.getElementById('relatedActivity').innerHTML = table(relatedRows, [ - {label:'Source IP', key:'source_ip'}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'}, + {label:'Entity', render:r => esc(r.entity || r.source_ip)}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'} ]); @@ -195,7 +203,7 @@ async function refresh() { const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id})); const correlations = data.cross_source_correlations || []; document.getElementById('diagnostics').innerHTML = - '

Cross-Source Correlations

' + table(correlations, [{label:'Source IP', key:'source_ip'}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) + + '

Cross-Source Correlations

' + table(correlations, [{label:'Entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) + '

Entities

' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) + '

Profile Baseline Readiness

' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', render:r => r.ready ? 'ready' : 'learning'}]) + '

Data Quality

' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) + diff --git a/src/fgai/entities.py b/src/fgai/entities.py new file mode 100644 index 0000000..b15cea0 --- /dev/null +++ b/src/fgai/entities.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import ipaddress +from collections.abc import Iterable + +from .models import LogEvent + + +ENTITY_FIELDS: dict[str, tuple[str, ...]] = { + "ip": ("srcip", "src_ip", "source_ip", "client_ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "event_data_ipaddress"), + "user": ("username", "user", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog_event_data_subjectusername"), + "host": ("hostname", "host", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "agent_name"), +} + + +def entity_type(value: str) -> str: + try: + ipaddress.ip_address(value) + return "ip" + except ValueError: + return "entity" + + +def event_entities(event: LogEvent) -> list[dict[str, str]]: + """Return normalized identities shared across network, endpoint, DNS, and web logs.""" + identities: list[dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + for kind, fields in ENTITY_FIELDS.items(): + for field in fields: + value = str(event.fields.get(field, "")).strip() + if not value or value in {"-", "unknown", "n/a"}: + continue + key = (kind, value.lower() if kind != "ip" else value) + if key not in seen: + seen.add(key) + identities.append({"entity": value, "entity_type": kind, "field": field}) + return identities + + +def profile_entity(event: LogEvent, field: str) -> str: + return str(event.fields.get(field.lower(), "")).strip() + + +def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]: + samples = [ + { + "stream": event.fields.get("fgai_stream", "local_syslog"), + "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), + "type": event.fields.get("type", ""), + "subtype": event.subtype, + "action": event.action, + "severity": event.severity, + "destination": event.dst_ip or event.fields.get("query_domain", event.fields.get("url", "")), + "service": event.fields.get("service", event.fields.get("query_type", "")), + "context": event.fields.get("query_domain", event.fields.get("qh", event.fields.get("url", event.fields.get("message", event.fields.get("msg", "")))))[:240], + } + for event in events + ] + return sorted(samples, key=lambda item: item["timestamp"])[-limit:] diff --git a/src/fgai/feedback.py b/src/fgai/feedback.py index 3c02283..3c7f312 100644 --- a/src/fgai/feedback.py +++ b/src/fgai/feedback.py @@ -21,7 +21,7 @@ class FeedbackStore: status = str(item.get("status", "")).lower() if status not in {"false_positive", "expected", "confirmed"}: raise ValueError("invalid feedback status") - entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)} - entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field")) != (entry["stream_id"], entry["entity"], entry["field"])] + entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "value": str(item.get("value", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)} + entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field"), value.get("value", "")) != (entry["stream_id"], entry["entity"], entry["field"], entry["value"])] entries.append(entry); self.path.parent.mkdir(parents=True, exist_ok=True); self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8") return entry diff --git a/src/fgai/graylog_source.py b/src/fgai/graylog_source.py index e84584b..2c82a33 100644 --- a/src/fgai/graylog_source.py +++ b/src/fgai/graylog_source.py @@ -41,11 +41,12 @@ def _records(value: object) -> Iterable[dict[str, object]]: class GraylogStreamSource: - def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "") -> None: + def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "", profile_fields: tuple[str, ...] = ()) -> None: self.client = client self.stream = stream self.query = query or "*" self.stream_label = stream_label or stream + self.profile_fields = tuple(str(field).lower() for field in profile_fields if field) try: self.mapping = json.loads(field_mapping) if field_mapping else {} except json.JSONDecodeError as exc: @@ -59,7 +60,7 @@ class GraylogStreamSource: arguments: dict[str, object] = { "query": self.query, "range_seconds": 300, - "fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields])), + "fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields, *self.profile_fields])), } if self.stream: arguments["streams"] = [self.stream] diff --git a/src/fgai/incidents.py b/src/fgai/incidents.py index f48d2de..e2d1b4a 100644 --- a/src/fgai/incidents.py +++ b/src/fgai/incidents.py @@ -1,17 +1,54 @@ from __future__ import annotations +from collections import defaultdict + +from .entities import entity_type from .models import AnomalyFinding def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]: - correlation_by_ip = {str(item.get("source_ip")): item for item in correlations} - incidents = [] + """Build investigation units from any log source, not only network source IPs.""" + groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []}) for anomaly in anomalies: - fields = field_deviations.get(anomaly.subject, []) - correlation = correlation_by_ip.get(anomaly.subject) - evidence = [*anomaly.reasons, *[str(item.get("reason", "")) for item in fields]] - score = min(100, anomaly.score + min(15, sum(int(item.get("score", 0)) for item in fields))) - if correlation: - score = min(100, score + 10); evidence.append(f"observed across {len(correlation.get('streams', []))} streams") - incidents.append({"entity": anomaly.subject, "score": score, "severity": anomaly.severity, "evidence": evidence[:8], "field_deviations": len(fields), "correlated_streams": correlation.get("streams", []) if correlation else []}) + group = groups[anomaly.subject] + group["score"] = max(int(group["score"]), anomaly.score) + group["evidence"].extend(anomaly.reasons) + for entity, deviations in field_deviations.items(): + group = groups[entity] + group["fields"].extend(deviations) + active = [item for item in deviations if item.get("feedback") not in {"expected", "false_positive"}] + group["score"] = min(100, int(group["score"]) + min(30, sum(int(item.get("score", 0)) for item in active))) + group["evidence"].extend(str(item.get("reason", "")) for item in active) + for item in active: + group["timeline"].extend(item.get("sample_events", [])) + for correlation in correlations: + entity = str(correlation.get("entity") or correlation.get("source_ip") or "") + if not entity: + continue + group = groups[entity] + group["correlations"].append(correlation) + group["score"] = min(100, int(group["score"]) + 10 + min(15, int(correlation.get("security_events", 0)) * 2)) + group["evidence"].append(f"observed across {len(correlation.get('streams', []))} streams") + group["timeline"].extend(correlation.get("samples", [])) + + incidents = [] + for entity, group in groups.items(): + if not group["evidence"]: + continue + score = int(group["score"]) + severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low" + streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")}) + timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20] + incidents.append({ + "entity": entity, + "entity_type": entity_type(entity), + "score": score, + "severity": severity, + "evidence": list(dict.fromkeys(str(item) for item in group["evidence"] if item))[:8], + "field_deviations": len(group["fields"]), + "correlated_streams": streams, + "timeline": timeline, + "first_seen": timeline[0].get("timestamp", "") if timeline else "", + "last_seen": timeline[-1].get("timestamp", "") if timeline else "", + }) return sorted(incidents, key=lambda item: int(item["score"]), reverse=True) diff --git a/src/fgai/llm.py b/src/fgai/llm.py index d51026b..2ea8793 100644 --- a/src/fgai/llm.py +++ b/src/fgai/llm.py @@ -63,6 +63,7 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None = "diagnostics": analysis.get("diagnostics", {}), "capabilities": analysis.get("capabilities", {}), "cross_source_correlations": analysis.get("cross_source_correlations", [])[:20], + "incidents": analysis.get("incidents", [])[:10], "field_deviations": analysis.get("field_deviations", {}), "feedback": analysis.get("feedback", []), } diff --git a/src/fgai/metrics.py b/src/fgai/metrics.py index 4ac5b3a..9d33ece 100644 --- a/src/fgai/metrics.py +++ b/src/fgai/metrics.py @@ -7,6 +7,7 @@ def prometheus_metrics(status: dict[str, object]) -> str: baseline = status.get("baseline", {}) if isinstance(status.get("baseline"), dict) else {} capabilities = status.get("capabilities", {}) if isinstance(status.get("capabilities"), dict) else {} mcp = capabilities.get("graylog_mcp", {}) if isinstance(capabilities.get("graylog_mcp"), dict) else {} + incidents = status.get("incidents", []) if isinstance(status.get("incidents"), list) else [] lines = ["# HELP fgai_events_total Events in the latest analysis window.", "# TYPE fgai_events_total gauge"] for key in ("total", "utm", "threat_actions", "critical_or_high"): lines.append(f'fgai_events_total{{kind="{key}"}} {int(summary.get(key, 0) or 0)}') @@ -16,4 +17,7 @@ def prometheus_metrics(status: dict[str, object]) -> str: lines += ["# HELP fgai_baseline_sources_ready Sources with sufficient baseline history.", "# TYPE fgai_baseline_sources_ready gauge", f'fgai_baseline_sources_ready {int(baseline.get("sources_ready", 0) or 0)}'] lines += ["# HELP fgai_graylog_mcp_connected Graylog MCP connectivity state.", "# TYPE fgai_graylog_mcp_connected gauge", f'fgai_graylog_mcp_connected {1 if mcp.get("status") == "connected" else 0}'] lines += ["# HELP fgai_graylog_events_fetched Events fetched from Graylog in the latest poll.", "# TYPE fgai_graylog_events_fetched gauge", f'fgai_graylog_events_fetched {int(mcp.get("events_fetched", 0) or 0)}'] + lines += ["# HELP signalscope_incidents Open incident candidates by severity.", "# TYPE signalscope_incidents gauge"] + for severity in ("critical", "high", "medium", "low"): + lines.append(f'signalscope_incidents{{severity="{severity}"}} {sum(1 for item in incidents if isinstance(item, dict) and item.get("severity") == severity)}') return "\n".join(lines) + "\n" diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index b9e1962..34d97a6 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -58,7 +58,14 @@ def build_status( events = [] for stream_config in stream_configs: stream_id = str(stream_config["id"]) - stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id))).fetch() + profile = stream_profiles.get(stream_id) + profile_fields = ( + str(getattr(profile, "entity_field", "")), + str(getattr(profile, "timestamp_field", "")), + *tuple(str(field) for field in getattr(profile, "categorical_fields", ())), + *tuple(str(field) for field in getattr(profile, "numeric_fields", ())), + ) if profile else () + stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id)), profile_fields).fetch() events.extend(stream_events) stream_statuses.append({"stream_id": stream_id, **stream_status}) mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)} @@ -71,7 +78,13 @@ def build_status( feedback = FeedbackStore().entries() for entity, deviations in field_deviations.items(): for deviation in deviations: - match = next((item for item in feedback if item.get("entity") == entity and item.get("stream_id") == deviation.get("stream_id") and item.get("field") == deviation.get("field")), None) + match = next(( + item for item in feedback + if item.get("entity") == entity + and item.get("stream_id") == deviation.get("stream_id") + and item.get("field") == deviation.get("field") + and (not item.get("value") or item.get("value") == deviation.get("value", "")) + ), None) if match: deviation["feedback"] = match["status"] if match["status"] in {"false_positive", "expected"}: diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 8d925e6..e00a86d 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -4,6 +4,7 @@ from pathlib import Path from fgai.baseline import BaselineStore from fgai.logs import parse_log_line +from fgai.stream_profiles import parse_profiles class BaselineTests(unittest.TestCase): @@ -15,3 +16,14 @@ class BaselineTests(unittest.TestCase): profiles = store.profiles({"10.0.0.1"}) self.assertEqual(profiles["10.0.0.1"]["samples"], 12) self.assertIn("1000", profiles["10.0.0.1"]["known_destination_ports"]) + + def test_profile_events_are_deduplicated_between_polls(self): + with tempfile.TemporaryDirectory() as directory: + store = BaselineStore(str(Path(directory) / "baseline.sqlite3")) + profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}]) + events = [ + parse_log_line(f"fgai_stream_id=windows username=alice action=login event={index}") + for index in range(12) + ] + self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000), 1) + self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_300), 0) diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 72b7f4d..13d5d1c 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -14,3 +14,12 @@ class CorrelationTests(unittest.TestCase): self.assertEqual(result[0]["source_ip"], "10.0.0.5") self.assertEqual(result[0]["streams"], ["DNS", "Fortigate"]) self.assertEqual(len(result[0]["samples"]), 2) + + def test_correlates_user_across_streams(self): + events = [ + parse_log_line("username=alice fgai_stream=Windows action=login timestamp=2026-06-24T10:00:00Z"), + parse_log_line("username=alice fgai_stream=VPN action=accept timestamp=2026-06-24T10:01:00Z"), + ] + result = correlate_source_ips(events) + self.assertEqual(result[0]["entity"], "alice") + self.assertEqual(result[0]["entity_type"], "user") diff --git a/tests/test_graylog_source.py b/tests/test_graylog_source.py index 268792e..660ed63 100644 --- a/tests/test_graylog_source.py +++ b/tests/test_graylog_source.py @@ -30,6 +30,12 @@ class GraylogSourceTests(unittest.TestCase): self.assertIn("client", client.arguments["fields"]) self.assertEqual(client.arguments["offset"], 0) + def test_requests_selected_profile_fields(self): + client = _Client() + GraylogStreamSource(client, "windows", profile_fields=("TargetUserName", "EventID")).fetch() + self.assertIn("targetusername", client.arguments["fields"]) + self.assertIn("eventid", client.arguments["fields"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_incidents.py b/tests/test_incidents.py index a7d187d..c383380 100644 --- a/tests/test_incidents.py +++ b/tests/test_incidents.py @@ -7,3 +7,8 @@ class IncidentTests(unittest.TestCase): anomaly = AnomalyFinding("10.0.0.1", 60, "high", "high", ["burst"], {}) result = build_incidents([anomaly], {"10.0.0.1": [{"score": 15, "reason": "new domain"}]}, [{"source_ip": "10.0.0.1", "streams": ["DNS", "Firewall"]}]) self.assertEqual(result[0]["score"], 85) + + def test_creates_incident_for_profile_entity_without_network_anomaly(self): + result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, []) + self.assertEqual(result[0]["entity"], "alice") + self.assertEqual(result[0]["field_deviations"], 1)