Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the next multi-source detection layer in this repository.

Browse Source
This commit is contained in:
larssand
2026-06-24 19:16:16 +02:00
parent f6bee0438c
commit 868022008a
15 changed files with 281 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@@ -64,6 +64,20 @@ The dashboard and Ollama then correlate behavior across sources, for example a
client IP appearing in FortiGate, AdGuard/DNS, Windows Security, Nginx, Squid,
VPN, or Proxmox.
Correlation is entity-aware rather than FortiGate-specific. SignalScope recognizes
common IP fields such as `srcip`, `source_ip`, `remote_addr`, and Windows event
IP fields; account fields such as `username`, `user`, and `TargetUserName`; and
host fields such as `hostname`, `computer`, and `winlog_computer_name`. Configure
the exact entity field per stream in the profile when your Graylog schema differs.
Each profile baseline is stored per stream, entity, selected field, and five-minute
bucket. Once enough history exists, SignalScope compares the current rate or
numeric value to the same UTC weekday/hour where possible, then falls back to the
stream's overall history. Repeated MCP pages are fingerprinted so the same
Graylog event is not learned repeatedly. Related anomalies, profile deviations,
and multi-stream correlations are grouped into investigation incidents with a
compact evidence timeline.
The current MCP endpoint is `http://<graylog-host>:9000/api/mcp`. Enable it in
Graylog under `System -> Configurations -> MCP` and use stream IDs internally;
the fgAI stream picker resolves titles in the UI.
@@ -77,7 +91,7 @@ http://127.0.0.1:8088/metrics
```
This endpoint is passive and has no Prometheus or Grafana dependency. It reports
low-cardinality event counts, anomaly severities, baseline readiness, and Graylog
low-cardinality event counts, anomaly severities, incident counts, baseline readiness, and Graylog
MCP health. Use it later as a Prometheus scrape target or as input for a Checkmk
local check. Do not use source IPs, domains, or raw event IDs as metric labels.

69
src/fgai/baseline.py
View File

@@ -4,11 +4,13 @@ import hashlib
import sqlite3
import time
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from statistics import mean, pstdev
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
from .entities import profile_entity
def _number(value: str | None) -> int:
@@ -18,6 +20,19 @@ def _number(value: str | None) -> int:
return 0
def _event_epoch(event: LogEvent, fallback: int) -> int:
value = event.fields.get("eventtime", event.fields.get("timestamp", ""))
if not value:
return fallback
try:
return int(float(value))
except ValueError:
try:
return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
except ValueError:
return fallback
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -48,6 +63,13 @@ class BaselineStore:
stream_id text not null, entity text not null, field text not null, value text not null,
seen_count integer not null, primary key (stream_id, entity, field, value)
);
create table if not exists profile_seen_events (fingerprint text primary key);
create table if not exists profile_temporal_buckets (
stream_id text not null, entity text not null, field text not null,
weekday integer not null, hour integer not null, bucket_start integer not null,
events integer not null, numeric_sum real not null, numeric_sum_squares real not null,
primary key (stream_id, entity, field, weekday, hour, bucket_start)
);
"""
)
@@ -92,34 +114,46 @@ class BaselineStore:
def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int:
observed_at = observed_at or int(time.time())
bucket = observed_at - (observed_at % self.bucket_seconds)
pending: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0])
pending_values: Counter[tuple[str, str, str, str]] = Counter()
with self._connect() as connection:
for event in events:
stream_id = event.fields.get("fgai_stream_id", "")
profile = profiles.get(stream_id)
if not profile:
continue
entity_field = str(getattr(profile, "entity_field", "")).lower()
entity = event.fields.get(entity_field)
fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1:
continue
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
continue
timestamp = _event_epoch(event, observed_at)
bucket = timestamp - (timestamp % self.bucket_seconds)
moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in fields:
key = (stream_id, entity, str(field).lower())
key = (stream_id, entity, str(field).lower(), bucket)
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
pending[key][0] += 1
pending[key][1] += value
pending[key][2] += value * value
temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
temporal_pending[temporal][0] += 1
temporal_pending[temporal][1] += value
temporal_pending[temporal][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
pending_values[(*key, raw_value)] += 1
with self._connect() as connection:
for (stream_id, entity, field), values in pending.items():
pending_values[(*key[:3], raw_value)] += 1
for (stream_id, entity, field, bucket), values in pending.items():
connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items():
connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values))
for key, count in pending_values.items():
connection.execute("""insert into profile_values values (?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count""", (*key, count))
@@ -131,7 +165,7 @@ class BaselineStore:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
continue
entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
continue
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
@@ -143,20 +177,26 @@ class BaselineStore:
output: dict[str, list[dict[str, object]]] = defaultdict(list)
with self._connect() as connection:
for (stream, entity, field), values in current.items():
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
baseline_scope = "same weekday/hour"
if len(rows) < 12:
rows = connection.execute("select events, numeric_sum from profile_buckets where stream_id=? and entity=? and field=? order by bucket_start desc limit 25", (stream, entity, field)).fetchall()
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
if values[1] == 0:
history = [row[0] for row in rows]
current_value = values[0]
reason = f"{field} event rate deviates from its stream baseline"
reason = f"{field} event rate deviates from its {baseline_scope} baseline"
else:
history = [row[1] / row[0] if row[0] else 0 for row in rows]
current_value = values[1] / values[0] if values[0] else 0
reason = f"{field} value deviates from its stream baseline"
reason = f"{field} value deviates from its {baseline_scope} baseline"
deviation = abs(current_value - mean(history))
if deviation > (pstdev(history) or 1.0) * 3:
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and event.fields.get(str(getattr(profiles.get(stream), "entity_field", "")).lower()) == entity and event.fields.get(field)]
samples = sorted({event.fields.get(field, "") for event in matching})[:5]
evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"field": field, "stream_id": stream, "score": 15, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": samples, "sample_events": evidence_events})
@@ -165,7 +205,7 @@ class BaselineStore:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
continue
entity = event.fields.get(str(getattr(profile, "entity_field", "")).lower())
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
continue
stream = event.fields.get("fgai_stream_id", "")
@@ -177,7 +217,8 @@ class BaselineStore:
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
known_total = connection.execute("select count(*) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 10:
evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value}
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
evidence = {"field": field, "stream_id": stream, "score": 12, "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
return output

32
src/fgai/correlation.py
View File

@@ -2,29 +2,33 @@ from __future__ import annotations
from collections import defaultdict
from .entities import event_entities, sample_timeline
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
grouped: dict[str, list[LogEvent]] = defaultdict(list)
"""Correlate IPs, users, and hosts across independently configured Graylog streams."""
grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
for event in events:
if event.src_ip:
grouped[event.src_ip].append(event)
for identity in event_entities(event):
grouped[(identity["entity_type"], identity["entity"])].append(event)
correlations = []
for source_ip, source_events in grouped.items():
for (kind, entity), source_events in grouped.items():
streams = sorted({event.fields.get("fgai_stream", "local_syslog") for event in source_events})
if len(streams) < 2:
continue
threat_events = sum(event.action in THREAT_ACTIONS or is_utm_event(event) for event in source_events)
samples = [
{
"stream": event.fields.get("fgai_stream", "local_syslog"), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
"type": event.fields.get("type", ""), "subtype": event.subtype, "action": event.action,
"severity": event.severity, "destination": event.dst_ip or "", "service": event.fields.get("service", ""),
"context": event.fields.get("query_domain", event.fields.get("qh", event.fields.get("message", ""))),
}
for event in source_events[:20]
]
correlations.append({"source_ip": source_ip, "streams": streams, "events": len(source_events), "security_events": threat_events, "samples": samples})
samples = sample_timeline(source_events)
correlations.append({
"entity": entity,
"entity_type": kind,
"source_ip": entity if kind == "ip" else "",
"streams": streams,
"events": len(source_events),
"security_events": threat_events,
"samples": samples,
"first_seen": samples[0]["timestamp"] if samples else "",
"last_seen": samples[-1]["timestamp"] if samples else "",
})
return sorted(correlations, key=lambda item: (int(item["security_events"]), int(item["events"])), reverse=True)[:limit]

20
src/fgai/dashboard.py
View File

@@ -74,7 +74,7 @@ HTML = """<!doctype html>
</div>
</section>
<nav class="tabs" aria-label="Dashboard views"><button class="tab active" data-tab="overview">Overview</button><button class="tab" data-tab="findings">Findings</button><button class="tab" data-tab="diagnostics">Diagnostics</button><button class="tab" data-tab="settings">Settings</button></nav>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
@@ -151,27 +151,35 @@ async function refresh() {
{label:'Policies', render:r => esc((r.related_policy_ids || []).join(', '))},
{label:'Services', render:r => esc((r.related_services || []).join(', '))}
]);
document.getElementById('incidents').innerHTML = table(data.incidents || [], [
{label:'Entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)},
{label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')},
{label:'Evidence', render:r => esc((r.evidence || []).join('; '))},
{label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('<br>'); return rows ? `<details><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }}
]);
document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
{label:'Source', key:'src_ip'},
{label:'Score', key:'score'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({source_ip: correlation.source_ip, ...sample})));
const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item})));
document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark confirmed</button></div>`}
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
]);
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Review note (optional):') || '';
const days = prompt('Expiry in days (0 = no expiry):', '0') || '0';
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, value:button.dataset.value, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
document.getElementById('feedbackNotice').textContent = response.ok ? 'Review saved. The matching pattern will be labeled on the next refresh.' : 'Could not save review.';
refresh();
}));
document.getElementById('relatedActivity').innerHTML = table(relatedRows, [
{label:'Source IP', key:'source_ip'}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
{label:'Entity', render:r => esc(r.entity || r.source_ip)}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
{label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'},
{label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'}
]);
@@ -195,7 +203,7 @@ async function refresh() {
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id}));
const correlations = data.cross_source_correlations || [];
document.getElementById('diagnostics').innerHTML =
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Source IP', key:'source_ip'}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', render:r => r.ready ? 'ready' : 'learning'}]) +
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +

59
src/fgai/entities.py Normal file
View File

@@ -0,0 +1,59 @@
from __future__ import annotations
import ipaddress
from collections.abc import Iterable
from .models import LogEvent
ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
"ip": ("srcip", "src_ip", "source_ip", "client_ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "event_data_ipaddress"),
"user": ("username", "user", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog_event_data_subjectusername"),
"host": ("hostname", "host", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "agent_name"),
}
def entity_type(value: str) -> str:
try:
ipaddress.ip_address(value)
return "ip"
except ValueError:
return "entity"
def event_entities(event: LogEvent) -> list[dict[str, str]]:
"""Return normalized identities shared across network, endpoint, DNS, and web logs."""
identities: list[dict[str, str]] = []
seen: set[tuple[str, str]] = set()
for kind, fields in ENTITY_FIELDS.items():
for field in fields:
value = str(event.fields.get(field, "")).strip()
if not value or value in {"-", "unknown", "n/a"}:
continue
key = (kind, value.lower() if kind != "ip" else value)
if key not in seen:
seen.add(key)
identities.append({"entity": value, "entity_type": kind, "field": field})
return identities
def profile_entity(event: LogEvent, field: str) -> str:
return str(event.fields.get(field.lower(), "")).strip()
def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]:
samples = [
{
"stream": event.fields.get("fgai_stream", "local_syslog"),
"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
"type": event.fields.get("type", ""),
"subtype": event.subtype,
"action": event.action,
"severity": event.severity,
"destination": event.dst_ip or event.fields.get("query_domain", event.fields.get("url", "")),
"service": event.fields.get("service", event.fields.get("query_type", "")),
"context": event.fields.get("query_domain", event.fields.get("qh", event.fields.get("url", event.fields.get("message", event.fields.get("msg", "")))))[:240],
}
for event in events
]
return sorted(samples, key=lambda item: item["timestamp"])[-limit:]

4
src/fgai/feedback.py
View File

@@ -21,7 +21,7 @@ class FeedbackStore:
status = str(item.get("status", "")).lower()
if status not in {"false_positive", "expected", "confirmed"}:
raise ValueError("invalid feedback status")
entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)}
entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field")) != (entry["stream_id"], entry["entity"], entry["field"])]
entry = {"stream_id": str(item.get("stream_id", "")), "entity": str(item.get("entity", "")), "field": str(item.get("field", "")), "value": str(item.get("value", "")), "status": status, "note": str(item.get("note", "")), "created_at": int(time.time()), "expires_at": int(item.get("expires_at", 0) or 0)}
entries = [value for value in self.entries() if (value.get("stream_id"), value.get("entity"), value.get("field"), value.get("value", "")) != (entry["stream_id"], entry["entity"], entry["field"], entry["value"])]
entries.append(entry); self.path.parent.mkdir(parents=True, exist_ok=True); self.path.write_text(json.dumps(entries, indent=2), encoding="utf-8")
return entry

5
src/fgai/graylog_source.py
View File

@@ -41,11 +41,12 @@ def _records(value: object) -> Iterable[dict[str, object]]:
class GraylogStreamSource:
def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "") -> None:
def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "", profile_fields: tuple[str, ...] = ()) -> None:
self.client = client
self.stream = stream
self.query = query or "*"
self.stream_label = stream_label or stream
self.profile_fields = tuple(str(field).lower() for field in profile_fields if field)
try:
self.mapping = json.loads(field_mapping) if field_mapping else {}
except json.JSONDecodeError as exc:
@@ -59,7 +60,7 @@ class GraylogStreamSource:
arguments: dict[str, object] = {
"query": self.query,
"range_seconds": 300,
"fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields])),
"fields": list(dict.fromkeys([*DEFAULT_FIELDS, *mapping_fields, *self.profile_fields])),
}
if self.stream:
arguments["streams"] = [self.stream]

55
src/fgai/incidents.py
View File

@@ -1,17 +1,54 @@
from __future__ import annotations
from collections import defaultdict
from .entities import entity_type
from .models import AnomalyFinding
def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]:
correlation_by_ip = {str(item.get("source_ip")): item for item in correlations}
incidents = []
"""Build investigation units from any log source, not only network source IPs."""
groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []})
for anomaly in anomalies:
fields = field_deviations.get(anomaly.subject, [])
correlation = correlation_by_ip.get(anomaly.subject)
evidence = [*anomaly.reasons, *[str(item.get("reason", "")) for item in fields]]
score = min(100, anomaly.score + min(15, sum(int(item.get("score", 0)) for item in fields)))
if correlation:
score = min(100, score + 10); evidence.append(f"observed across {len(correlation.get('streams', []))} streams")
incidents.append({"entity": anomaly.subject, "score": score, "severity": anomaly.severity, "evidence": evidence[:8], "field_deviations": len(fields), "correlated_streams": correlation.get("streams", []) if correlation else []})
group = groups[anomaly.subject]
group["score"] = max(int(group["score"]), anomaly.score)
group["evidence"].extend(anomaly.reasons)
for entity, deviations in field_deviations.items():
group = groups[entity]
group["fields"].extend(deviations)
active = [item for item in deviations if item.get("feedback") not in {"expected", "false_positive"}]
group["score"] = min(100, int(group["score"]) + min(30, sum(int(item.get("score", 0)) for item in active)))
group["evidence"].extend(str(item.get("reason", "")) for item in active)
for item in active:
group["timeline"].extend(item.get("sample_events", []))
for correlation in correlations:
entity = str(correlation.get("entity") or correlation.get("source_ip") or "")
if not entity:
continue
group = groups[entity]
group["correlations"].append(correlation)
group["score"] = min(100, int(group["score"]) + 10 + min(15, int(correlation.get("security_events", 0)) * 2))
group["evidence"].append(f"observed across {len(correlation.get('streams', []))} streams")
group["timeline"].extend(correlation.get("samples", []))
incidents = []
for entity, group in groups.items():
if not group["evidence"]:
continue
score = int(group["score"])
severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")})
timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
incidents.append({
"entity": entity,
"entity_type": entity_type(entity),
"score": score,
"severity": severity,
"evidence": list(dict.fromkeys(str(item) for item in group["evidence"] if item))[:8],
"field_deviations": len(group["fields"]),
"correlated_streams": streams,
"timeline": timeline,
"first_seen": timeline[0].get("timestamp", "") if timeline else "",
"last_seen": timeline[-1].get("timestamp", "") if timeline else "",
})
return sorted(incidents, key=lambda item: int(item["score"]), reverse=True)

1
src/fgai/llm.py
View File

@@ -63,6 +63,7 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
"diagnostics": analysis.get("diagnostics", {}),
"capabilities": analysis.get("capabilities", {}),
"cross_source_correlations": analysis.get("cross_source_correlations", [])[:20],
"incidents": analysis.get("incidents", [])[:10],
"field_deviations": analysis.get("field_deviations", {}),
"feedback": analysis.get("feedback", []),
}

4
src/fgai/metrics.py
View File

@@ -7,6 +7,7 @@ def prometheus_metrics(status: dict[str, object]) -> str:
baseline = status.get("baseline", {}) if isinstance(status.get("baseline"), dict) else {}
capabilities = status.get("capabilities", {}) if isinstance(status.get("capabilities"), dict) else {}
mcp = capabilities.get("graylog_mcp", {}) if isinstance(capabilities.get("graylog_mcp"), dict) else {}
incidents = status.get("incidents", []) if isinstance(status.get("incidents"), list) else []
lines = ["# HELP fgai_events_total Events in the latest analysis window.", "# TYPE fgai_events_total gauge"]
for key in ("total", "utm", "threat_actions", "critical_or_high"):
lines.append(f'fgai_events_total{{kind="{key}"}} {int(summary.get(key, 0) or 0)}')
@@ -16,4 +17,7 @@ def prometheus_metrics(status: dict[str, object]) -> str:
lines += ["# HELP fgai_baseline_sources_ready Sources with sufficient baseline history.", "# TYPE fgai_baseline_sources_ready gauge", f'fgai_baseline_sources_ready {int(baseline.get("sources_ready", 0) or 0)}']
lines += ["# HELP fgai_graylog_mcp_connected Graylog MCP connectivity state.", "# TYPE fgai_graylog_mcp_connected gauge", f'fgai_graylog_mcp_connected {1 if mcp.get("status") == "connected" else 0}']
lines += ["# HELP fgai_graylog_events_fetched Events fetched from Graylog in the latest poll.", "# TYPE fgai_graylog_events_fetched gauge", f'fgai_graylog_events_fetched {int(mcp.get("events_fetched", 0) or 0)}']
lines += ["# HELP signalscope_incidents Open incident candidates by severity.", "# TYPE signalscope_incidents gauge"]
for severity in ("critical", "high", "medium", "low"):
lines.append(f'signalscope_incidents{{severity="{severity}"}} {sum(1 for item in incidents if isinstance(item, dict) and item.get("severity") == severity)}')
return "\n".join(lines) + "\n"

17
src/fgai/monitor.py
View File

@@ -58,7 +58,14 @@ def build_status(
events = []
for stream_config in stream_configs:
stream_id = str(stream_config["id"])
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id))).fetch()
profile = stream_profiles.get(stream_id)
profile_fields = (
str(getattr(profile, "entity_field", "")),
str(getattr(profile, "timestamp_field", "")),
*tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
) if profile else ()
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id)), profile_fields).fetch()
events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, **stream_status})
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
@@ -71,7 +78,13 @@ def build_status(
feedback = FeedbackStore().entries()
for entity, deviations in field_deviations.items():
for deviation in deviations:
match = next((item for item in feedback if item.get("entity") == entity and item.get("stream_id") == deviation.get("stream_id") and item.get("field") == deviation.get("field")), None)
match = next((
item for item in feedback
if item.get("entity") == entity
and item.get("stream_id") == deviation.get("stream_id")
and item.get("field") == deviation.get("field")
and (not item.get("value") or item.get("value") == deviation.get("value", ""))
), None)
if match:
deviation["feedback"] = match["status"]
if match["status"] in {"false_positive", "expected"}:

12
tests/test_baseline.py
View File

@@ -4,6 +4,7 @@ from pathlib import Path
from fgai.baseline import BaselineStore
from fgai.logs import parse_log_line
from fgai.stream_profiles import parse_profiles
class BaselineTests(unittest.TestCase):
@@ -15,3 +16,14 @@ class BaselineTests(unittest.TestCase):
profiles = store.profiles({"10.0.0.1"})
self.assertEqual(profiles["10.0.0.1"]["samples"], 12)
self.assertIn("1000", profiles["10.0.0.1"]["known_destination_ports"])
def test_profile_events_are_deduplicated_between_polls(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "categorical_fields": ["action"]}])
events = [
parse_log_line(f"fgai_stream_id=windows username=alice action=login event={index}")
for index in range(12)
]
self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000), 1)
self.assertEqual(store.ingest_profile_fields(events, profiles, observed_at=1_700_000_300), 0)

9
tests/test_correlation.py
View File

@@ -14,3 +14,12 @@ class CorrelationTests(unittest.TestCase):
self.assertEqual(result[0]["source_ip"], "10.0.0.5")
self.assertEqual(result[0]["streams"], ["DNS", "Fortigate"])
self.assertEqual(len(result[0]["samples"]), 2)
def test_correlates_user_across_streams(self):
events = [
parse_log_line("username=alice fgai_stream=Windows action=login timestamp=2026-06-24T10:00:00Z"),
parse_log_line("username=alice fgai_stream=VPN action=accept timestamp=2026-06-24T10:01:00Z"),
]
result = correlate_source_ips(events)
self.assertEqual(result[0]["entity"], "alice")
self.assertEqual(result[0]["entity_type"], "user")

6
tests/test_graylog_source.py
View File

@@ -30,6 +30,12 @@ class GraylogSourceTests(unittest.TestCase):
self.assertIn("client", client.arguments["fields"])
self.assertEqual(client.arguments["offset"], 0)
def test_requests_selected_profile_fields(self):
client = _Client()
GraylogStreamSource(client, "windows", profile_fields=("TargetUserName", "EventID")).fetch()
self.assertIn("targetusername", client.arguments["fields"])
self.assertIn("eventid", client.arguments["fields"])
if __name__ == "__main__":
unittest.main()

5
tests/test_incidents.py
View File

@@ -7,3 +7,8 @@ class IncidentTests(unittest.TestCase):
anomaly = AnomalyFinding("10.0.0.1", 60, "high", "high", ["burst"], {})
result = build_incidents([anomaly], {"10.0.0.1": [{"score": 15, "reason": "new domain"}]}, [{"source_ip": "10.0.0.1", "streams": ["DNS", "Firewall"]}])
self.assertEqual(result[0]["score"], 85)
def test_creates_incident_for_profile_entity_without_network_anomaly(self):
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, [])
self.assertEqual(result[0]["entity"], "alice")
self.assertEqual(result[0]["field_deviations"], 1)