Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixat orsaken till att Findings/Correlation blinkade bort.

Browse Source
This commit is contained in:
larssand
2026-06-25 18:51:04 +02:00
parent 7e6b510b9e
commit 899dbdfc84
5 changed files with 43 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -117,6 +117,10 @@ The current MCP endpoint is `http://<graylog-host>:9000/api/mcp`. Enable it in
Graylog under `System -> Configurations -> MCP` and use stream IDs internally;
the fgAI stream picker resolves titles in the UI.
For live monitoring, `graylog_range_seconds` controls how far back each MCP poll
searches. The default is one hour, which keeps findings and correlations more
stable than a very short window while still limiting MCP query cost.
## Monitoring Export
The dashboard also exposes Prometheus text metrics at:

6
src/fgai/config.py
View File

@@ -12,6 +12,7 @@ DEFAULT_CONFIG: dict[str, object] = {
"graylog_streams": [],
"graylog_stream_profiles": [],
"graylog_query": "*",
"graylog_range_seconds": 3600,
"graylog_field_mapping": "",
"llm_enabled": False,
"llm_model": "",
@@ -48,6 +49,11 @@ class ConfigStore:
current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value
elif key == "graylog_range_seconds":
try:
current[key] = max(60, int(value))
except (TypeError, ValueError):
continue
elif key == "graylog_streams" and isinstance(value, list):
current[key] = [
{"id": str(item.get("id", "")), "title": str(item.get("title", "")), "enabled": bool(item.get("enabled"))}

23
src/fgai/dashboard.py
View File

@@ -80,7 +80,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main>
<script>
function esc(value) {
@@ -90,6 +90,7 @@ function metric(label, value) {
return `<div class="panel"><div class="metric">${esc(value)}</div><div class="label">${esc(label)}</div></div>`;
}
const tableSort = {};
const uiCache = {correlations: [], fieldRows: []};
function table(rows, columns, id = '') {
if (!rows || rows.length === 0) return '<p class="muted">No data.</p>';
const sort = tableSort[id];
@@ -111,9 +112,13 @@ async function refresh() {
const threat = (data.capabilities || {}).threat_intel || {};
const mcp = (data.capabilities || {}).graylog_mcp || {};
const configuration = data.configuration || {};
const rawCorrelations = data.cross_source_correlations || [];
const correlationsCached = rawCorrelations.length === 0 && uiCache.correlations.length > 0;
const correlations = rawCorrelations.length ? rawCorrelations : uiCache.correlations;
if (rawCorrelations.length) uiCache.correlations = rawCorrelations;
drawTrend(data.history || []);
drawCorrelationGraph(data.cross_source_correlations || []);
document.getElementById('correlationGraphInfo').textContent = `${(data.cross_source_correlations || []).length} entities correlated across enabled streams. Blue entities connect to green stream nodes; red edges carry security-event activity.`;
drawCorrelationGraph(correlations);
document.getElementById('correlationGraphInfo').textContent = `${correlations.length} entities correlated across enabled streams${correlationsCached ? ' (cached from previous non-empty poll)' : ''}. Blue entities connect to green stream nodes; red edges carry security-event activity.`;
document.getElementById('stamp').textContent = data.generated_at ? `Updated ${new Date(data.generated_at * 1000).toLocaleString()}` : 'Waiting for monitor data';
document.getElementById('metrics').innerHTML = [
metric('Total events', s.total || 0),
@@ -136,7 +141,7 @@ async function refresh() {
`Baseline sources ready: ${esc((data.baseline || {}).sources_ready || 0)}`
].join('<br>');
document.getElementById('health').innerHTML = [
metric('Baseline sources ready', baseline.sources_ready || 0), metric('MCP events fetched', mcp.events_fetched || 0), metric('Profiles active', (data.stream_profiles || []).length), metric('Correlated entities', (data.cross_source_correlations || []).length)
metric('Baseline sources ready', baseline.sources_ready || 0), metric('MCP events fetched', mcp.events_fetched || 0), metric('Profiles active', (data.stream_profiles || []).length), metric('Correlated entities', correlations.length)
].join('');
const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`;
@@ -175,10 +180,13 @@ async function refresh() {
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
], 'blocks');
const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item})));
document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [
const rawFieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, ...item, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})));
const fieldRowsCached = rawFieldRows.length === 0 && uiCache.fieldRows.length > 0;
const fieldRows = rawFieldRows.length ? rawFieldRows : uiCache.fieldRows;
if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Confidence', key:'confidence'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; samples ${r.baseline_samples ?? '-'}; scope ${r.baseline_scope ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
], 'field-deviations');
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
@@ -212,7 +220,6 @@ async function refresh() {
const quality = data.data_quality || {};
const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id]));
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id}));
const correlations = data.cross_source_correlations || [];
document.getElementById('diagnostics').innerHTML =
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +

12
src/fgai/monitor.py
View File

@@ -43,6 +43,13 @@ def _profile_name(stream_id: str, stream_titles: dict[str, str], profile: object
return name
def _range_seconds(value: object) -> int:
try:
return max(60, int(value))
except (TypeError, ValueError):
return 3600
def build_status(
log_path: str,
*,
@@ -76,6 +83,7 @@ def build_status(
stream_configs = [{"id": str(runtime_values.get("graylog_stream", "")), "title": "Graylog"}]
stream_statuses = []
events = []
range_seconds = _range_seconds(runtime_values.get("graylog_range_seconds", 3600))
for stream_config in stream_configs:
stream_id = str(stream_config["id"])
profile = stream_profiles.get(stream_id)
@@ -86,10 +94,10 @@ def build_status(
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
) if profile else ()
stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch()
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(range_seconds=range_seconds)
events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status})
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events), "range_seconds": range_seconds}
except RuntimeError as exc:
mcp_status = {"status": "error", "error": str(exc)}
events = []

8
tests/test_config.py
View File

@@ -15,6 +15,14 @@ class ConfigTests(unittest.TestCase):
self.assertTrue(public["graylog_mcp_token_configured"])
self.assertNotIn("graylog_mcp_token", public)
def test_graylog_range_seconds_is_numeric_and_bounded(self):
with tempfile.TemporaryDirectory() as directory:
store = ConfigStore(str(Path(directory) / "config.json"))
public = store.update({"graylog_range_seconds": "30"})
self.assertEqual(public["graylog_range_seconds"], 60)
public = store.update({"graylog_range_seconds": "7200"})
self.assertEqual(public["graylog_range_seconds"], 7200)
if __name__ == "__main__":
unittest.main()