Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the next multi-source detection layer in this repository.

Browse Source
This commit is contained in:
larssand
2026-06-24 19:16:16 +02:00
parent f6bee0438c
commit 868022008a
15 changed files with 281 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/fgai/dashboard.py
View File

@@ -74,7 +74,7 @@ HTML = """<!doctype html>
</div>
</section>
<nav class="tabs" aria-label="Dashboard views"><button class="tab active" data-tab="overview">Overview</button><button class="tab" data-tab="findings">Findings</button><button class="tab" data-tab="diagnostics">Diagnostics</button><button class="tab" data-tab="settings">Settings</button></nav>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
@@ -151,27 +151,35 @@ async function refresh() {
{label:'Policies', render:r => esc((r.related_policy_ids || []).join(', '))},
{label:'Services', render:r => esc((r.related_services || []).join(', '))}
]);
document.getElementById('incidents').innerHTML = table(data.incidents || [], [
{label:'Entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)},
{label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')},
{label:'Evidence', render:r => esc((r.evidence || []).join('; '))},
{label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)).join('<br>'); return rows ? `<details><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }}
]);
document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
{label:'Source', key:'src_ip'},
{label:'Score', key:'score'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({source_ip: correlation.source_ip, ...sample})));
const relatedRows = (data.cross_source_correlations || []).flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item})));
document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark confirmed</button></div>`}
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
]);
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Review note (optional):') || '';
const days = prompt('Expiry in days (0 = no expiry):', '0') || '0';
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, value:button.dataset.value, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
document.getElementById('feedbackNotice').textContent = response.ok ? 'Review saved. The matching pattern will be labeled on the next refresh.' : 'Could not save review.';
refresh();
}));
document.getElementById('relatedActivity').innerHTML = table(relatedRows, [
{label:'Source IP', key:'source_ip'}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
{label:'Entity', render:r => esc(r.entity || r.source_ip)}, {label:'Stream', key:'stream'}, {label:'Time', key:'timestamp'},
{label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'},
{label:'Destination', key:'destination'}, {label:'Service', key:'service'}, {label:'Context', key:'context'}
]);
@@ -195,7 +203,7 @@ async function refresh() {
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id}));
const correlations = data.cross_source_correlations || [];
document.getElementById('diagnostics').innerHTML =
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Source IP', key:'source_ip'}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', render:r => r.ready ? 'ready' : 'learning'}]) +
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +