Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix entity dsiplay names instead of ip

Browse Source
This commit is contained in:
larssand
2026-07-02 20:18:55 +02:00
parent 20a9401e02
commit e260e51e67
7 changed files with 162 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/fgai/correlation.py
View File

@@ -3,10 +3,38 @@ from __future__ import annotations
from collections import defaultdict from collections import defaultdict
from .entities import event_entities, sample_timeline from .entities import event_entities, sample_timeline
from .entities import ENTITY_FIELDS
from .logs import THREAT_ACTIONS, is_utm_event from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent from .models import LogEvent
def _best_related_value(events: list[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str:
counts: dict[str, int] = defaultdict(int)
excluded = exclude.lower()
for event in events:
for field in fields:
value = str(event.fields.get(field, "")).strip()
if not value or value in {"-", "unknown", "n/a"}:
continue
if excluded and value.lower() == excluded:
continue
counts[value] += 1
if not counts:
return ""
return sorted(counts.items(), key=lambda item: (item[1], len(item[0]) <= 64), reverse=True)[0][0]
def _entity_display(kind: str, entity: str, events: list[LogEvent]) -> dict[str, str]:
if kind == "ip":
hostname = _best_related_value(events, ENTITY_FIELDS["host"], exclude=entity)
username = _best_related_value(events, ENTITY_FIELDS["user"], exclude=entity)
if hostname:
return {"entity_display": hostname, "entity_label": f"{hostname} ({entity})", "entity_detail": entity}
if username:
return {"entity_display": username, "entity_label": f"{username} ({entity})", "entity_detail": entity}
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]: def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
"""Correlate IPs, users, and hosts across independently configured Graylog streams.""" """Correlate IPs, users, and hosts across independently configured Graylog streams."""
grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list) grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
@@ -23,6 +51,7 @@ def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dic
correlations.append({ correlations.append({
"entity": entity, "entity": entity,
"entity_type": kind, "entity_type": kind,
**_entity_display(kind, entity, source_events),
"source_ip": entity if kind == "ip" else "", "source_ip": entity if kind == "ip" else "",
"streams": streams, "streams": streams,
"events": len(source_events), "events": len(source_events),

82
src/fgai/dashboard.py
View File

@@ -201,7 +201,7 @@ function graylogEvidenceLink(query, configuration, label='Open in Graylog') {
return ` <a href="${esc(href)}" target="_blank" rel="noreferrer">${esc(label)}</a>`; return ` <a href="${esc(href)}" target="_blank" rel="noreferrer">${esc(label)}</a>`;
} }
const tableSort = {}; const tableSort = {};
const uiCache = {correlations: [], fieldRows: [], correlationHitboxes: [], selectedCorrelationKey: ''}; const uiCache = {correlations: [], fieldRows: [], correlationHitboxes: [], selectedCorrelationKey: '', entityLabels: {}};
window.availableStreams = []; window.availableStreams = [];
window.streamSelection = {}; window.streamSelection = {};
window.currentCorrelations = []; window.currentCorrelations = [];
@@ -258,7 +258,7 @@ function compactRelatedActivity(rows) {
const groups = new Map(); const groups = new Map();
for (const row of rows || []) { for (const row of rows || []) {
const key = `${row.entity || row.source_ip || '-'}|${row.stream || '-'}|${row.type || '-'}`; const key = `${row.entity || row.source_ip || '-'}|${row.stream || '-'}|${row.type || '-'}`;
const group = groups.get(key) || {entity: row.entity || row.source_ip || '-', stream: row.stream || '-', type: row.type || '-', count: 0, security: 0, actions: new Set(), destinations: new Set(), services: new Set(), first: row.timestamp || '', last: row.timestamp || '', samples: []}; const group = groups.get(key) || {entity: row.entity || row.source_ip || '-', entity_label: row.entity_label || entityDisplay(row.entity || row.source_ip, row), stream: row.stream || '-', type: row.type || '-', count: 0, security: 0, actions: new Set(), destinations: new Set(), services: new Set(), first: row.timestamp || '', last: row.timestamp || '', samples: []};
group.count += 1; group.count += 1;
if (row.severity && !['-','info','notice','low'].includes(String(row.severity).toLowerCase())) group.security += 1; if (row.severity && !['-','info','notice','low'].includes(String(row.severity).toLowerCase())) group.security += 1;
if (row.action && row.action !== '-') group.actions.add(row.action); if (row.action && row.action !== '-') group.actions.add(row.action);
@@ -274,6 +274,42 @@ function compactRelatedActivity(rows) {
function correlationKey(item) { function correlationKey(item) {
return String(item.entity || item.source_ip || ''); return String(item.entity || item.source_ip || '');
} }
function correlationLabel(item) {
return String(item.entity_label || item.entity_display || item.entity || item.source_ip || '-');
}
function correlationShortLabel(item) {
const label = correlationLabel(item);
const key = correlationKey(item);
return label && label !== key ? label : key || '-';
}
function rememberEntityLabel(key, label, detail='') {
key = String(key || '');
label = String(label || key || '');
if (!key || !label) return;
uiCache.entityLabels[key] = {label, detail: String(detail || '')};
}
function entityDisplay(value, row={}) {
const key = String(row.entity || row.source_ip || row.src_ip || row.subject || value || '');
const explicit = row.entity_label || row.entity_display;
if (explicit) return String(row.entity_label || row.entity_display);
return (uiCache.entityLabels[key] || {}).label || key || String(value || '-');
}
function entityCell(row, key='entity') {
const value = row[key] || row.entity || row.source_ip || row.src_ip || row.subject || '';
const label = entityDisplay(value, row);
return esc(label || '-');
}
function refreshEntityLabels(correlations, context) {
for (const item of correlations || []) {
rememberEntityLabel(correlationKey(item), correlationShortLabel(item), item.entity_detail || item.source_ip || '');
}
for (const item of (context.source_profiles || [])) {
rememberEntityLabel(item.entity, item.entity_label || item.entity_display || item.entity, item.entity_detail || '');
}
for (const item of (context.security_event_samples || [])) {
rememberEntityLabel(item.entity, item.entity_label || item.entity_display || item.entity, item.entity_detail || '');
}
}
function renderCorrelationExplorer(correlations, configuration) { function renderCorrelationExplorer(correlations, configuration) {
const target = document.getElementById('correlationExplorer'); const target = document.getElementById('correlationExplorer');
const rows = [...(correlations || [])].sort((a,b)=>(Number(b.security_events)||0)-(Number(a.security_events)||0) || (Number(b.events)||0)-(Number(a.events)||0)).slice(0,10); const rows = [...(correlations || [])].sort((a,b)=>(Number(b.security_events)||0)-(Number(a.security_events)||0) || (Number(b.events)||0)-(Number(a.events)||0)).slice(0,10);
@@ -287,14 +323,17 @@ function renderCorrelationExplorer(correlations, configuration) {
const selected = rows.find(item => correlationKey(item) === uiCache.selectedCorrelationKey) || rows[0]; const selected = rows.find(item => correlationKey(item) === uiCache.selectedCorrelationKey) || rows[0];
const chips = rows.map(item => { const chips = rows.map(item => {
const key = correlationKey(item); const key = correlationKey(item);
const label = `${key || '-'} (${Number(item.security_events || 0)}/${Number(item.events || 0)})`; const label = `${correlationShortLabel(item)} (${Number(item.security_events || 0)}/${Number(item.events || 0)})`;
return `<button type="button" class="entity-chip ${key === uiCache.selectedCorrelationKey ? 'active' : ''}" data-correlation-key="${esc(key)}">${esc(label)}</button>`; return `<button type="button" class="entity-chip ${key === uiCache.selectedCorrelationKey ? 'active' : ''}" data-correlation-key="${esc(key)}">${esc(label)}</button>`;
}).join(''); }).join('');
const samples = (selected.samples || []).slice(0,8).map(item => { const samples = (selected.samples || []).slice(0,8).map(item => {
const line = `${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.service || ''} | ${item.context || item.message || ''}`; const line = `${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.service || ''} | ${item.context || item.message || ''}`;
return `<div class="evidence-item"><div>${esc(line)}</div>${item.graylog_query ? `<code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}</div>`; return `<div class="evidence-item"><div>${esc(line)}</div>${item.graylog_query ? `<code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}</div>`;
}).join(''); }).join('');
target.innerHTML = `<div class="entity-picker">${chips}</div><div><strong>${esc(correlationKey(selected) || '-')}</strong><div class="muted">${esc((selected.streams || []).join(', ') || 'single stream')} · ${esc(selected.events || 0)} events · ${esc(selected.security_events || 0)} security events</div></div><div class="evidence-list">${samples || '<p class="muted">No sample evidence for this entity.</p>'}</div>`; const selectedKey = correlationKey(selected);
const selectedLabel = correlationShortLabel(selected);
const keyNote = selectedLabel !== selectedKey ? ` · key ${selectedKey}` : '';
target.innerHTML = `<div class="entity-picker">${chips}</div><div><strong>${esc(selectedLabel || '-')}</strong><div class="muted">${esc((selected.streams || []).join(', ') || 'single stream')} · ${esc(selected.events || 0)} events · ${esc(selected.security_events || 0)} security events${esc(keyNote)}</div></div><div class="evidence-list">${samples || '<p class="muted">No sample evidence for this entity.</p>'}</div>`;
target.querySelectorAll('.entity-chip').forEach(button => button.addEventListener('click', () => { target.querySelectorAll('.entity-chip').forEach(button => button.addEventListener('click', () => {
uiCache.selectedCorrelationKey = button.dataset.correlationKey || ''; uiCache.selectedCorrelationKey = button.dataset.correlationKey || '';
renderCorrelationExplorer(window.currentCorrelations || [], window.currentConfiguration || {}); renderCorrelationExplorer(window.currentCorrelations || [], window.currentConfiguration || {});
@@ -424,7 +463,7 @@ function drawCorrelationGraph(correlations) {
ctx.fillText(short(label,24), x, y+r+14); ctx.fillText(short(label,24), x, y+r+14);
if (sub) { ctx.font='11px Arial'; ctx.fillStyle='#91abc4'; ctx.fillText(short(sub,28), x, y+r+28); } if (sub) { ctx.font='11px Arial'; ctx.fillStyle='#91abc4'; ctx.fillText(short(sub,28), x, y+r+28); }
}; };
drawNode(center.x, center.y, 24, Number(selected.security_events||0) ? '#d95f5f' : '#2389cc', correlationKey(selected), `${Number(selected.events)||0} events / ${Number(selected.security_events)||0} security`, correlationKey(selected)); drawNode(center.x, center.y, 24, Number(selected.security_events||0) ? '#d95f5f' : '#2389cc', correlationShortLabel(selected), `${Number(selected.events)||0} events / ${Number(selected.security_events)||0} security`, correlationKey(selected));
nodePositions.forEach(node => { nodePositions.forEach(node => {
const color=node.kind==='stream' ? '#238b5d' : '#6f55c8'; const color=node.kind==='stream' ? '#238b5d' : '#6f55c8';
const count=samples.filter(sample => sample.stream===node.name || sample.destination===node.name || sample.service===node.name || sample.type===node.name).length; const count=samples.filter(sample => sample.stream===node.name || sample.destination===node.name || sample.service===node.name || sample.type===node.name).length;
@@ -439,8 +478,8 @@ function drawCorrelationGraph(correlations) {
const key=correlationKey(item); const key=correlationKey(item);
const y=sideY+22+index*23; const y=sideY+22+index*23;
ctx.fillStyle=key===correlationKey(selected)?'#1ea9ff':'#91abc4'; ctx.fillStyle=key===correlationKey(selected)?'#1ea9ff':'#91abc4';
ctx.fillText(`${short(key,18)} ${Number(item.security_events||0)}/${Number(item.events||0)}`, sideX, y); ctx.fillText(`${short(correlationShortLabel(item),24)} ${Number(item.security_events||0)}/${Number(item.events||0)}`, sideX, y);
uiCache.correlationHitboxes.push({key,x:sideX-4,y:y-14,w:150,h:20}); uiCache.correlationHitboxes.push({key,x:sideX-4,y:y-14,w:210,h:20});
}); });
ctx.textAlign='left'; ctx.fillStyle='#91abc4'; ctx.font='12px Arial'; ctx.fillText('Selected entity is centered. Green nodes are streams; purple nodes are destinations/services. Click a listed entity or center node evidence chip to change focus.', 8, ch-14); ctx.textAlign='left'; ctx.fillStyle='#91abc4'; ctx.font='12px Arial'; ctx.fillText('Selected entity is centered. Green nodes are streams; purple nodes are destinations/services. Click a listed entity or center node evidence chip to change focus.', 8, ch-14);
} }
@@ -465,6 +504,7 @@ async function refresh() {
const configuration = data.configuration || {}; const configuration = data.configuration || {};
const cache = data.status_cache || {}; const cache = data.status_cache || {};
const streamCoverage = data.stream_coverage || []; const streamCoverage = data.stream_coverage || [];
const context = data.event_context || {};
const enabledStreams = streamCoverage.filter(item => item.enabled); const enabledStreams = streamCoverage.filter(item => item.enabled);
const streamsMissingProfile = enabledStreams.filter(item => !item.profile_ready).length; const streamsMissingProfile = enabledStreams.filter(item => !item.profile_ready).length;
const enabledWithNoRawEvents = enabledStreams.filter(item => Number(item.events_fetched || 0) === 0); const enabledWithNoRawEvents = enabledStreams.filter(item => Number(item.events_fetched || 0) === 0);
@@ -477,6 +517,7 @@ async function refresh() {
const correlationsCached = rawCorrelations.length === 0 && uiCache.correlations.length > 0; const correlationsCached = rawCorrelations.length === 0 && uiCache.correlations.length > 0;
const correlations = rawCorrelations.length ? rawCorrelations : uiCache.correlations; const correlations = rawCorrelations.length ? rawCorrelations : uiCache.correlations;
if (rawCorrelations.length) uiCache.correlations = rawCorrelations; if (rawCorrelations.length) uiCache.correlations = rawCorrelations;
refreshEntityLabels(correlations, context);
window.currentCorrelations = correlations; window.currentCorrelations = correlations;
window.currentConfiguration = configuration; window.currentConfiguration = configuration;
drawTrend(data.history || []); drawTrend(data.history || []);
@@ -552,7 +593,7 @@ async function refresh() {
const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.'); const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`; document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`;
document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [ document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [
{label:'Source', key:'subject'}, {label:'Source', key:'subject', render:r => entityCell(r, 'subject')},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`}, {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Confidence', key:'confidence'}, {label:'Confidence', key:'confidence'},
@@ -564,7 +605,7 @@ async function refresh() {
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))} {label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
], 'anomalies'); ], 'anomalies');
document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [ document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [
{label:'Subject', key:'subject'}, {label:'Subject', key:'subject', render:r => entityCell(r, 'subject')},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`}, {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Title', key:'title'}, {label:'Title', key:'title'},
@@ -573,7 +614,7 @@ async function refresh() {
{label:'Services', render:r => esc((r.related_services || []).join(', '))} {label:'Services', render:r => esc((r.related_services || []).join(', '))}
], 'recommendations'); ], 'recommendations');
document.getElementById('incidents').innerHTML = table(data.incidents || [], [ document.getElementById('incidents').innerHTML = table(data.incidents || [], [
{label:'Entity', key:'entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)}, {label:'Entity', key:'entity', render:r => esc(`${entityDisplay(r.entity, r)} (${r.entity_type || 'entity'})`)},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`}, {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'State', render:r => esc(r.lifecycle_status || 'open')}, {label:'State', render:r => esc(r.lifecycle_status || 'open')},
@@ -589,15 +630,15 @@ async function refresh() {
refresh(); refresh();
})); }));
document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [ document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
{label:'Source', key:'src_ip'}, {label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))} {label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
], 'blocks'); ], 'blocks');
const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel})); const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample}))); const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, entity_label: correlation.entity_label || correlation.entity_display || correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id])); const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
document.getElementById('triageQueue').innerHTML = table(data.triage_queue || [], [ document.getElementById('triageQueue').innerHTML = table(data.triage_queue || [], [
{label:'Entity', key:'entity'}, {label:'Entity', key:'entity', render:r => entityCell(r)},
{label:'Score', key:'score'}, {label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`}, {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'State', key:'state'}, {label:'State', key:'state'},
@@ -621,7 +662,7 @@ async function refresh() {
document.getElementById('findingSummary').textContent = `${fieldRows.length} shown from ${sourceFieldRows.length} raw deviations. Reviewed and low-score findings are hidden by default.`; document.getElementById('findingSummary').textContent = `${fieldRows.length} shown from ${sourceFieldRows.length} raw deviations. Reviewed and low-score findings are hidden by default.`;
if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows; if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [ document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">False positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Confirm</button></div>`} {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">False positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Confirm</button></div>`}
], 'field-deviations'); ], 'field-deviations');
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => { document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Review note (optional):') || ''; const note = prompt('Review note (optional):') || '';
@@ -632,7 +673,7 @@ async function refresh() {
})); }));
const relatedGroups = compactRelatedActivity(relatedRows); const relatedGroups = compactRelatedActivity(relatedRows);
document.getElementById('relatedActivity').innerHTML = `<p class="muted">${relatedGroups.length} grouped rows shown from ${relatedRows.length} raw related events.</p>` + table(relatedGroups, [ document.getElementById('relatedActivity').innerHTML = `<p class="muted">${relatedGroups.length} grouped rows shown from ${relatedRows.length} raw related events.</p>` + table(relatedGroups, [
{label:'Entity', key:'entity'}, {label:'Entity', key:'entity', render:r => entityCell(r)},
{label:'Stream', key:'stream'}, {label:'Stream', key:'stream'},
{label:'Type', key:'type'}, {label:'Type', key:'type'},
{label:'Events', key:'count'}, {label:'Events', key:'count'},
@@ -657,24 +698,23 @@ async function refresh() {
{label:'Detail', key:'detail'} {label:'Detail', key:'detail'}
], 'policies'); ], 'policies');
const d = data.diagnostics || {}; const d = data.diagnostics || {};
const context = data.event_context || {};
const quality = data.data_quality || {}; const quality = data.data_quality || {};
const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id])); const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id]));
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})); const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id}));
document.getElementById('diagnostics').innerHTML = document.getElementById('diagnostics').innerHTML =
'<h3>Stream Coverage</h3>' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') + '<h3>Stream Coverage</h3>' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') +
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') + '<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') + '<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') + '<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) + '<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
'<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) + '<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}]) +
'<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Policy IDs</h3>' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Policy IDs</h3>' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Destination Ports</h3>' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Destination Ports</h3>' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Source Ports</h3>' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Source Ports</h3>' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Services</h3>' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Services</h3>' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Local-in Failures</h3>' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip'}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]); '<h3>Local-in Failures</h3>' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]);
document.querySelectorAll('details[data-detail-id]').forEach(item => { if (openDetails.has(item.dataset.detailId)) item.open = true; }); document.querySelectorAll('details[data-detail-id]').forEach(item => { if (openDetails.has(item.dataset.detailId)) item.open = true; });
document.querySelectorAll('[data-sort-table]').forEach(button => button.addEventListener('click', () => { const current=tableSort[button.dataset.sortTable]; tableSort[button.dataset.sortTable]={key:button.dataset.sortKey,direction:current && current.key===button.dataset.sortKey ? -current.direction : 1}; refresh(); })); document.querySelectorAll('[data-sort-table]').forEach(button => button.addEventListener('click', () => { const current=tableSort[button.dataset.sortTable]; tableSort[button.dataset.sortTable]={key:button.dataset.sortKey,direction:current && current.key===button.dataset.sortKey ? -current.direction : 1}; refresh(); }));
document.querySelectorAll('.apply-suggested-profile').forEach(button => button.addEventListener('click', () => applySuggestedProfile(button.dataset.streamId))); document.querySelectorAll('.apply-suggested-profile').forEach(button => button.addEventListener('click', () => applySuggestedProfile(button.dataset.streamId)));

31
src/fgai/event_context.py
View File

@@ -1,7 +1,9 @@
from __future__ import annotations from __future__ import annotations
import ipaddress
from collections import Counter, defaultdict from collections import Counter, defaultdict
from .entities import ENTITY_FIELDS
from .logs import THREAT_ACTIONS, is_utm_event from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent from .models import LogEvent
from .normalization import canonical_value from .normalization import canonical_value
@@ -11,6 +13,32 @@ def _entity(event: LogEvent) -> str:
return event.src_ip or event.fields.get("user") or event.fields.get("username") or event.fields.get("hostname") or event.fields.get("source") or "unknown" return event.src_ip or event.fields.get("user") or event.fields.get("username") or event.fields.get("hostname") or event.fields.get("source") or "unknown"
def _related_value(events: list[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str:
counts: Counter[str] = Counter()
excluded = exclude.lower()
for event in events:
for field in fields:
value = str(event.fields.get(field, "")).strip()
if not value or value in {"-", "unknown", "n/a"}:
continue
if excluded and value.lower() == excluded:
continue
counts[value] += 1
return counts.most_common(1)[0][0] if counts else ""
def _display_label(entity: str, events: list[LogEvent]) -> dict[str, str]:
try:
ipaddress.ip_address(entity)
except ValueError:
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
hostname = _related_value(events, ENTITY_FIELDS["host"], exclude=entity)
username = _related_value(events, ENTITY_FIELDS["user"], exclude=entity)
display = hostname or username or entity
label = f"{display} ({entity})" if display != entity else entity
return {"entity_display": display, "entity_label": label, "entity_detail": entity if display != entity else ""}
def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sample_limit: int = 15) -> dict[str, object]: def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sample_limit: int = 15) -> dict[str, object]:
grouped: dict[str, list[LogEvent]] = defaultdict(list) grouped: dict[str, list[LogEvent]] = defaultdict(list)
field_presence: Counter[str] = Counter() field_presence: Counter[str] = Counter()
@@ -24,6 +52,7 @@ def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sampl
destinations = {event.dst_ip for event in source_events if event.dst_ip} destinations = {event.dst_ip for event in source_events if event.dst_ip}
source_profiles.append({ source_profiles.append({
"entity": entity, "events": len(source_events), "utm_events": sum(is_utm_event(event) for event in source_events), "entity": entity, "events": len(source_events), "utm_events": sum(is_utm_event(event) for event in source_events),
**_display_label(entity, source_events),
"deny_or_threat_actions": sum(event.action in THREAT_ACTIONS for event in source_events), "deny_or_threat_actions": sum(event.action in THREAT_ACTIONS for event in source_events),
"distinct_destinations": len(destinations), "top_actions": [action for action, _ in actions.most_common(3)], "distinct_destinations": len(destinations), "top_actions": [action for action, _ in actions.most_common(3)],
}) })
@@ -31,7 +60,7 @@ def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sampl
suspicious = [event for event in events if is_utm_event(event) or event.action in THREAT_ACTIONS or event.severity in {"critical", "high", "alert", "emergency"}] suspicious = [event for event in events if is_utm_event(event) or event.action in THREAT_ACTIONS or event.severity in {"critical", "high", "alert", "emergency"}]
samples = [{ samples = [{
"entity": _entity(event), "type": canonical_value(event.fields, "type"), "subtype": event.subtype, "entity": _entity(event), **_display_label(_entity(event), [event]), "type": canonical_value(event.fields, "type"), "subtype": event.subtype,
"action": event.action, "severity": event.severity, "dst": event.dst_ip or canonical_value(event.fields, "context"), "service": canonical_value(event.fields, "service"), "action": event.action, "severity": event.severity, "dst": event.dst_ip or canonical_value(event.fields, "context"), "service": canonical_value(event.fields, "service"),
"policyid": event.fields.get("policyid", ""), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "policyid": event.fields.get("policyid", ""), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
} for event in suspicious[:sample_limit]] } for event in suspicious[:sample_limit]]

10
src/fgai/incidents.py
View File

@@ -13,6 +13,15 @@ from .models import AnomalyFinding
def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]: def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]:
"""Build investigation units from any log source, not only network source IPs.""" """Build investigation units from any log source, not only network source IPs."""
groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []}) groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []})
entity_labels = {
str(item.get("entity") or item.get("source_ip")): {
"entity_display": str(item.get("entity_display") or item.get("entity") or item.get("source_ip") or ""),
"entity_label": str(item.get("entity_label") or item.get("entity_display") or item.get("entity") or item.get("source_ip") or ""),
"entity_detail": str(item.get("entity_detail") or ""),
}
for item in correlations
if item.get("entity") or item.get("source_ip")
}
for anomaly in anomalies: for anomaly in anomalies:
group = groups[anomaly.subject] group = groups[anomaly.subject]
group["score"] = max(int(group["score"]), anomaly.score) group["score"] = max(int(group["score"]), anomaly.score)
@@ -47,6 +56,7 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str,
incidents.append({ incidents.append({
"id": incident_id, "id": incident_id,
"entity": entity, "entity": entity,
**entity_labels.get(entity, {"entity_display": entity, "entity_label": entity, "entity_detail": ""}),
"entity_type": entity_type(entity), "entity_type": entity_type(entity),
"score": score, "score": score,
"severity": severity, "severity": severity,

12
src/fgai/triage.py
View File

@@ -18,6 +18,9 @@ def build_triage_queue(
"""Summarize raw findings into analyst-sized investigation candidates.""" """Summarize raw findings into analyst-sized investigation candidates."""
groups: dict[str, dict[str, object]] = defaultdict(lambda: { groups: dict[str, dict[str, object]] = defaultdict(lambda: {
"entity": "", "entity": "",
"entity_display": "",
"entity_label": "",
"entity_detail": "",
"score": 0, "score": 0,
"streams": set(), "streams": set(),
"detectors": Counter(), "detectors": Counter(),
@@ -35,6 +38,9 @@ def build_triage_queue(
continue continue
group = groups[entity] group = groups[entity]
group["entity"] = entity group["entity"] = entity
group["entity_display"] = str(incident.get("entity_display") or group["entity_display"] or entity)
group["entity_label"] = str(incident.get("entity_label") or group["entity_label"] or group["entity_display"] or entity)
group["entity_detail"] = str(incident.get("entity_detail") or group["entity_detail"] or "")
group["score"] = max(int(group["score"]), int(incident.get("score", 0) or 0)) group["score"] = max(int(group["score"]), int(incident.get("score", 0) or 0))
group["incident_id"] = str(incident.get("id", "")) group["incident_id"] = str(incident.get("id", ""))
group["state"] = str(incident.get("lifecycle_status", "open") or "open") group["state"] = str(incident.get("lifecycle_status", "open") or "open")
@@ -63,6 +69,9 @@ def build_triage_queue(
continue continue
group = groups[entity] group = groups[entity]
group["entity"] = entity group["entity"] = entity
group["entity_display"] = str(correlation.get("entity_display") or group["entity_display"] or entity)
group["entity_label"] = str(correlation.get("entity_label") or group["entity_label"] or group["entity_display"] or entity)
group["entity_detail"] = str(correlation.get("entity_detail") or group["entity_detail"] or "")
streams = [str(item) for item in correlation.get("streams", []) if item] streams = [str(item) for item in correlation.get("streams", []) if item]
group["streams"].update(streams) group["streams"].update(streams)
group["correlated_streams"] = max(int(group["correlated_streams"]), len(streams)) group["correlated_streams"] = max(int(group["correlated_streams"]), len(streams))
@@ -96,6 +105,9 @@ def build_triage_queue(
next_action = "follow confirmed incident response" next_action = "follow confirmed incident response"
queue.append({ queue.append({
"entity": group["entity"], "entity": group["entity"],
"entity_display": group["entity_display"] or group["entity"],
"entity_label": group["entity_label"] or group["entity_display"] or group["entity"],
"entity_detail": group["entity_detail"],
"score": score, "score": score,
"severity": _severity(score), "severity": _severity(score),
"state": group["state"], "state": group["state"],

11
tests/test_correlation.py
View File

@@ -23,3 +23,14 @@ class CorrelationTests(unittest.TestCase):
result = correlate_source_ips(events) result = correlate_source_ips(events)
self.assertEqual(result[0]["entity"], "alice") self.assertEqual(result[0]["entity"], "alice")
self.assertEqual(result[0]["entity_type"], "user") self.assertEqual(result[0]["entity_type"], "user")
def test_ip_entity_prefers_hostname_display_label(self):
events = [
parse_log_line("srcip=10.0.0.5 hostname=win01 fgai_stream=Windows action=login"),
parse_log_line("srcip=10.0.0.5 hostname=win01 fgai_stream=Firewall action=blocked"),
]
result = correlate_source_ips(events)
self.assertEqual(result[0]["entity"], "10.0.0.5")
self.assertEqual(result[0]["entity_display"], "win01")
self.assertEqual(result[0]["entity_label"], "win01 (10.0.0.5)")
self.assertEqual(result[0]["entity_detail"], "10.0.0.5")

9
tests/test_event_context.py
View File

@@ -15,6 +15,15 @@ class EventContextTests(unittest.TestCase):
self.assertEqual(context["source_profiles"][0]["entity"], "10.0.0.2") self.assertEqual(context["source_profiles"][0]["entity"], "10.0.0.2")
self.assertEqual(len(context["security_event_samples"]), 1) self.assertEqual(len(context["security_event_samples"]), 1)
def test_ip_entity_prefers_hostname_display_label(self):
events = [
parse_log_line("srcip=10.0.0.2 hostname=win01 dstip=8.8.8.8 type=utm action=blocked severity=high"),
]
context = build_event_context(events)
self.assertEqual(context["source_profiles"][0]["entity"], "10.0.0.2")
self.assertEqual(context["source_profiles"][0]["entity_label"], "win01 (10.0.0.2)")
self.assertEqual(context["security_event_samples"][0]["entity_label"], "win01 (10.0.0.2)")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()