diff --git a/src/fgai/correlation.py b/src/fgai/correlation.py index 6a1a476..e2ff87d 100644 --- a/src/fgai/correlation.py +++ b/src/fgai/correlation.py @@ -3,10 +3,38 @@ from __future__ import annotations from collections import defaultdict from .entities import event_entities, sample_timeline +from .entities import ENTITY_FIELDS from .logs import THREAT_ACTIONS, is_utm_event from .models import LogEvent +def _best_related_value(events: list[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str: + counts: dict[str, int] = defaultdict(int) + excluded = exclude.lower() + for event in events: + for field in fields: + value = str(event.fields.get(field, "")).strip() + if not value or value in {"-", "unknown", "n/a"}: + continue + if excluded and value.lower() == excluded: + continue + counts[value] += 1 + if not counts: + return "" + return sorted(counts.items(), key=lambda item: (item[1], len(item[0]) <= 64), reverse=True)[0][0] + + +def _entity_display(kind: str, entity: str, events: list[LogEvent]) -> dict[str, str]: + if kind == "ip": + hostname = _best_related_value(events, ENTITY_FIELDS["host"], exclude=entity) + username = _best_related_value(events, ENTITY_FIELDS["user"], exclude=entity) + if hostname: + return {"entity_display": hostname, "entity_label": f"{hostname} ({entity})", "entity_detail": entity} + if username: + return {"entity_display": username, "entity_label": f"{username} ({entity})", "entity_detail": entity} + return {"entity_display": entity, "entity_label": entity, "entity_detail": ""} + + def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]: """Correlate IPs, users, and hosts across independently configured Graylog streams.""" grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list) @@ -23,6 +51,7 @@ def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dic correlations.append({ "entity": entity, "entity_type": kind, + **_entity_display(kind, entity, source_events), "source_ip": entity if kind == "ip" else "", "streams": streams, "events": len(source_events), diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index cb326e7..c245808 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -201,7 +201,7 @@ function graylogEvidenceLink(query, configuration, label='Open in Graylog') { return ` ${esc(label)}`; } const tableSort = {}; -const uiCache = {correlations: [], fieldRows: [], correlationHitboxes: [], selectedCorrelationKey: ''}; +const uiCache = {correlations: [], fieldRows: [], correlationHitboxes: [], selectedCorrelationKey: '', entityLabels: {}}; window.availableStreams = []; window.streamSelection = {}; window.currentCorrelations = []; @@ -258,7 +258,7 @@ function compactRelatedActivity(rows) { const groups = new Map(); for (const row of rows || []) { const key = `${row.entity || row.source_ip || '-'}|${row.stream || '-'}|${row.type || '-'}`; - const group = groups.get(key) || {entity: row.entity || row.source_ip || '-', stream: row.stream || '-', type: row.type || '-', count: 0, security: 0, actions: new Set(), destinations: new Set(), services: new Set(), first: row.timestamp || '', last: row.timestamp || '', samples: []}; + const group = groups.get(key) || {entity: row.entity || row.source_ip || '-', entity_label: row.entity_label || entityDisplay(row.entity || row.source_ip, row), stream: row.stream || '-', type: row.type || '-', count: 0, security: 0, actions: new Set(), destinations: new Set(), services: new Set(), first: row.timestamp || '', last: row.timestamp || '', samples: []}; group.count += 1; if (row.severity && !['-','info','notice','low'].includes(String(row.severity).toLowerCase())) group.security += 1; if (row.action && row.action !== '-') group.actions.add(row.action); @@ -274,6 +274,42 @@ function compactRelatedActivity(rows) { function correlationKey(item) { return String(item.entity || item.source_ip || ''); } +function correlationLabel(item) { + return String(item.entity_label || item.entity_display || item.entity || item.source_ip || '-'); +} +function correlationShortLabel(item) { + const label = correlationLabel(item); + const key = correlationKey(item); + return label && label !== key ? label : key || '-'; +} +function rememberEntityLabel(key, label, detail='') { + key = String(key || ''); + label = String(label || key || ''); + if (!key || !label) return; + uiCache.entityLabels[key] = {label, detail: String(detail || '')}; +} +function entityDisplay(value, row={}) { + const key = String(row.entity || row.source_ip || row.src_ip || row.subject || value || ''); + const explicit = row.entity_label || row.entity_display; + if (explicit) return String(row.entity_label || row.entity_display); + return (uiCache.entityLabels[key] || {}).label || key || String(value || '-'); +} +function entityCell(row, key='entity') { + const value = row[key] || row.entity || row.source_ip || row.src_ip || row.subject || ''; + const label = entityDisplay(value, row); + return esc(label || '-'); +} +function refreshEntityLabels(correlations, context) { + for (const item of correlations || []) { + rememberEntityLabel(correlationKey(item), correlationShortLabel(item), item.entity_detail || item.source_ip || ''); + } + for (const item of (context.source_profiles || [])) { + rememberEntityLabel(item.entity, item.entity_label || item.entity_display || item.entity, item.entity_detail || ''); + } + for (const item of (context.security_event_samples || [])) { + rememberEntityLabel(item.entity, item.entity_label || item.entity_display || item.entity, item.entity_detail || ''); + } +} function renderCorrelationExplorer(correlations, configuration) { const target = document.getElementById('correlationExplorer'); const rows = [...(correlations || [])].sort((a,b)=>(Number(b.security_events)||0)-(Number(a.security_events)||0) || (Number(b.events)||0)-(Number(a.events)||0)).slice(0,10); @@ -287,14 +323,17 @@ function renderCorrelationExplorer(correlations, configuration) { const selected = rows.find(item => correlationKey(item) === uiCache.selectedCorrelationKey) || rows[0]; const chips = rows.map(item => { const key = correlationKey(item); - const label = `${key || '-'} (${Number(item.security_events || 0)}/${Number(item.events || 0)})`; + const label = `${correlationShortLabel(item)} (${Number(item.security_events || 0)}/${Number(item.events || 0)})`; return ``; }).join(''); const samples = (selected.samples || []).slice(0,8).map(item => { const line = `${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.service || ''} | ${item.context || item.message || ''}`; return `
${esc(line)}
${item.graylog_query ? `${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}
`; }).join(''); - target.innerHTML = `
${chips}
${esc(correlationKey(selected) || '-')}
${esc((selected.streams || []).join(', ') || 'single stream')} · ${esc(selected.events || 0)} events · ${esc(selected.security_events || 0)} security events
${samples || '

No sample evidence for this entity.

'}
`; + const selectedKey = correlationKey(selected); + const selectedLabel = correlationShortLabel(selected); + const keyNote = selectedLabel !== selectedKey ? ` · key ${selectedKey}` : ''; + target.innerHTML = `
${chips}
${esc(selectedLabel || '-')}
${esc((selected.streams || []).join(', ') || 'single stream')} · ${esc(selected.events || 0)} events · ${esc(selected.security_events || 0)} security events${esc(keyNote)}
${samples || '

No sample evidence for this entity.

'}
`; target.querySelectorAll('.entity-chip').forEach(button => button.addEventListener('click', () => { uiCache.selectedCorrelationKey = button.dataset.correlationKey || ''; renderCorrelationExplorer(window.currentCorrelations || [], window.currentConfiguration || {}); @@ -424,7 +463,7 @@ function drawCorrelationGraph(correlations) { ctx.fillText(short(label,24), x, y+r+14); if (sub) { ctx.font='11px Arial'; ctx.fillStyle='#91abc4'; ctx.fillText(short(sub,28), x, y+r+28); } }; - drawNode(center.x, center.y, 24, Number(selected.security_events||0) ? '#d95f5f' : '#2389cc', correlationKey(selected), `${Number(selected.events)||0} events / ${Number(selected.security_events)||0} security`, correlationKey(selected)); + drawNode(center.x, center.y, 24, Number(selected.security_events||0) ? '#d95f5f' : '#2389cc', correlationShortLabel(selected), `${Number(selected.events)||0} events / ${Number(selected.security_events)||0} security`, correlationKey(selected)); nodePositions.forEach(node => { const color=node.kind==='stream' ? '#238b5d' : '#6f55c8'; const count=samples.filter(sample => sample.stream===node.name || sample.destination===node.name || sample.service===node.name || sample.type===node.name).length; @@ -439,8 +478,8 @@ function drawCorrelationGraph(correlations) { const key=correlationKey(item); const y=sideY+22+index*23; ctx.fillStyle=key===correlationKey(selected)?'#1ea9ff':'#91abc4'; - ctx.fillText(`${short(key,18)} ${Number(item.security_events||0)}/${Number(item.events||0)}`, sideX, y); - uiCache.correlationHitboxes.push({key,x:sideX-4,y:y-14,w:150,h:20}); + ctx.fillText(`${short(correlationShortLabel(item),24)} ${Number(item.security_events||0)}/${Number(item.events||0)}`, sideX, y); + uiCache.correlationHitboxes.push({key,x:sideX-4,y:y-14,w:210,h:20}); }); ctx.textAlign='left'; ctx.fillStyle='#91abc4'; ctx.font='12px Arial'; ctx.fillText('Selected entity is centered. Green nodes are streams; purple nodes are destinations/services. Click a listed entity or center node evidence chip to change focus.', 8, ch-14); } @@ -465,6 +504,7 @@ async function refresh() { const configuration = data.configuration || {}; const cache = data.status_cache || {}; const streamCoverage = data.stream_coverage || []; + const context = data.event_context || {}; const enabledStreams = streamCoverage.filter(item => item.enabled); const streamsMissingProfile = enabledStreams.filter(item => !item.profile_ready).length; const enabledWithNoRawEvents = enabledStreams.filter(item => Number(item.events_fetched || 0) === 0); @@ -477,6 +517,7 @@ async function refresh() { const correlationsCached = rawCorrelations.length === 0 && uiCache.correlations.length > 0; const correlations = rawCorrelations.length ? rawCorrelations : uiCache.correlations; if (rawCorrelations.length) uiCache.correlations = rawCorrelations; + refreshEntityLabels(correlations, context); window.currentCorrelations = correlations; window.currentConfiguration = configuration; drawTrend(data.history || []); @@ -552,7 +593,7 @@ async function refresh() { const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '
') : esc(llm.error || 'LLM assessment disabled or waiting for first run.'); document.getElementById('llmAssessment').innerHTML = `
Status: ${esc(llm.status || 'unknown')}

${llmText}

`; document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [ - {label:'Source', key:'subject'}, + {label:'Source', key:'subject', render:r => entityCell(r, 'subject')}, {label:'Score', key:'score'}, {label:'Severity', render:r => `${esc(r.severity)}`}, {label:'Confidence', key:'confidence'}, @@ -564,7 +605,7 @@ async function refresh() { {label:'Reasons', render:r => esc((r.reasons || []).join('; '))} ], 'anomalies'); document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [ - {label:'Subject', key:'subject'}, + {label:'Subject', key:'subject', render:r => entityCell(r, 'subject')}, {label:'Score', key:'score'}, {label:'Severity', render:r => `${esc(r.severity)}`}, {label:'Title', key:'title'}, @@ -573,7 +614,7 @@ async function refresh() { {label:'Services', render:r => esc((r.related_services || []).join(', '))} ], 'recommendations'); document.getElementById('incidents').innerHTML = table(data.incidents || [], [ - {label:'Entity', key:'entity', render:r => esc(`${r.entity} (${r.entity_type || 'entity'})`)}, + {label:'Entity', key:'entity', render:r => esc(`${entityDisplay(r.entity, r)} (${r.entity_type || 'entity'})`)}, {label:'Score', key:'score'}, {label:'Severity', render:r => `${esc(r.severity)}`}, {label:'State', render:r => esc(r.lifecycle_status || 'open')}, @@ -589,15 +630,15 @@ async function refresh() { refresh(); })); document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [ - {label:'Source', key:'src_ip'}, + {label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')}, {label:'Score', key:'score'}, {label:'Reasons', render:r => esc((r.reasons || []).join('; '))} ], 'blocks'); const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel})); - const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample}))); + const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, entity_label: correlation.entity_label || correlation.entity_display || correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample}))); const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id])); document.getElementById('triageQueue').innerHTML = table(data.triage_queue || [], [ - {label:'Entity', key:'entity'}, + {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Score', key:'score'}, {label:'Severity', render:r => `${esc(r.severity)}`}, {label:'State', key:'state'}, @@ -621,7 +662,7 @@ async function refresh() { document.getElementById('findingSummary').textContent = `${fieldRows.length} shown from ${sourceFieldRows.length} raw deviations. Reviewed and low-score findings are hidden by default.`; if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows; document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '

No current field deviations in this poll; showing cached findings from the last non-empty poll.

' : ''}` + table(fieldRows, [ - {label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `
${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('
'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} + {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `
${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('
'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} ], 'field-deviations'); document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => { const note = prompt('Review note (optional):') || ''; @@ -632,7 +673,7 @@ async function refresh() { })); const relatedGroups = compactRelatedActivity(relatedRows); document.getElementById('relatedActivity').innerHTML = `

${relatedGroups.length} grouped rows shown from ${relatedRows.length} raw related events.

` + table(relatedGroups, [ - {label:'Entity', key:'entity'}, + {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream'}, {label:'Type', key:'type'}, {label:'Events', key:'count'}, @@ -657,24 +698,23 @@ async function refresh() { {label:'Detail', key:'detail'} ], 'policies'); const d = data.diagnostics || {}; - const context = data.event_context || {}; const quality = data.data_quality || {}; const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id])); const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})); document.getElementById('diagnostics').innerHTML = '

Stream Coverage

' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') + - '

Cross-Source Correlations

' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') + - '

Entities

' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') + + '

Cross-Source Correlations

' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') + + '

Entities

' + table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') + '

Profile Baseline Readiness

' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') + '

Data Quality

' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) + - '

Security Event Samples

' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) + - '

Top Sources

' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + + '

Security Event Samples

' + table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) + + '

Top Sources

' + table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}]) + '

Top Destinations

' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '

Top Policy IDs

' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '

Top Destination Ports

' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '

Top Source Ports

' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '

Top Services

' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + - '

Local-in Failures

' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip'}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]); + '

Local-in Failures

' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]); document.querySelectorAll('details[data-detail-id]').forEach(item => { if (openDetails.has(item.dataset.detailId)) item.open = true; }); document.querySelectorAll('[data-sort-table]').forEach(button => button.addEventListener('click', () => { const current=tableSort[button.dataset.sortTable]; tableSort[button.dataset.sortTable]={key:button.dataset.sortKey,direction:current && current.key===button.dataset.sortKey ? -current.direction : 1}; refresh(); })); document.querySelectorAll('.apply-suggested-profile').forEach(button => button.addEventListener('click', () => applySuggestedProfile(button.dataset.streamId))); diff --git a/src/fgai/event_context.py b/src/fgai/event_context.py index e1b8f05..b118e50 100644 --- a/src/fgai/event_context.py +++ b/src/fgai/event_context.py @@ -1,7 +1,9 @@ from __future__ import annotations +import ipaddress from collections import Counter, defaultdict +from .entities import ENTITY_FIELDS from .logs import THREAT_ACTIONS, is_utm_event from .models import LogEvent from .normalization import canonical_value @@ -11,6 +13,32 @@ def _entity(event: LogEvent) -> str: return event.src_ip or event.fields.get("user") or event.fields.get("username") or event.fields.get("hostname") or event.fields.get("source") or "unknown" +def _related_value(events: list[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str: + counts: Counter[str] = Counter() + excluded = exclude.lower() + for event in events: + for field in fields: + value = str(event.fields.get(field, "")).strip() + if not value or value in {"-", "unknown", "n/a"}: + continue + if excluded and value.lower() == excluded: + continue + counts[value] += 1 + return counts.most_common(1)[0][0] if counts else "" + + +def _display_label(entity: str, events: list[LogEvent]) -> dict[str, str]: + try: + ipaddress.ip_address(entity) + except ValueError: + return {"entity_display": entity, "entity_label": entity, "entity_detail": ""} + hostname = _related_value(events, ENTITY_FIELDS["host"], exclude=entity) + username = _related_value(events, ENTITY_FIELDS["user"], exclude=entity) + display = hostname or username or entity + label = f"{display} ({entity})" if display != entity else entity + return {"entity_display": display, "entity_label": label, "entity_detail": entity if display != entity else ""} + + def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sample_limit: int = 15) -> dict[str, object]: grouped: dict[str, list[LogEvent]] = defaultdict(list) field_presence: Counter[str] = Counter() @@ -24,6 +52,7 @@ def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sampl destinations = {event.dst_ip for event in source_events if event.dst_ip} source_profiles.append({ "entity": entity, "events": len(source_events), "utm_events": sum(is_utm_event(event) for event in source_events), + **_display_label(entity, source_events), "deny_or_threat_actions": sum(event.action in THREAT_ACTIONS for event in source_events), "distinct_destinations": len(destinations), "top_actions": [action for action, _ in actions.most_common(3)], }) @@ -31,7 +60,7 @@ def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sampl suspicious = [event for event in events if is_utm_event(event) or event.action in THREAT_ACTIONS or event.severity in {"critical", "high", "alert", "emergency"}] samples = [{ - "entity": _entity(event), "type": canonical_value(event.fields, "type"), "subtype": event.subtype, + "entity": _entity(event), **_display_label(_entity(event), [event]), "type": canonical_value(event.fields, "type"), "subtype": event.subtype, "action": event.action, "severity": event.severity, "dst": event.dst_ip or canonical_value(event.fields, "context"), "service": canonical_value(event.fields, "service"), "policyid": event.fields.get("policyid", ""), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), } for event in suspicious[:sample_limit]] diff --git a/src/fgai/incidents.py b/src/fgai/incidents.py index 3434a8e..73a72e2 100644 --- a/src/fgai/incidents.py +++ b/src/fgai/incidents.py @@ -13,6 +13,15 @@ from .models import AnomalyFinding def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, list[dict[str, object]]], correlations: list[dict[str, object]]) -> list[dict[str, object]]: """Build investigation units from any log source, not only network source IPs.""" groups: dict[str, dict[str, object]] = defaultdict(lambda: {"score": 0, "evidence": [], "fields": [], "correlations": [], "timeline": []}) + entity_labels = { + str(item.get("entity") or item.get("source_ip")): { + "entity_display": str(item.get("entity_display") or item.get("entity") or item.get("source_ip") or ""), + "entity_label": str(item.get("entity_label") or item.get("entity_display") or item.get("entity") or item.get("source_ip") or ""), + "entity_detail": str(item.get("entity_detail") or ""), + } + for item in correlations + if item.get("entity") or item.get("source_ip") + } for anomaly in anomalies: group = groups[anomaly.subject] group["score"] = max(int(group["score"]), anomaly.score) @@ -47,6 +56,7 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, incidents.append({ "id": incident_id, "entity": entity, + **entity_labels.get(entity, {"entity_display": entity, "entity_label": entity, "entity_detail": ""}), "entity_type": entity_type(entity), "score": score, "severity": severity, diff --git a/src/fgai/triage.py b/src/fgai/triage.py index e42967e..ed56dd8 100644 --- a/src/fgai/triage.py +++ b/src/fgai/triage.py @@ -18,6 +18,9 @@ def build_triage_queue( """Summarize raw findings into analyst-sized investigation candidates.""" groups: dict[str, dict[str, object]] = defaultdict(lambda: { "entity": "", + "entity_display": "", + "entity_label": "", + "entity_detail": "", "score": 0, "streams": set(), "detectors": Counter(), @@ -35,6 +38,9 @@ def build_triage_queue( continue group = groups[entity] group["entity"] = entity + group["entity_display"] = str(incident.get("entity_display") or group["entity_display"] or entity) + group["entity_label"] = str(incident.get("entity_label") or group["entity_label"] or group["entity_display"] or entity) + group["entity_detail"] = str(incident.get("entity_detail") or group["entity_detail"] or "") group["score"] = max(int(group["score"]), int(incident.get("score", 0) or 0)) group["incident_id"] = str(incident.get("id", "")) group["state"] = str(incident.get("lifecycle_status", "open") or "open") @@ -63,6 +69,9 @@ def build_triage_queue( continue group = groups[entity] group["entity"] = entity + group["entity_display"] = str(correlation.get("entity_display") or group["entity_display"] or entity) + group["entity_label"] = str(correlation.get("entity_label") or group["entity_label"] or group["entity_display"] or entity) + group["entity_detail"] = str(correlation.get("entity_detail") or group["entity_detail"] or "") streams = [str(item) for item in correlation.get("streams", []) if item] group["streams"].update(streams) group["correlated_streams"] = max(int(group["correlated_streams"]), len(streams)) @@ -96,6 +105,9 @@ def build_triage_queue( next_action = "follow confirmed incident response" queue.append({ "entity": group["entity"], + "entity_display": group["entity_display"] or group["entity"], + "entity_label": group["entity_label"] or group["entity_display"] or group["entity"], + "entity_detail": group["entity_detail"], "score": score, "severity": _severity(score), "state": group["state"], diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 13d5d1c..0372248 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -23,3 +23,14 @@ class CorrelationTests(unittest.TestCase): result = correlate_source_ips(events) self.assertEqual(result[0]["entity"], "alice") self.assertEqual(result[0]["entity_type"], "user") + + def test_ip_entity_prefers_hostname_display_label(self): + events = [ + parse_log_line("srcip=10.0.0.5 hostname=win01 fgai_stream=Windows action=login"), + parse_log_line("srcip=10.0.0.5 hostname=win01 fgai_stream=Firewall action=blocked"), + ] + result = correlate_source_ips(events) + self.assertEqual(result[0]["entity"], "10.0.0.5") + self.assertEqual(result[0]["entity_display"], "win01") + self.assertEqual(result[0]["entity_label"], "win01 (10.0.0.5)") + self.assertEqual(result[0]["entity_detail"], "10.0.0.5") diff --git a/tests/test_event_context.py b/tests/test_event_context.py index 51a3d32..0e2148a 100644 --- a/tests/test_event_context.py +++ b/tests/test_event_context.py @@ -15,6 +15,15 @@ class EventContextTests(unittest.TestCase): self.assertEqual(context["source_profiles"][0]["entity"], "10.0.0.2") self.assertEqual(len(context["security_event_samples"]), 1) + def test_ip_entity_prefers_hostname_display_label(self): + events = [ + parse_log_line("srcip=10.0.0.2 hostname=win01 dstip=8.8.8.8 type=utm action=blocked severity=high"), + ] + context = build_event_context(events) + self.assertEqual(context["source_profiles"][0]["entity"], "10.0.0.2") + self.assertEqual(context["source_profiles"][0]["entity_label"], "win01 (10.0.0.2)") + self.assertEqual(context["security_event_samples"][0]["entity_label"], "win01 (10.0.0.2)") + if __name__ == "__main__": unittest.main()