Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added data-quality health checks.

Browse Source
This commit is contained in:
larssand
2026-06-23 22:56:54 +02:00
parent e3938c2581
commit 865c79a5be
4 changed files with 23 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/fgai/dashboard.py
View File

@@ -186,12 +186,14 @@ async function refresh() {
]);
const d = data.diagnostics || {};
const context = data.event_context || {};
const quality = data.data_quality || {};
const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id}));
const correlations = data.cross_source_correlations || [];
document.getElementById('diagnostics').innerHTML =
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Source IP', key:'source_ip'}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', render:r => r.ready ? 'ready' : 'learning'}]) +
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
'<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +

11
src/fgai/data_quality.py Normal file
View File

@@ -0,0 +1,11 @@
from __future__ import annotations
from .models import LogEvent
def assess_data_quality(events: list[LogEvent], stream_status: dict[str, object]) -> dict[str, object]:
total = len(events)
missing_timestamp = sum(not (event.fields.get("eventtime") or event.fields.get("timestamp") or (event.fields.get("date") and event.fields.get("time"))) for event in events)
missing_source = sum(not event.src_ip or event.src_ip == "-" for event in events)
streams = stream_status.get("streams", []) if isinstance(stream_status.get("streams"), list) else []
return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]}

2
src/fgai/monitor.py
View File

@@ -14,6 +14,7 @@ from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource
from .history import HistoryStore
from .incidents import build_incidents
from .data_quality import assess_data_quality
from .llm import ollama_dashboard_assessment
from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import parse_allowlist, suggest_block_candidates
@@ -134,6 +135,7 @@ def build_status(
"feedback": feedback,
"cross_source_correlations": correlations,
"incidents": build_incidents(anomalies, field_deviations, correlations),
"data_quality": assess_data_quality(events, mcp_status),
"anomalies": [
{
"subject": finding.subject,