From 865c79a5be9ab03b66d9ff1ba6d9ff5c95635363208e8600c15ec4340d242579 Mon Sep 17 00:00:00 2001
From: larssand <larssand@LPW0JW6L1>
Date: Tue, 23 Jun 2026 22:56:54 +0200
Subject: [PATCH] Added data-quality health checks.

---
 src/fgai/dashboard.py      |  2 ++
 src/fgai/data_quality.py   | 11 +++++++++++
 src/fgai/monitor.py        |  2 ++
 tests/test_data_quality.py |  8 ++++++++
 4 files changed, 23 insertions(+)
 create mode 100644 src/fgai/data_quality.py
 create mode 100644 tests/test_data_quality.py

diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py
index e2f9eb9..45a4058 100644
--- a/src/fgai/dashboard.py
+++ b/src/fgai/dashboard.py
@@ -186,12 +186,14 @@ async function refresh() {
   ]);
   const d = data.diagnostics || {};
   const context = data.event_context || {};
+  const quality = data.data_quality || {};
   const profileReadiness = (data.profile_readiness || []).map(item => ({...item, stream_title: streamTitles[item.stream_id] || item.stream_id}));
   const correlations = data.cross_source_correlations || [];
   document.getElementById('diagnostics').innerHTML =
     '<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Source IP', key:'source_ip'}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}]) +
     '<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) +
     '<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', render:r => r.ready ? 'ready' : 'learning'}]) +
+    '<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
     '<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
     '<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
     '<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
diff --git a/src/fgai/data_quality.py b/src/fgai/data_quality.py
new file mode 100644
index 0000000..ad242d3
--- /dev/null
+++ b/src/fgai/data_quality.py
@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from .models import LogEvent
+
+
+def assess_data_quality(events: list[LogEvent], stream_status: dict[str, object]) -> dict[str, object]:
+    total = len(events)
+    missing_timestamp = sum(not (event.fields.get("eventtime") or event.fields.get("timestamp") or (event.fields.get("date") and event.fields.get("time"))) for event in events)
+    missing_source = sum(not event.src_ip or event.src_ip == "-" for event in events)
+    streams = stream_status.get("streams", []) if isinstance(stream_status.get("streams"), list) else []
+    return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]}
diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py
index 2b3ceca..b9e1962 100644
--- a/src/fgai/monitor.py
+++ b/src/fgai/monitor.py
@@ -14,6 +14,7 @@ from .graylog_mcp import GraylogMcpClient
 from .graylog_source import GraylogStreamSource
 from .history import HistoryStore
 from .incidents import build_incidents
+from .data_quality import assess_data_quality
 from .llm import ollama_dashboard_assessment
 from .logs import local_in_failures, read_events, summarize_events, top_field_values
 from .mitigation import parse_allowlist, suggest_block_candidates
@@ -134,6 +135,7 @@ def build_status(
         "feedback": feedback,
         "cross_source_correlations": correlations,
         "incidents": build_incidents(anomalies, field_deviations, correlations),
+        "data_quality": assess_data_quality(events, mcp_status),
         "anomalies": [
             {
                 "subject": finding.subject,
diff --git a/tests/test_data_quality.py b/tests/test_data_quality.py
new file mode 100644
index 0000000..97ad187
--- /dev/null
+++ b/tests/test_data_quality.py
@@ -0,0 +1,8 @@
+import unittest
+from fgai.data_quality import assess_data_quality
+from fgai.logs import parse_log_line
+
+class DataQualityTests(unittest.TestCase):
+    def test_reports_missing_source_and_timestamp(self):
+        quality = assess_data_quality([parse_log_line("srcip=10.0.0.1 timestamp=now"), parse_log_line("action=deny")], {})
+        self.assertEqual(quality["missing_source"], 1)