stream_coverage i status-API.

README.md

@@ -68,6 +68,11 @@ checkboxes decide which streams are monitored. Click `Edit profile` on one strea
 to load its fields and edit only that stream's profile; saving with no active
 profile editor leaves existing profiles unchanged.
 
+When many Graylog streams are available, use `Diagnostics -> Stream Coverage` to
+see which streams are enabled, which have profiles, how many profile fields are
+baseline-ready, how many events were fetched, and whether a stream is `ready`,
+`learning`, `missing_profile`, `no_events`, or `not_enabled`.
+
 Enabled streams are normalized through the same event model. Stream profiles
 define the entity, timestamp, categorical, and numeric fields used for baselines.
 The dashboard and Ollama then correlate behavior across sources, for example a

ROADMAP.md

@@ -90,6 +90,7 @@ Acceptance: deployment, restart, upgrade, backup, and monitoring have documented
 Goal: optimize the UI for security investigation rather than raw tables.
 
 - [x] Separate stream enablement from one-profile-at-a-time profile editing.
+- [x] Add stream coverage diagnostics for enabled/profiled/ready/no-event streams.
 - [ ] Replace remaining long tables with compact incident and entity cards where appropriate.
 - [ ] Add incident filters for stream, severity, entity type, review state, and time range.
 - [ ] Add baseline versus current charts per selected entity and field.
@@ -104,6 +105,7 @@ Acceptance: common triage can be completed from the dashboard without manually p
 Goal: add log sources and outputs without adding source-specific logic everywhere.
 
 - [ ] Define versioned stream-profile templates for FortiGate, Windows, DNS/AdGuard, Nginx, Squid, VPN, and Proxmox.
+- [x] Add inventory-style stream coverage to guide which streams need profiles before templates are added.
 - [ ] Add import/export for profile templates and detector settings.
 - [ ] Separate source adapters, normalizers, detectors, enrichers, and output adapters into explicit extension interfaces.
 - [ ] Add optional webhook/SIEM ticket output for confirmed high-severity incidents.

src/fgai/dashboard.py

@@ -114,6 +114,9 @@ async function refresh() {
   const threat = (data.capabilities || {}).threat_intel || {};
   const mcp = (data.capabilities || {}).graylog_mcp || {};
   const configuration = data.configuration || {};
+  const streamCoverage = data.stream_coverage || [];
+  const enabledStreams = streamCoverage.filter(item => item.enabled);
+  const streamsMissingProfile = enabledStreams.filter(item => !item.profile_ready).length;
   const rawCorrelations = data.cross_source_correlations || [];
   const correlationsCached = rawCorrelations.length === 0 && uiCache.correlations.length > 0;
   const correlations = rawCorrelations.length ? rawCorrelations : uiCache.correlations;
@@ -143,7 +146,7 @@ async function refresh() {
     `Baseline sources ready: ${esc((data.baseline || {}).sources_ready || 0)}`
   ].join('<br>');
   document.getElementById('health').innerHTML = [
-    metric('Baseline sources ready', baseline.sources_ready || 0), metric('MCP events fetched', mcp.events_fetched || 0), metric('Profiles active', (data.stream_profiles || []).length), metric('Correlated entities', correlations.length)
+    metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Correlated entities', correlations.length)
   ].join('');
   const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
   document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`;
@@ -223,6 +226,7 @@ async function refresh() {
   const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id]));
   const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id}));
   document.getElementById('diagnostics').innerHTML =
+    '<h3>Stream Coverage</h3>' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Events', key:'events_fetched'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', key:'health'}], 'stream-coverage') +
     '<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
     '<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +
     '<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +

src/fgai/monitor.py

@@ -51,6 +51,47 @@ def _range_seconds(value: object) -> int:
         return 3600
 
 
+def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[str, object], stream_status: dict[str, object], profile_readiness: list[dict[str, object]], stream_titles: dict[str, str]) -> list[dict[str, object]]:
+    configured = [
+        item for item in runtime_values.get("graylog_streams", [])
+        if isinstance(item, dict) and item.get("id")
+    ]
+    status_by_id = {
+        str(item.get("stream_id", "")): item
+        for item in stream_status.get("streams", [])
+        if isinstance(item, dict) and item.get("stream_id")
+    }
+    readiness_by_stream: dict[str, list[dict[str, object]]] = {}
+    for item in profile_readiness:
+        readiness_by_stream.setdefault(str(item.get("stream_id", "")), []).append(item)
+    ids = list(dict.fromkeys([str(item.get("id", "")) for item in configured] + list(stream_profiles) + list(status_by_id)))
+    rows = []
+    for stream_id in ids:
+        profile = stream_profiles.get(stream_id)
+        readiness = readiness_by_stream.get(stream_id, [])
+        ready_fields = sum(1 for item in readiness if item.get("ready"))
+        total_fields = len(readiness)
+        status = status_by_id.get(stream_id, {})
+        enabled = next((bool(item.get("enabled")) for item in configured if str(item.get("id", "")) == stream_id), False)
+        rows.append({
+            "stream_id": stream_id,
+            "stream_name": _stream_name(stream_id, stream_titles, profile),
+            "enabled": enabled,
+            "profile": _profile_name(stream_id, stream_titles, profile) if profile else "",
+            "profile_ready": bool(profile),
+            "entity_field": str(getattr(profile, "entity_field", "")) if profile else "",
+            "tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) if profile else 0,
+            "ready_fields": ready_fields,
+            "total_fields": total_fields,
+            "readiness": f"{ready_fields}/{total_fields}" if total_fields else "0/0",
+            "events_fetched": int(status.get("events_fetched", 0) or 0),
+            "latest_event_time": str(status.get("latest_event_time", "")),
+            "truncated": bool(status.get("truncated")),
+            "health": "not_enabled" if not enabled else "missing_profile" if not profile else "no_events" if int(status.get("events_fetched", 0) or 0) == 0 else "learning" if total_fields and ready_fields < total_fields else "ready" if total_fields else "profile_needs_fields",
+        })
+    return rows
+
+
 def build_status(
     log_path: str,
     *,
@@ -146,6 +187,7 @@ def build_status(
         }
         for item in profile_readiness
     ]
+    stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
     intel_ips = sorted(
         {
             ip
@@ -184,6 +226,7 @@ def build_status(
         "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
         "configuration": runtime_config,
         "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
+        "stream_coverage": stream_coverage,
         "profile_readiness": profile_readiness,
         "diagnostics": {
             "top_source_ips": top_field_values(events, "srcip", limit=10),

tests/test_monitor.py

@@ -54,6 +54,29 @@ class MonitorTests(unittest.TestCase):
             self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile")
             self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile")
 
+    def test_stream_coverage_marks_missing_profiles(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            config_path = Path(tmp) / "config.json"
+            config_path.write_text(
+                json.dumps(
+                    {
+                        "graylog_streams": [
+                            {"id": "firewall", "title": "Firewall", "enabled": True},
+                            {"id": "dns", "title": "DNS", "enabled": True},
+                        ],
+                        "graylog_stream_profiles": [{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}],
+                    }
+                ),
+                encoding="utf-8",
+            )
+            with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[{"stream_id": "firewall", "field": "action", "buckets": 12, "ready": True}]):
+                status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path))
+
+            coverage = {item["stream_id"]: item for item in status["stream_coverage"]}
+            self.assertEqual(coverage["firewall"]["health"], "no_events")
+            self.assertEqual(coverage["firewall"]["readiness"], "1/1")
+            self.assertEqual(coverage["dns"]["health"], "missing_profile")
+
     def test_add_llm_assessment_records_error_without_ollama(self):
         status = {"summary": {}, "anomalies": []}