diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 6fa4780..42ebb54 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -834,7 +834,7 @@ async function refresh() { '

Stream Coverage

' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') + '

Cross-Source Correlations

' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') + '

Entities

' + table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') + - '

Profile Baseline Readiness

' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') + + '

Profile Baseline Readiness

' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `` : '-'}], 'profile-readiness') + '

Data Quality

' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) + '

Security Event Samples

' + table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) + '

Top Sources

' + table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}]) + @@ -1107,6 +1107,11 @@ document.getElementById('profileSuggestions').addEventListener('click', event => if (!applyButton) return; applySuggestedProfile(applyButton.dataset.streamId); }); +document.getElementById('diagnostics').addEventListener('click', event => { + const deleteButton = event.target.closest('.delete-stream-profile'); + if (!deleteButton) return; + deleteStreamProfile(deleteButton.dataset.streamId, deleteButton.dataset.streamName); +}); document.getElementById('streamPicker').addEventListener('change', event => { const checkbox = event.target.closest('.graylog-stream'); if (!checkbox || !window.streamSelection[checkbox.dataset.id]) return; diff --git a/src/fgai/llm.py b/src/fgai/llm.py index 531bcec..656d763 100644 --- a/src/fgai/llm.py +++ b/src/fgai/llm.py @@ -106,9 +106,34 @@ def _compact_field_deviations(field_deviations: object, *, limit: int = 12) -> l def _compact_event_context(event_context: object) -> dict[str, object]: if not isinstance(event_context, dict): return {} + source_profiles = [] + for item in event_context.get("source_profiles", [])[:10] if isinstance(event_context.get("source_profiles"), list) else []: + if not isinstance(item, dict): + continue + source_profiles.append({ + "entity": item.get("entity_label") or item.get("entity_display") or item.get("entity"), + "events": item.get("events"), + "utm_events": item.get("utm_events"), + "deny_or_threat_actions": item.get("deny_or_threat_actions"), + "distinct_destinations": item.get("distinct_destinations"), + "top_actions": item.get("top_actions", [])[:3] if isinstance(item.get("top_actions"), list) else [], + }) + related_activity = [] + for item in event_context.get("related_activity", [])[:12] if isinstance(event_context.get("related_activity"), list) else []: + if not isinstance(item, dict): + continue + related_activity.append({ + "entity": item.get("entity_label") or item.get("entity_display") or item.get("entity"), + "stream": item.get("stream"), + "type": item.get("type"), + "action": item.get("action"), + "severity": item.get("severity"), + "destination": item.get("dst") or item.get("destination"), + "service": item.get("service"), + }) return { - "source_profiles": event_context.get("source_profiles", [])[:10] if isinstance(event_context.get("source_profiles"), list) else [], - "related_activity": event_context.get("related_activity", [])[:20] if isinstance(event_context.get("related_activity"), list) else [], + "source_profiles": source_profiles, + "related_activity": related_activity, "entity_labels": event_context.get("entity_labels", {}) if isinstance(event_context.get("entity_labels"), dict) else {}, } @@ -162,26 +187,104 @@ def _compact_profile_suggestion_for_dashboard(item: object) -> dict[str, object] } +def _short_text(value: object, limit: int = 180) -> str: + text = str(value or "") + return text if len(text) <= limit else f"{text[:limit].rstrip()}..." + + +def _compact_anomaly(item: object) -> dict[str, object]: + if not isinstance(item, dict): + return {} + evidence = item.get("evidence", {}) if isinstance(item.get("evidence"), dict) else {} + return { + "entity": item.get("entity_label") or item.get("entity_display") or item.get("subject"), + "score": item.get("score"), + "severity": item.get("severity"), + "confidence": item.get("confidence"), + "reasons": [_short_text(reason, 140) for reason in item.get("reasons", [])[:3]] if isinstance(item.get("reasons"), list) else [], + "events": evidence.get("timed_events") or evidence.get("events"), + "distinct_dst_ports": evidence.get("distinct_dst_ports"), + "distinct_src_ports": evidence.get("distinct_src_ports"), + } + + +def _compact_recommendation(item: object) -> dict[str, object]: + if not isinstance(item, dict): + return {} + return { + "subject": item.get("entity_label") or item.get("entity_display") or item.get("subject"), + "score": item.get("score"), + "severity": item.get("severity"), + "title": item.get("title"), + "recommendation": _short_text(item.get("recommendation"), 220), + "related_policy_ids": item.get("related_policy_ids", [])[:5] if isinstance(item.get("related_policy_ids"), list) else [], + "related_services": item.get("related_services", [])[:5] if isinstance(item.get("related_services"), list) else [], + } + + +def _compact_correlation(item: object) -> dict[str, object]: + if not isinstance(item, dict): + return {} + return { + "entity": item.get("entity_label") or item.get("entity_display") or item.get("entity") or item.get("source_ip"), + "entity_type": item.get("entity_type"), + "streams": item.get("streams", [])[:6] if isinstance(item.get("streams"), list) else [], + "events": item.get("events"), + "security_events": item.get("security_events"), + "distinct_destinations": item.get("distinct_destinations"), + } + + +def _compact_incident(item: object) -> dict[str, object]: + if not isinstance(item, dict): + return {} + return { + "entity": item.get("entity_label") or item.get("entity_display") or item.get("entity"), + "entity_type": item.get("entity_type"), + "score": item.get("score"), + "severity": item.get("severity"), + "state": item.get("lifecycle_status"), + "streams": item.get("correlated_streams", [])[:6] if isinstance(item.get("correlated_streams"), list) else [], + "evidence": [_short_text(value, 160) for value in item.get("evidence", [])[:4]] if isinstance(item.get("evidence"), list) else [], + "first_seen": item.get("first_seen"), + "last_seen": item.get("last_seen"), + "field_deviations": item.get("field_deviations"), + } + + +def _compact_feedback(item: object) -> dict[str, object]: + if not isinstance(item, dict): + return {} + return { + "stream_id": item.get("stream_id"), + "entity": item.get("entity"), + "field": item.get("field"), + "status": item.get("status"), + } + + def compact_dashboard_analysis(analysis: dict[str, object]) -> dict[str, object]: return { "summary": analysis.get("summary", {}), "anomaly_summary": analysis.get("anomaly_summary", {}), "baseline": analysis.get("baseline", {}), - "top_anomalies": analysis.get("anomalies", [])[:5], - "top_recommendations": analysis.get("recommendations", [])[:5], + "top_anomalies": [_compact_anomaly(item) for item in analysis.get("anomalies", [])[:5]], + "top_recommendations": [_compact_recommendation(item) for item in analysis.get("recommendations", [])[:5]], "block_candidates": analysis.get("block_candidates", [])[:5], "policy_findings": analysis.get("policy_findings", [])[:5], "event_context": _compact_event_context(analysis.get("event_context", {})), - "diagnostics": analysis.get("diagnostics", {}), + "diagnostics": { + "data_quality": analysis.get("diagnostics", {}).get("data_quality", {}) if isinstance(analysis.get("diagnostics"), dict) else {}, + }, "capabilities": _compact_mcp_capabilities(analysis.get("capabilities", {})), - "cross_source_correlations": analysis.get("cross_source_correlations", [])[:10], - "incidents": analysis.get("incidents", [])[:8], + "cross_source_correlations": [_compact_correlation(item) for item in analysis.get("cross_source_correlations", [])[:8]], + "incidents": [_compact_incident(item) for item in analysis.get("incidents", [])[:6]], "profile_suggestions": [ _compact_profile_suggestion_for_dashboard(item) for item in analysis.get("profile_suggestions", [])[:8] ], "field_deviations": _compact_field_deviations(analysis.get("field_deviations", {})), - "feedback": analysis.get("feedback", [])[:10] if isinstance(analysis.get("feedback"), list) else [], + "feedback": [_compact_feedback(item) for item in analysis.get("feedback", [])[:8]] if isinstance(analysis.get("feedback"), list) else [], } diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index b45786c..e8abbfd 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -774,7 +774,15 @@ def monitor_loop( cached = cached_status_with_error(status_cache_path, mcp) if cached: status = cached - status["llm_assessment"] = {"enabled": bool(effective_llm), "status": "cached" if last_llm_text else ("pending" if effective_llm else "disabled"), "generated_at": last_llm_at, "text": last_llm_text or ""} + status["llm_assessment"] = { + "enabled": bool(effective_llm), + "status": "cached" if last_llm_text else ("pending" if effective_llm else "disabled"), + "generated_at": last_llm_at, + "started_at": int(time.time()) if effective_llm and not last_llm_text else 0, + "model": effective_model or "", + "timeout_seconds": effective_llm_timeout if effective_llm else 0, + "text": last_llm_text or "", + } if effective_llm: if status_cache_path and isinstance(mcp, dict) and mcp.get("status") not in {"error", "refreshing"}: StatusSnapshotStore(status_cache_path).save("last_good", status) diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 110abc3..5beaef7 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -39,6 +39,7 @@ class DashboardTests(unittest.TestCase): self.assertIn("function deleteStreamProfile", HTML) self.assertIn("delete-stream-profile", HTML) self.assertIn("danger-action", HTML) + self.assertIn("profile-readiness", HTML) def test_dashboard_has_direct_stream_selection_save(self): self.assertIn("Save streams", HTML) diff --git a/tests/test_llm.py b/tests/test_llm.py index d2ca54c..6e7edb1 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -52,11 +52,52 @@ class LlmTests(unittest.TestCase): self.assertEqual(compact["capabilities"]["graylog_mcp"]["status"], "connected") self.assertNotIn("streams", compact["capabilities"]["graylog_mcp"]) self.assertEqual(len(compact["event_context"]["source_profiles"]), 10) - self.assertEqual(len(compact["event_context"]["related_activity"]), 20) + self.assertEqual(len(compact["event_context"]["related_activity"]), 12) self.assertEqual(compact["field_deviations"][0]["entity"], "alice") self.assertNotIn("sample_events", compact["field_deviations"][0]) self.assertNotIn("graylog_query", str(compact)) + def test_dashboard_compaction_strips_incident_and_correlation_samples(self): + status = { + "incidents": [ + { + "entity": "10.0.0.5", + "score": 90, + "severity": "critical", + "correlated_streams": ["Firewall", "Proxy"], + "evidence": ["large burst of blocked traffic"], + "timeline": [{"message": "raw incident evidence", "graylog_query": "srcip:10.0.0.5"}], + } + ], + "cross_source_correlations": [ + { + "entity": "10.0.0.5", + "streams": ["Firewall", "Proxy"], + "events": 20, + "samples": [{"message": "raw correlation evidence", "graylog_query": "srcip:10.0.0.5"}], + } + ], + "anomalies": [ + { + "subject": "10.0.0.5", + "score": 90, + "severity": "critical", + "reasons": ["burst"], + "evidence": {"timed_events": 100, "sample_events": [{"message": "raw anomaly"}]}, + } + ], + } + + compact = compact_dashboard_analysis(status) + compact_text = str(compact) + + self.assertNotIn("timeline", compact_text) + self.assertNotIn("samples", compact_text) + self.assertNotIn("sample_events", compact_text) + self.assertNotIn("graylog_query", compact_text) + self.assertNotIn("raw incident evidence", compact_text) + self.assertEqual(compact["incidents"][0]["entity"], "10.0.0.5") + if __name__ == "__main__": unittest.main()