Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix delete och ollama

Browse Source
This commit is contained in:
larssand
2026-07-06 23:10:42 +02:00
parent 469b2f14b4
commit e9e0ab8a46
5 changed files with 169 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/fgai/dashboard.py
View File

@@ -834,7 +834,7 @@ async function refresh() {
'<h3>Stream Coverage</h3>' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') +
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `<button type="button" class="delete-stream-profile small-action danger-action" title="Delete profile" data-stream-id="${esc(r.stream_id)}" data-stream-name="${esc(r.stream_title || r.profile_name || r.stream_id)}">Remove</button>` : '-'}], 'profile-readiness') +
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
'<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}]) +
@@ -1107,6 +1107,11 @@ document.getElementById('profileSuggestions').addEventListener('click', event =>
if (!applyButton) return;
applySuggestedProfile(applyButton.dataset.streamId);
});
document.getElementById('diagnostics').addEventListener('click', event => {
const deleteButton = event.target.closest('.delete-stream-profile');
if (!deleteButton) return;
deleteStreamProfile(deleteButton.dataset.streamId, deleteButton.dataset.streamName);
});
document.getElementById('streamPicker').addEventListener('change', event => {
const checkbox = event.target.closest('.graylog-stream');
if (!checkbox || !window.streamSelection[checkbox.dataset.id]) return;

119
src/fgai/llm.py
View File

@@ -106,9 +106,34 @@ def _compact_field_deviations(field_deviations: object, *, limit: int = 12) -> l
def _compact_event_context(event_context: object) -> dict[str, object]:
if not isinstance(event_context, dict):
return {}
source_profiles = []
for item in event_context.get("source_profiles", [])[:10] if isinstance(event_context.get("source_profiles"), list) else []:
if not isinstance(item, dict):
continue
source_profiles.append({
"entity": item.get("entity_label") or item.get("entity_display") or item.get("entity"),
"events": item.get("events"),
"utm_events": item.get("utm_events"),
"deny_or_threat_actions": item.get("deny_or_threat_actions"),
"distinct_destinations": item.get("distinct_destinations"),
"top_actions": item.get("top_actions", [])[:3] if isinstance(item.get("top_actions"), list) else [],
})
related_activity = []
for item in event_context.get("related_activity", [])[:12] if isinstance(event_context.get("related_activity"), list) else []:
if not isinstance(item, dict):
continue
related_activity.append({
"entity": item.get("entity_label") or item.get("entity_display") or item.get("entity"),
"stream": item.get("stream"),
"type": item.get("type"),
"action": item.get("action"),
"severity": item.get("severity"),
"destination": item.get("dst") or item.get("destination"),
"service": item.get("service"),
})
return {
"source_profiles": event_context.get("source_profiles", [])[:10] if isinstance(event_context.get("source_profiles"), list) else [],
"related_activity": event_context.get("related_activity", [])[:20] if isinstance(event_context.get("related_activity"), list) else [],
"source_profiles": source_profiles,
"related_activity": related_activity,
"entity_labels": event_context.get("entity_labels", {}) if isinstance(event_context.get("entity_labels"), dict) else {},
}
@@ -162,26 +187,104 @@ def _compact_profile_suggestion_for_dashboard(item: object) -> dict[str, object]
}
def _short_text(value: object, limit: int = 180) -> str:
text = str(value or "")
return text if len(text) <= limit else f"{text[:limit].rstrip()}..."
def _compact_anomaly(item: object) -> dict[str, object]:
if not isinstance(item, dict):
return {}
evidence = item.get("evidence", {}) if isinstance(item.get("evidence"), dict) else {}
return {
"entity": item.get("entity_label") or item.get("entity_display") or item.get("subject"),
"score": item.get("score"),
"severity": item.get("severity"),
"confidence": item.get("confidence"),
"reasons": [_short_text(reason, 140) for reason in item.get("reasons", [])[:3]] if isinstance(item.get("reasons"), list) else [],
"events": evidence.get("timed_events") or evidence.get("events"),
"distinct_dst_ports": evidence.get("distinct_dst_ports"),
"distinct_src_ports": evidence.get("distinct_src_ports"),
}
def _compact_recommendation(item: object) -> dict[str, object]:
if not isinstance(item, dict):
return {}
return {
"subject": item.get("entity_label") or item.get("entity_display") or item.get("subject"),
"score": item.get("score"),
"severity": item.get("severity"),
"title": item.get("title"),
"recommendation": _short_text(item.get("recommendation"), 220),
"related_policy_ids": item.get("related_policy_ids", [])[:5] if isinstance(item.get("related_policy_ids"), list) else [],
"related_services": item.get("related_services", [])[:5] if isinstance(item.get("related_services"), list) else [],
}
def _compact_correlation(item: object) -> dict[str, object]:
if not isinstance(item, dict):
return {}
return {
"entity": item.get("entity_label") or item.get("entity_display") or item.get("entity") or item.get("source_ip"),
"entity_type": item.get("entity_type"),
"streams": item.get("streams", [])[:6] if isinstance(item.get("streams"), list) else [],
"events": item.get("events"),
"security_events": item.get("security_events"),
"distinct_destinations": item.get("distinct_destinations"),
}
def _compact_incident(item: object) -> dict[str, object]:
if not isinstance(item, dict):
return {}
return {
"entity": item.get("entity_label") or item.get("entity_display") or item.get("entity"),
"entity_type": item.get("entity_type"),
"score": item.get("score"),
"severity": item.get("severity"),
"state": item.get("lifecycle_status"),
"streams": item.get("correlated_streams", [])[:6] if isinstance(item.get("correlated_streams"), list) else [],
"evidence": [_short_text(value, 160) for value in item.get("evidence", [])[:4]] if isinstance(item.get("evidence"), list) else [],
"first_seen": item.get("first_seen"),
"last_seen": item.get("last_seen"),
"field_deviations": item.get("field_deviations"),
}
def _compact_feedback(item: object) -> dict[str, object]:
if not isinstance(item, dict):
return {}
return {
"stream_id": item.get("stream_id"),
"entity": item.get("entity"),
"field": item.get("field"),
"status": item.get("status"),
}
def compact_dashboard_analysis(analysis: dict[str, object]) -> dict[str, object]:
return {
"summary": analysis.get("summary", {}),
"anomaly_summary": analysis.get("anomaly_summary", {}),
"baseline": analysis.get("baseline", {}),
"top_anomalies": analysis.get("anomalies", [])[:5],
"top_recommendations": analysis.get("recommendations", [])[:5],
"top_anomalies": [_compact_anomaly(item) for item in analysis.get("anomalies", [])[:5]],
"top_recommendations": [_compact_recommendation(item) for item in analysis.get("recommendations", [])[:5]],
"block_candidates": analysis.get("block_candidates", [])[:5],
"policy_findings": analysis.get("policy_findings", [])[:5],
"event_context": _compact_event_context(analysis.get("event_context", {})),
"diagnostics": analysis.get("diagnostics", {}),
"diagnostics": {
"data_quality": analysis.get("diagnostics", {}).get("data_quality", {}) if isinstance(analysis.get("diagnostics"), dict) else {},
},
"capabilities": _compact_mcp_capabilities(analysis.get("capabilities", {})),
"cross_source_correlations": analysis.get("cross_source_correlations", [])[:10],
"incidents": analysis.get("incidents", [])[:8],
"cross_source_correlations": [_compact_correlation(item) for item in analysis.get("cross_source_correlations", [])[:8]],
"incidents": [_compact_incident(item) for item in analysis.get("incidents", [])[:6]],
"profile_suggestions": [
_compact_profile_suggestion_for_dashboard(item)
for item in analysis.get("profile_suggestions", [])[:8]
],
"field_deviations": _compact_field_deviations(analysis.get("field_deviations", {})),
"feedback": analysis.get("feedback", [])[:10] if isinstance(analysis.get("feedback"), list) else [],
"feedback": [_compact_feedback(item) for item in analysis.get("feedback", [])[:8]] if isinstance(analysis.get("feedback"), list) else [],
}

10
src/fgai/monitor.py
View File

@@ -774,7 +774,15 @@ def monitor_loop(
cached = cached_status_with_error(status_cache_path, mcp)
if cached:
status = cached
status["llm_assessment"] = {"enabled": bool(effective_llm), "status": "cached" if last_llm_text else ("pending" if effective_llm else "disabled"), "generated_at": last_llm_at, "text": last_llm_text or ""}
status["llm_assessment"] = {
"enabled": bool(effective_llm),
"status": "cached" if last_llm_text else ("pending" if effective_llm else "disabled"),
"generated_at": last_llm_at,
"started_at": int(time.time()) if effective_llm and not last_llm_text else 0,
"model": effective_model or "",
"timeout_seconds": effective_llm_timeout if effective_llm else 0,
"text": last_llm_text or "",
}
if effective_llm:
if status_cache_path and isinstance(mcp, dict) and mcp.get("status") not in {"error", "refreshing"}:
StatusSnapshotStore(status_cache_path).save("last_good", status)

1
tests/test_dashboard.py
View File

@@ -39,6 +39,7 @@ class DashboardTests(unittest.TestCase):
self.assertIn("function deleteStreamProfile", HTML)
self.assertIn("delete-stream-profile", HTML)
self.assertIn("danger-action", HTML)
self.assertIn("profile-readiness", HTML)
def test_dashboard_has_direct_stream_selection_save(self):
self.assertIn("Save streams", HTML)

43
tests/test_llm.py
View File

@@ -52,11 +52,52 @@ class LlmTests(unittest.TestCase):
self.assertEqual(compact["capabilities"]["graylog_mcp"]["status"], "connected")
self.assertNotIn("streams", compact["capabilities"]["graylog_mcp"])
self.assertEqual(len(compact["event_context"]["source_profiles"]), 10)
self.assertEqual(len(compact["event_context"]["related_activity"]), 20)
self.assertEqual(len(compact["event_context"]["related_activity"]), 12)
self.assertEqual(compact["field_deviations"][0]["entity"], "alice")
self.assertNotIn("sample_events", compact["field_deviations"][0])
self.assertNotIn("graylog_query", str(compact))
def test_dashboard_compaction_strips_incident_and_correlation_samples(self):
status = {
"incidents": [
{
"entity": "10.0.0.5",
"score": 90,
"severity": "critical",
"correlated_streams": ["Firewall", "Proxy"],
"evidence": ["large burst of blocked traffic"],
"timeline": [{"message": "raw incident evidence", "graylog_query": "srcip:10.0.0.5"}],
}
],
"cross_source_correlations": [
{
"entity": "10.0.0.5",
"streams": ["Firewall", "Proxy"],
"events": 20,
"samples": [{"message": "raw correlation evidence", "graylog_query": "srcip:10.0.0.5"}],
}
],
"anomalies": [
{
"subject": "10.0.0.5",
"score": 90,
"severity": "critical",
"reasons": ["burst"],
"evidence": {"timed_events": 100, "sample_events": [{"message": "raw anomaly"}]},
}
],
}
compact = compact_dashboard_analysis(status)
compact_text = str(compact)
self.assertNotIn("timeline", compact_text)
self.assertNotIn("samples", compact_text)
self.assertNotIn("sample_events", compact_text)
self.assertNotIn("graylog_query", compact_text)
self.assertNotIn("raw incident evidence", compact_text)
self.assertEqual(compact["incidents"][0]["entity"], "10.0.0.5")
if __name__ == "__main__":
unittest.main()