diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 556b74b..470b17c 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -406,7 +406,7 @@ async function refresh() { const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id])); const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})); document.getElementById('diagnostics').innerHTML = - '

Stream Coverage

' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', key:'health'}, {label:'Error', render:r => esc(r.error || '-')}], 'stream-coverage') + + '

Stream Coverage

' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', key:'health'}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') + '

Cross-Source Correlations

' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') + '

Entities

' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') + '

Profile Baseline Readiness

' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') + diff --git a/src/fgai/graylog_aggregate.py b/src/fgai/graylog_aggregate.py index fba4806..4c0d306 100644 --- a/src/fgai/graylog_aggregate.py +++ b/src/fgai/graylog_aggregate.py @@ -113,52 +113,6 @@ class GraylogAggregateSource: "groupings": [], "metrics": [{"function": "count"}], }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - "group_by": [], - "metrics": ["count()"], - }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - "group_by": [], - "metrics": ["count"], - }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - "group_by": [], - "metrics": [{"function": "count"}], - }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - "groups": [], - "series": ["count()"], - }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - "groups": [], - "series": [{"function": "count"}], - }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - "limit": 1, - }, - { - "query": self.query, - "streams": [self.stream] if self.stream else [], - "range_seconds": max(1, int(range_seconds)), - }, ] variants = [*schema_variants, *fallback_variants] seen_variants: set[str] = set() diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index 2f70a4e..5485e0e 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -94,7 +94,9 @@ def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[st "latest_event_time": str(status.get("latest_event_time", "")), "truncated": bool(status.get("truncated")), "partial": bool(status.get("partial")), - "error": str(status.get("error", "") or status.get("aggregate_error", "")), + "raw_error": str(status.get("error", "")), + "aggregate_error": str(status.get("aggregate_error", "")), + "error": str(status.get("aggregate_error", "") or status.get("error", "")), "health": "not_enabled" if not enabled else "partial_fetch" if status.get("partial") else "missing_profile" if not profile else "no_events" if int(status.get("events_fetched", 0) or 0) == 0 else "learning" if total_fields and ready_fields < total_fields else "ready" if total_fields else "profile_needs_fields", }) return rows diff --git a/tests/test_graylog_aggregate.py b/tests/test_graylog_aggregate.py index aab99d4..45acef7 100644 --- a/tests/test_graylog_aggregate.py +++ b/tests/test_graylog_aggregate.py @@ -45,6 +45,20 @@ class GraylogAggregateTests(unittest.TestCase): self.assertEqual(status["aggregate_events"], 42) self.assertEqual(len(client.arguments), 2) + def test_fallbacks_only_use_graylog_supported_aggregate_fields(self): + client = _AggregateClient([ + {"result": {"isError": True, "content": [{"type": "text", "text": "bad count parens"}]}}, + {"result": {"isError": True, "content": [{"type": "text", "text": "bad count"}]}}, + {"result": {"isError": True, "content": [{"type": "text", "text": "bad object metric"}]}}, + ]) + + status = GraylogAggregateSource(client, "firewall").fetch_count() + + self.assertEqual(status["aggregate_status"], "error") + self.assertEqual(len(client.arguments), 3) + for arguments in client.arguments: + self.assertEqual(set(arguments), {"query", "streams", "range_seconds", "groupings", "metrics"}) + def test_uses_tool_schema_to_avoid_unsupported_fields(self): schema = {"properties": {"query": {}, "streams": {}, "range_seconds": {}, "metrics": {}, "groupings": {}}} client = _AggregateClient([