From 4d669be66d11591ce2085649019dcb3c80392eb3c7620bd39df87592228ab500 Mon Sep 17 00:00:00 2001 From: larssand Date: Tue, 30 Jun 2026 11:34:11 +0200 Subject: [PATCH] add agregated search --- README.md | 14 ++++ src/fgai/config.py | 6 +- src/fgai/dashboard.py | 13 ++-- src/fgai/graylog_aggregate.py | 114 ++++++++++++++++++++++++++++++++ src/fgai/monitor.py | 35 ++++++++-- tests/test_config.py | 7 ++ tests/test_graylog_aggregate.py | 45 +++++++++++++ 7 files changed, 225 insertions(+), 9 deletions(-) create mode 100644 src/fgai/graylog_aggregate.py create mode 100644 tests/test_graylog_aggregate.py diff --git a/README.md b/README.md index 0a1989c..b4cbab6 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,20 @@ usable data is available, the dashboard keeps showing the last good findings, graphs, incidents, and correlations with a stale-data warning instead of going blank. +Graylog fetch mode controls how high-volume streams are read: + +- `raw`: fetch raw events up to `graylog_max_events_per_stream`. +- `aggregate`: use Graylog MCP `aggregate_messages` for total event volume, then + fetch only `graylog_raw_sample_events` raw events per stream for findings and + drill-down context. +- `auto`: use aggregate mode automatically when `graylog_max_events_per_stream` + is larger than `graylog_raw_sample_events`. + +For high EPS environments, keep `graylog_range_seconds` at 300, set +`graylog_fetch_mode` to `auto` or `aggregate`, and use a modest raw sample such +as 5000. The dashboard then shows aggregate event volume without forcing every +raw log line through MCP each poll. + ## Monitoring Export The dashboard also exposes Prometheus text metrics at: diff --git a/src/fgai/config.py b/src/fgai/config.py index 78de764..5bba9c3 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -12,8 +12,10 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_streams": [], "graylog_stream_profiles": [], "graylog_query": "*", + "graylog_fetch_mode": "auto", "graylog_range_seconds": 300, "graylog_max_events_per_stream": 5000, + "graylog_raw_sample_events": 5000, "baseline_retention_days": 14, "baseline_value_retention_days": 7, "baseline_max_values_per_field": 2000, @@ -66,7 +68,9 @@ class ConfigStore: current[key] = bool(value) elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: current[key] = value - elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: + elif key == "graylog_fetch_mode" and value in {"auto", "raw", "aggregate"}: + current[key] = value + elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: minimum = 60 if key == "graylog_range_seconds" else 1 current[key] = max(minimum, int(value)) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index f090ae8..9600e90 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -116,7 +116,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Correlation Map

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

+

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration