diff --git a/README.md b/README.md index 0a1989c..b4cbab6 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,20 @@ usable data is available, the dashboard keeps showing the last good findings, graphs, incidents, and correlations with a stale-data warning instead of going blank. +Graylog fetch mode controls how high-volume streams are read: + +- `raw`: fetch raw events up to `graylog_max_events_per_stream`. +- `aggregate`: use Graylog MCP `aggregate_messages` for total event volume, then + fetch only `graylog_raw_sample_events` raw events per stream for findings and + drill-down context. +- `auto`: use aggregate mode automatically when `graylog_max_events_per_stream` + is larger than `graylog_raw_sample_events`. + +For high EPS environments, keep `graylog_range_seconds` at 300, set +`graylog_fetch_mode` to `auto` or `aggregate`, and use a modest raw sample such +as 5000. The dashboard then shows aggregate event volume without forcing every +raw log line through MCP each poll. + ## Monitoring Export The dashboard also exposes Prometheus text metrics at: diff --git a/src/fgai/config.py b/src/fgai/config.py index 78de764..5bba9c3 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -12,8 +12,10 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_streams": [], "graylog_stream_profiles": [], "graylog_query": "*", + "graylog_fetch_mode": "auto", "graylog_range_seconds": 300, "graylog_max_events_per_stream": 5000, + "graylog_raw_sample_events": 5000, "baseline_retention_days": 14, "baseline_value_retention_days": 7, "baseline_max_values_per_field": 2000, @@ -66,7 +68,9 @@ class ConfigStore: current[key] = bool(value) elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: current[key] = value - elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: + elif key == "graylog_fetch_mode" and value in {"auto", "raw", "aggregate"}: + current[key] = value + elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: minimum = 60 if key == "graylog_range_seconds" else 1 current[key] = max(minimum, int(value)) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index f090ae8..9600e90 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -116,7 +116,7 @@ HTML = """