diff --git a/README.md b/README.md index 0a1989c..b4cbab6 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,20 @@ usable data is available, the dashboard keeps showing the last good findings, graphs, incidents, and correlations with a stale-data warning instead of going blank. +Graylog fetch mode controls how high-volume streams are read: + +- `raw`: fetch raw events up to `graylog_max_events_per_stream`. +- `aggregate`: use Graylog MCP `aggregate_messages` for total event volume, then + fetch only `graylog_raw_sample_events` raw events per stream for findings and + drill-down context. +- `auto`: use aggregate mode automatically when `graylog_max_events_per_stream` + is larger than `graylog_raw_sample_events`. + +For high EPS environments, keep `graylog_range_seconds` at 300, set +`graylog_fetch_mode` to `auto` or `aggregate`, and use a modest raw sample such +as 5000. The dashboard then shows aggregate event volume without forcing every +raw log line through MCP each poll. + ## Monitoring Export The dashboard also exposes Prometheus text metrics at: diff --git a/src/fgai/config.py b/src/fgai/config.py index 78de764..5bba9c3 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -12,8 +12,10 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_streams": [], "graylog_stream_profiles": [], "graylog_query": "*", + "graylog_fetch_mode": "auto", "graylog_range_seconds": 300, "graylog_max_events_per_stream": 5000, + "graylog_raw_sample_events": 5000, "baseline_retention_days": 14, "baseline_value_retention_days": 7, "baseline_max_values_per_field": 2000, @@ -66,7 +68,9 @@ class ConfigStore: current[key] = bool(value) elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: current[key] = value - elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: + elif key == "graylog_fetch_mode" and value in {"auto", "raw", "aggregate"}: + current[key] = value + elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: minimum = 60 if key == "graylog_range_seconds" else 1 current[key] = max(minimum, int(value)) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index f090ae8..9600e90 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -116,7 +116,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Correlation Map

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

+

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration