diff --git a/src/fgai/config.py b/src/fgai/config.py index 79ea3d6..8115aa1 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -17,6 +17,8 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_range_seconds": 300, "graylog_max_events_per_stream": 5000, "graylog_raw_sample_events": 5000, + "graylog_mcp_call_timeout_seconds": 8, + "graylog_mcp_poll_timeout_seconds": 120, "baseline_retention_days": 14, "baseline_value_retention_days": 7, "baseline_max_values_per_field": 2000, @@ -71,9 +73,9 @@ class ConfigStore: current[key] = value elif key == "graylog_fetch_mode" and value in {"auto", "raw", "aggregate"}: current[key] = value - elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: + elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "graylog_raw_sample_events", "graylog_mcp_call_timeout_seconds", "graylog_mcp_poll_timeout_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: - minimum = 60 if key == "graylog_range_seconds" else 1 + minimum = 60 if key in {"graylog_range_seconds", "graylog_mcp_poll_timeout_seconds"} else 1 current[key] = max(minimum, int(value)) except (TypeError, ValueError): continue diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 64dbf33..6a2188e 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -184,7 +184,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Operator Guidance

Waiting for monitor data.

Correlation Map

Waiting for correlated entities.

Investigation Incidents

Anomalies

Recommendations

AI Assessment

LLM assessment disabled.

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

+

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

How To Use SignalScope

1. Normal workflow

  1. Settings: connect Graylog MCP and enable streams.
  2. Settings: apply recommended profiles for missing streams.
  3. Diagnostics: confirm raw samples, aggregate counts, and profile readiness.
  4. Overview: use Operator Guidance, incidents, trends, and correlation map.
  5. Findings: review only high-signal deviations first, then mark decisions.

2. Stream health

  • ready: profile exists, events are arriving, and baseline fields are ready.
  • learning: profile exists and events arrive, but baseline age or buckets are still too low.
  • missing_profile: stream is enabled but no profile exists. Apply or edit one.
  • no_events: stream is enabled but the current poll has no raw sample events.
  • partial_fetch: Graylog returned only part of the requested raw sample.

3. Ready fields

0/8 means 8 profile fields are tracked but none are mature enough yet. A field needs at least 12 baseline buckets and the configured Baseline training days before it is ready.

During learning, treat findings as signals to tune profiles, not as final alerts.

4. Profiles

  • Entity fields define who or what behavior is tracked, such as user, host, source IP, or application actor.
  • Baseline fields define the changing behavior to learn, such as action, event ID, service, URL, status, or counters.
  • Relationships learn pairs such as username to srcip or host to process.
  • Detectors add burst checks for auth failures, DNS queries, and deny actions.

5. Findings

  • Start with Triage Queue and incidents, not raw long tables.
  • Open evidence details before confirming a finding.
  • Use Expected for known behavior, False positive for bad signal, Confirmed for real investigation items.
  • Use expiry when a behavior is expected only temporarily.

6. High EPS / MCP

  • Use aggregate or auto fetch mode for high EPS streams.
  • Keep raw samples small enough for context; aggregate counts represent the full window.
  • Sample capped is normal in aggregate mode. Truncated raw mode means you may miss context.
  • If MCP is stale, the UI shows cached status so you can still inspect previous findings.

7. Ollama

  • Dashboard assessment summarizes current evidence.
  • Profile advisor maps unknown/custom fields and suggests relationships.
  • Ollama advice is constrained to fields discovered from Graylog; unknown fields are rejected.

8. What to fix first

  1. No streams enabled.
  2. Enabled streams with missing profiles.
  3. Enabled streams with zero raw events.
  4. Profiles stuck at 0 ready fields after the training window.
  5. Too many repeated findings without review feedback.
@@ -586,10 +586,14 @@ async function refresh() { `MCP fetch mode: ${esc(displayedFetchMode)}${mcp.status === 'refreshing' ? ' (refreshing, showing previous counters)' : ''}`, `MCP poll window: ${esc(mcp.range_seconds || configuration.graylog_range_seconds || 0)}s`, `MCP max events/stream: ${esc(mcp.max_events_per_stream || configuration.graylog_max_events_per_stream || 0)}`, + `MCP call timeout: ${esc(mcp.call_timeout_seconds || configuration.graylog_mcp_call_timeout_seconds || 0)}s`, + `MCP poll budget: ${esc(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 0)}s`, + mcp.poll_duration_seconds ? `MCP last poll duration: ${esc(mcp.poll_duration_seconds)}s` : '', Number(mcp.max_events_per_stream || configuration.graylog_max_events_per_stream || 0) > 100000 ? `MCP max events/stream is very high. Use aggregate mode with a smaller raw sample, for example max 5000 and raw sample 1000-5000.` : '', `MCP aggregate events: ${esc(displayedAggregateEvents)}`, `MCP raw sample events: ${esc(displayedRawEvents)}`, `MCP coverage: ${esc(displayedCoverage)}${mcp.partial_streams ? ` (${esc(mcp.partial_streams)} partial)` : ''}${mcp.truncated_streams ? ` (${esc(mcp.truncated_streams)} truncated)` : ''}${mcp.sample_limited_streams ? ` (${esc(mcp.sample_limited_streams)} sample capped)` : ''}`, + mcp.skipped_streams ? `${esc(mcp.skipped_streams)} stream(s) skipped because the MCP poll budget was reached.` : '', configuration.log_source === 'graylog_mcp' && mcp.status === 'no_streams_enabled' ? `No Graylog streams are enabled. Open Settings, Load streams, tick streams, then click Save streams.` : '', configuration.log_source === 'graylog_mcp' && mcp.status !== 'refreshing' && enabledStreams.length > 0 && displayedRawEvents === 0 ? `No raw events fetched from enabled streams. Check poll window, Graylog query, stream permissions, and Diagnostics -> Stream Coverage.` : '', enabledWithNoRawEvents.length && mcp.status !== 'refreshing' ? `${esc(enabledWithNoRawEvents.length)} enabled stream(s) returned zero raw events in ${esc(zeroRawWindow)}.` : '', @@ -903,6 +907,8 @@ async function applySuggestedProfile(streamId) { graylog_range_seconds: config.graylog_range_seconds || 300, graylog_max_events_per_stream: config.graylog_max_events_per_stream || 5000, graylog_raw_sample_events: config.graylog_raw_sample_events || 5000, + graylog_mcp_call_timeout_seconds: config.graylog_mcp_call_timeout_seconds || 8, + graylog_mcp_poll_timeout_seconds: config.graylog_mcp_poll_timeout_seconds || 120, graylog_field_mapping: config.graylog_field_mapping || '', baseline_training_days: config.baseline_training_days || 7, baseline_retention_days: config.baseline_retention_days || 14, diff --git a/src/fgai/graylog_source.py b/src/fgai/graylog_source.py index 5e7455e..ba1602a 100644 --- a/src/fgai/graylog_source.py +++ b/src/fgai/graylog_source.py @@ -94,7 +94,7 @@ class GraylogStreamSource: if self.stream: arguments["streams"] = [self.stream] events: list[LogEvent] = [] - page_size = 1_000 + page_size = min(5_000, max(1, int(max_events))) pages = 0 partial_error = "" while len(events) < max_events: diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index 4a71091..3aee78d 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -92,7 +92,7 @@ def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[st status = status_by_id.get(stream_id, {}) enabled = next((bool(item.get("enabled")) for item in configured if str(item.get("id", "")) == stream_id), False) events_fetched = int(status.get("events_fetched", 0) or 0) - health = "not_enabled" if not enabled else "partial_fetch" if status.get("partial") else "missing_profile" if not profile else "no_events" if events_fetched == 0 else "learning" if total_fields and ready_fields < total_fields else "ready" if total_fields else "profile_needs_fields" + health = "not_enabled" if not enabled else "poll_budget_skipped" if status.get("error") == "skipped_poll_budget" else "partial_fetch" if status.get("partial") else "missing_profile" if not profile else "no_events" if events_fetched == 0 else "learning" if total_fields and ready_fields < total_fields else "ready" if total_fields else "profile_needs_fields" rows.append({ "stream_id": stream_id, "stream_name": _stream_name(stream_id, stream_titles, profile), @@ -115,7 +115,7 @@ def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[st "aggregate_error": str(status.get("aggregate_error", "")), "error": str(status.get("aggregate_error", "") or status.get("error", "")), "health": health, - "health_detail": "No raw events returned for this stream in the current MCP poll window." if enabled and events_fetched == 0 else "", + "health_detail": str(status.get("health_detail", "")) or ("No raw events returned for this stream in the current MCP poll window." if enabled and events_fetched == 0 else ""), }) return rows @@ -165,15 +165,36 @@ def build_status( range_seconds = _range_seconds(runtime_values.get("graylog_range_seconds", 300)) max_events_per_stream = max(1, int(runtime_values.get("graylog_max_events_per_stream", 5000) or 5000)) raw_sample_events = max(1, int(runtime_values.get("graylog_raw_sample_events", 5000) or 5000)) + mcp_call_timeout = max(1, int(runtime_values.get("graylog_mcp_call_timeout_seconds", 8) or 8)) + mcp_poll_timeout = max(60, int(runtime_values.get("graylog_mcp_poll_timeout_seconds", 120) or 120)) fetch_mode = str(runtime_values.get("graylog_fetch_mode", "auto") or "auto") use_aggregate = fetch_mode == "aggregate" or (fetch_mode == "auto" and max_events_per_stream > raw_sample_events) aggregate_events_total = 0 - client = GraylogMcpClient(url, token, verify_tls=verify_tls) + poll_started_monotonic = time.monotonic() + poll_deadline = poll_started_monotonic + mcp_poll_timeout + client = GraylogMcpClient(url, token, timeout=mcp_call_timeout, verify_tls=verify_tls) probe_status = client.probe() discovery_store = FieldDiscoveryStore(history_path) if history_path else None catalog_fields_total = 0 for stream_config in stream_configs: stream_id = str(stream_config["id"]) + stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id) + if time.monotonic() >= poll_deadline: + stream_statuses.append({ + "stream_id": stream_id, + "stream_name": stream_name, + "source": "graylog_mcp", + "events_fetched": 0, + "aggregate_events": 0, + "pages": 0, + "partial": True, + "error": "skipped_poll_budget", + "truncated": False, + "latest_event_time": "", + "raw_sample_limit": min(raw_sample_events, 10_000) if use_aggregate else max_events_per_stream, + "health_detail": "Skipped because the MCP poll time budget was reached before this stream.", + }) + continue profile = stream_profiles.get(stream_id) profile_fields = ( str(getattr(profile, "entity_field", "")), @@ -184,7 +205,6 @@ def build_status( *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())), *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())), ) if profile else () - stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id) if discovery_store: try: catalog_fields_total += discovery_store.ingest_catalog(stream_id, stream_name, _graylog_fields_from_result(client.call_tool("list_fields", {"streams": [stream_id]}))) @@ -201,8 +221,11 @@ def build_status( sample_limited_streams = [item for item in stream_statuses if item.get("truncated") and use_aggregate] truncated_streams = [item for item in stream_statuses if item.get("truncated") and not use_aggregate] partial_streams = [item for item in stream_statuses if item.get("partial")] + skipped_streams = [item for item in stream_statuses if item.get("error") == "skipped_poll_budget"] aggregate_errors = [item for item in stream_statuses if item.get("aggregate_status") == "error"] warnings = [] + if skipped_streams: + warnings.append(f"{len(skipped_streams)} stream(s) skipped because the MCP poll time budget was reached.") if aggregate_errors: warnings.append(f"{len(aggregate_errors)} stream(s) returned aggregate MCP errors.") if partial_streams: @@ -216,11 +239,15 @@ def build_status( "raw_events_fetched": len(events), "aggregate_events": aggregate_events_total, "poll_completed_at": int(time.time()), + "poll_duration_seconds": round(time.monotonic() - poll_started_monotonic, 2), "fetch_mode": "aggregate" if use_aggregate else "raw", "range_seconds": range_seconds, "max_events_per_stream": max_events_per_stream, "raw_sample_events": raw_sample_events, + "call_timeout_seconds": mcp_call_timeout, + "poll_timeout_seconds": mcp_poll_timeout, "partial_streams": len(partial_streams), + "skipped_streams": len(skipped_streams), "sample_limited_streams": len(sample_limited_streams), "truncated_streams": len(truncated_streams), "aggregate_error_streams": len(aggregate_errors), diff --git a/tests/test_config.py b/tests/test_config.py index fff3171..4f40f32 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,6 +38,16 @@ class ConfigTests(unittest.TestCase): self.assertEqual(public["graylog_fetch_mode"], "aggregate") self.assertEqual(public["graylog_raw_sample_events"], 2500) + def test_graylog_mcp_timeout_settings_are_numeric_and_bounded(self): + with tempfile.TemporaryDirectory() as directory: + store = ConfigStore(str(Path(directory) / "config.json")) + public = store.update({"graylog_mcp_call_timeout_seconds": "0", "graylog_mcp_poll_timeout_seconds": "30"}) + self.assertEqual(public["graylog_mcp_call_timeout_seconds"], 1) + self.assertEqual(public["graylog_mcp_poll_timeout_seconds"], 60) + public = store.update({"graylog_mcp_call_timeout_seconds": "6", "graylog_mcp_poll_timeout_seconds": "180"}) + self.assertEqual(public["graylog_mcp_call_timeout_seconds"], 6) + self.assertEqual(public["graylog_mcp_poll_timeout_seconds"], 180) + def test_graylog_tls_verify_can_be_disabled(self): with tempfile.TemporaryDirectory() as directory: store = ConfigStore(str(Path(directory) / "config.json")) diff --git a/tests/test_graylog_source.py b/tests/test_graylog_source.py index 57d4ca3..1bbf57d 100644 --- a/tests/test_graylog_source.py +++ b/tests/test_graylog_source.py @@ -62,6 +62,11 @@ class GraylogSourceTests(unittest.TestCase): self.assertEqual(client.arguments["limit"], 2) self.assertTrue(status["truncated"]) + def test_uses_larger_single_page_for_raw_sample(self): + client = _Client() + GraylogStreamSource(client, "vpn").fetch(max_events=5000) + self.assertEqual(client.arguments["limit"], 5000) + def test_returns_partial_status_instead_of_raising_on_search_error(self): events, status = GraylogStreamSource(_ErrorClient(), "vpn").fetch() self.assertEqual(events, [])