diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 520ba1b..84ea3f5 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -556,6 +556,7 @@ class BaselineStore: value_retention_days: int = DEFAULT_VALUE_RETENTION_DAYS, max_values_per_field: int = DEFAULT_MAX_VALUES_PER_FIELD, vacuum: bool = False, + include_rows: bool = False, ) -> dict[str, object]: now = int(time.time()) bucket_cutoff = now - max(1, retention_days) * 86400 @@ -597,35 +598,39 @@ class BaselineStore: if vacuum: with self._connect() as connection: connection.execute("vacuum") - stats = self.stats() + stats = self.stats(include_rows=include_rows) return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, **stats} - def stats(self) -> dict[str, object]: + def stats(self, *, include_rows: bool = False) -> dict[str, object]: with self._connect() as connection: - tables = ( - "seen_events", - "source_buckets", - "source_values", - "profile_seen_events", - "profile_buckets", - "profile_temporal_buckets", - "profile_detector_buckets", - "profile_detector_temporal_buckets", - "profile_values", - ) - rows = {table: int(connection.execute(f"select count(*) from {table}").fetchone()[0]) for table in tables} page_size = int(connection.execute("pragma page_size").fetchone()[0]) page_count = int(connection.execute("pragma page_count").fetchone()[0]) freelist_count = int(connection.execute("pragma freelist_count").fetchone()[0]) + rows = {} + if include_rows: + tables = ( + "seen_events", + "source_buckets", + "source_values", + "profile_seen_events", + "profile_buckets", + "profile_temporal_buckets", + "profile_detector_buckets", + "profile_detector_temporal_buckets", + "profile_values", + ) + rows = {table: int(connection.execute(f"select count(*) from {table}").fetchone()[0]) for table in tables} size_bytes = self.path.stat().st_size if self.path.exists() else 0 - return { + stats: dict[str, object] = { "size_bytes": size_bytes, "page_size": page_size, "page_count": page_count, "freelist_count": freelist_count, "reclaimable_bytes": freelist_count * page_size, - "rows": rows, } + if include_rows: + stats["rows"] = rows + return stats def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] diff --git a/src/fgai/cli.py b/src/fgai/cli.py index 71f84a0..0b3ea71 100644 --- a/src/fgai/cli.py +++ b/src/fgai/cli.py @@ -245,6 +245,7 @@ def baseline_maintenance(args: argparse.Namespace) -> int: value_retention_days=args.value_retention_days, max_values_per_field=args.max_values_per_field, vacuum=args.vacuum, + include_rows=True, ) _print_json(result) return 0 diff --git a/src/fgai/config.py b/src/fgai/config.py index 3324ba5..a0c2de9 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -18,7 +18,7 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_max_events_per_stream": 5000, "graylog_raw_sample_events": 5000, "graylog_mcp_call_timeout_seconds": 8, - "graylog_mcp_poll_timeout_seconds": 120, + "graylog_mcp_poll_timeout_seconds": 240, "baseline_retention_days": 7, "baseline_value_retention_days": 3, "baseline_max_values_per_field": 500, diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 84e9e3f..80c52e9 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -234,7 +234,7 @@ function mcpCapability(mcp, configuration, pollRunningSeconds=0) { if (status === 'connected') return {state: 'on', detail: 'connected'}; if (status === 'partial') return {state: 'warn', detail: 'partial'}; if (status === 'refreshing') { - const pollBudget = Number(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 120); + const pollBudget = Number(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 240); if (pollRunningSeconds > pollBudget) return {state: 'warn', detail: `poll over budget ${pollRunningSeconds}s`}; const previous = String(mcp.previous_status || ''); if (['connected', 'partial'].includes(previous) || Number(mcp.previous_events_fetched || mcp.events_fetched || 0) > 0 || Number(mcp.previous_aggregate_events || mcp.aggregate_events || 0) > 0) { @@ -626,7 +626,7 @@ async function refresh() { `MCP status: ${esc(mcp.status || 'unknown')}`, mcp.status === 'refreshing' && pollStartedAt ? `MCP poll running: ${esc(pollRunningSeconds)}s` : '', pollCompletedAt ? `Last completed MCP poll: ${esc(pollCompletedAge)}s ago` : '', - mcp.status === 'refreshing' && pollRunningSeconds > Number(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 120) ? `MCP poll has been refreshing for ${esc(pollRunningSeconds)}s, which is over the configured poll budget. Restart monitor or lower enabled streams/sample size if this keeps happening.` : '', + mcp.status === 'refreshing' && pollRunningSeconds > Number(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 240) ? `MCP poll has been refreshing for ${esc(pollRunningSeconds)}s, which is over the configured poll budget. Restart monitor or lower enabled streams/sample size if this keeps happening.` : '', mcp.error ? `MCP error: ${esc(mcp.error)}` : '', `Enabled streams: ${esc(enabledStreams.length)}`, `MCP fetch mode: ${esc(displayedFetchMode)}${mcp.status === 'refreshing' ? ' (refreshing, showing previous counters)' : ''}`, diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index e8363f4..740b5ed 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -254,7 +254,7 @@ def build_status( max_events_per_stream = max(1, int(runtime_values.get("graylog_max_events_per_stream", 5000) or 5000)) raw_sample_events = max(1, int(runtime_values.get("graylog_raw_sample_events", 5000) or 5000)) mcp_call_timeout = max(1, int(runtime_values.get("graylog_mcp_call_timeout_seconds", 8) or 8)) - mcp_poll_timeout = max(60, int(runtime_values.get("graylog_mcp_poll_timeout_seconds", 120) or 120)) + mcp_poll_timeout = max(60, int(runtime_values.get("graylog_mcp_poll_timeout_seconds", 240) or 240)) fetch_mode = str(runtime_values.get("graylog_fetch_mode", "auto") or "auto") use_aggregate = fetch_mode == "aggregate" or (fetch_mode == "auto" and max_events_per_stream > raw_sample_events) aggregate_events_total = 0 @@ -739,11 +739,11 @@ def monitor_loop( effective_llm = bool(runtime.get("llm_enabled")) if runtime else llm effective_model = str(runtime.get("llm_model") or llm_model or "") mcp_call_timeout = max(1, int(runtime.get("graylog_mcp_call_timeout_seconds", 8) or 8)) - mcp_poll_timeout = max(60, int(runtime.get("graylog_mcp_poll_timeout_seconds", 120) or 120)) + mcp_poll_timeout = max(60, int(runtime.get("graylog_mcp_poll_timeout_seconds", 240) or 240)) if runtime.get("log_source") == "graylog_mcp": write_refreshing_status(output, cache_path=status_cache_path, call_timeout_seconds=mcp_call_timeout, poll_timeout_seconds=mcp_poll_timeout, runtime_values=runtime) try: - status_timeout = mcp_poll_timeout + max(30, mcp_call_timeout * 2) + status_timeout = mcp_poll_timeout + max(90, mcp_call_timeout * 4) with _cycle_timeout(status_timeout if runtime.get("log_source") == "graylog_mcp" else 0): status = build_status( log_path, policy_path=policy_path, anomaly_limit=anomaly_limit, diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 6a77a8b..a30dd83 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -103,7 +103,7 @@ class BaselineTests(unittest.TestCase): store.ingest([parse_log_line("srcip=10.0.0.2 dstport=443 recent=1")], observed_at=recent) store.ingest_profile_fields([parse_log_line("fgai_stream_id=windows username=bob action=login recent=1")], profiles, observed_at=recent) - result = store.maintenance(retention_days=7, value_retention_days=7, max_values_per_field=2000) + result = store.maintenance(retention_days=7, value_retention_days=7, max_values_per_field=2000, include_rows=True) self.assertGreaterEqual(result["deleted"]["source_buckets"], 1) self.assertGreaterEqual(result["deleted"]["profile_buckets"], 1) @@ -122,7 +122,7 @@ class BaselineTests(unittest.TestCase): ] store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000) - stats = store.stats() + stats = store.stats(include_rows=True) self.assertEqual(stats["rows"]["profile_buckets"], 2) with store._connect() as connection: