diff --git a/README.md b/README.md index e425776..f1f3c32 100644 --- a/README.md +++ b/README.md @@ -458,16 +458,19 @@ five-minute windows. Historical rate and hitcount-rate deviations then contribut to its anomaly score. Set `FGAI_BASELINE_DB` to use another location. SignalScope prunes old baseline buckets during each monitor cycle. The defaults -keep 14 days of buckets and dedupe history, prune stale one-off categorical -values after 7 days, and cap high-cardinality values per stream/entity/field. +keep 7 days of buckets and dedupe history, prune stale one-off categorical +values after 3 days, and cap high-cardinality values per stream/entity/field. +Very noisy fields such as raw messages, URLs, payloads, request/response bodies, +tokens, sessions, hashes and long values are still counted in bucket baselines +but are not stored as distinct rare-value candidates. Tune these in the dashboard or in `state/fgai-config.json`: ```json { "baseline_training_days": 7, - "baseline_retention_days": 14, - "baseline_value_retention_days": 7, - "baseline_max_values_per_field": 2000 + "baseline_retention_days": 7, + "baseline_value_retention_days": 3, + "baseline_max_values_per_field": 500 } ``` @@ -480,11 +483,12 @@ If an existing baseline database has already grown large, stop the monitor and run a manual prune plus SQLite compaction: ```bash -signalscope baseline-maintenance --baseline-db state/fgai-baseline.sqlite3 --retention-days 14 --value-retention-days 7 --max-values-per-field 2000 --vacuum +signalscope baseline-maintenance --baseline-db state/fgai-baseline.sqlite3 --retention-days 7 --value-retention-days 3 --max-values-per-field 500 --vacuum ``` -`VACUUM` can take time on a large database and should not be run while the -monitor is actively writing. +`VACUUM` can take time on a large database, needs free disk space roughly equal +to the database size, and should not be run while the monitor is actively +writing. Without `--vacuum`, SQLite may delete rows but keep the file size. Analyze local logs: diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index b7ffed5..520ba1b 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -15,12 +15,37 @@ from .detectors import DETECTOR_MINIMUMS, event_detector_categories from .normalization import canonical_value from .query_details import event_query_details -DEFAULT_RETENTION_DAYS = 14 -DEFAULT_VALUE_RETENTION_DAYS = 7 -DEFAULT_MAX_VALUES_PER_FIELD = 2000 +DEFAULT_RETENTION_DAYS = 7 +DEFAULT_VALUE_RETENTION_DAYS = 3 +DEFAULT_MAX_VALUES_PER_FIELD = 500 DEFAULT_TRAINING_DAYS = 7 MIN_REPORTED_DEVIATION_SCORE = 15 MAX_RARE_VALUES_PER_ENTITY = 5 +MAX_PROFILE_VALUE_LENGTH = 160 +NOISY_VALUE_FIELD_PARTS = ( + "answer", + "body", + "commandline", + "command_line", + "context", + "cookie", + "ephemeral", + "fingerprint", + "full_message", + "hash", + "message", + "payload", + "queryparameter", + "raw", + "request", + "response", + "session", + "stack", + "token", + "uri", + "url", + "user_agent", +) def _number(value: str | None) -> int: @@ -97,6 +122,17 @@ def _relationship_key(left: str, right: str) -> str: return f"relationship:{left.lower()}->{right.lower()}" +def _should_store_profile_value(field: str, value: str) -> bool: + field = field.lower() + if field.startswith("relationship:"): + return True + if len(value) > MAX_PROFILE_VALUE_LENGTH: + return False + if any(part in field for part in NOISY_VALUE_FIELD_PARTS): + return False + return True + + class BaselineStore: """Persistent five-minute behavior baseline, implemented with stdlib SQLite.""" @@ -206,7 +242,14 @@ class BaselineStore: ) return inserted - def ingest_profile_fields(self, events: list[LogEvent], profiles: dict[str, object], *, observed_at: int | None = None) -> int: + def ingest_profile_fields( + self, + events: list[LogEvent], + profiles: dict[str, object], + *, + observed_at: int | None = None, + max_values_per_field: int = DEFAULT_MAX_VALUES_PER_FIELD, + ) -> int: observed_at = observed_at or int(time.time()) pending: dict[tuple[str, str, str, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0]) temporal_pending: dict[tuple[str, str, str, int, int, int], list[float]] = defaultdict(lambda: [0, 0.0, 0.0]) @@ -253,7 +296,7 @@ class BaselineStore: temporal_pending[temporal][2] += value * value if key[2] not in numeric: raw_value = event.fields.get(key[2]) - if raw_value: + if raw_value and _should_store_profile_value(key[2], raw_value): pending_values[(*key[:3], raw_value)] += 1 for (stream_id, entity, field, bucket), values in pending.items(): connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?) @@ -261,7 +304,23 @@ class BaselineStore: for (stream_id, entity, field, weekday, hour, bucket), values in temporal_pending.items(): connection.execute("""insert into profile_temporal_buckets values (?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict(stream_id, entity, field, weekday, hour, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, weekday, hour, bucket, *values)) + value_counts: dict[tuple[str, str, str], int] = {} for key, count in pending_values.items(): + if max_values_per_field > 0: + group_key = key[:3] + known = connection.execute( + "select 1 from profile_values where stream_id=? and entity=? and field=? and value=?", + key, + ).fetchone() + if known is None: + if group_key not in value_counts: + value_counts[group_key] = int(connection.execute( + "select count(*) from profile_values where stream_id=? and entity=? and field=?", + group_key, + ).fetchone()[0]) + if value_counts[group_key] >= max_values_per_field: + continue + value_counts[group_key] += 1 connection.execute("""insert into profile_values values (?, ?, ?, ?, ?, ?) on conflict(stream_id, entity, field, value) do update set seen_count=seen_count+excluded.seen_count,last_seen=excluded.last_seen""", (*key, count, observed_at)) for (stream_id, entity, detector, bucket), count in detector_pending.items(): @@ -538,7 +597,35 @@ class BaselineStore: if vacuum: with self._connect() as connection: connection.execute("vacuum") - return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0} + stats = self.stats() + return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, **stats} + + def stats(self) -> dict[str, object]: + with self._connect() as connection: + tables = ( + "seen_events", + "source_buckets", + "source_values", + "profile_seen_events", + "profile_buckets", + "profile_temporal_buckets", + "profile_detector_buckets", + "profile_detector_temporal_buckets", + "profile_values", + ) + rows = {table: int(connection.execute(f"select count(*) from {table}").fetchone()[0]) for table in tables} + page_size = int(connection.execute("pragma page_size").fetchone()[0]) + page_count = int(connection.execute("pragma page_count").fetchone()[0]) + freelist_count = int(connection.execute("pragma freelist_count").fetchone()[0]) + size_bytes = self.path.stat().st_size if self.path.exists() else 0 + return { + "size_bytes": size_bytes, + "page_size": page_size, + "page_count": page_count, + "freelist_count": freelist_count, + "reclaimable_bytes": freelist_count * page_size, + "rows": rows, + } def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] diff --git a/src/fgai/config.py b/src/fgai/config.py index 703eec5..3324ba5 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -19,9 +19,9 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_raw_sample_events": 5000, "graylog_mcp_call_timeout_seconds": 8, "graylog_mcp_poll_timeout_seconds": 120, - "baseline_retention_days": 14, - "baseline_value_retention_days": 7, - "baseline_max_values_per_field": 2000, + "baseline_retention_days": 7, + "baseline_value_retention_days": 3, + "baseline_max_values_per_field": 500, "baseline_training_days": 7, "graylog_field_mapping": "", "llm_enabled": False, diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 7e43c90..ffa0a8d 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -184,7 +184,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Operator Guidance

Waiting for monitor data.

Correlation Map

Waiting for correlated entities.

Investigation Incidents

Anomalies

Recommendations

AI Assessment

LLM assessment disabled.

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

+

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

How To Use SignalScope

1. Normal workflow

  1. Settings: connect Graylog MCP and enable streams.
  2. Settings: apply recommended profiles for missing streams.
  3. Diagnostics: confirm raw samples, aggregate counts, and profile readiness.
  4. Overview: use Operator Guidance, incidents, trends, and correlation map.
  5. Findings: review only high-signal deviations first, then mark decisions.

2. Stream health

  • ready: profile exists, events are arriving, and baseline fields are ready.
  • learning: profile exists and events arrive, but baseline age or buckets are still too low.
  • missing_profile: stream is enabled but no profile exists. Apply or edit one.
  • no_events: stream is enabled but the current poll has no raw sample events.
  • partial_fetch: Graylog returned only part of the requested raw sample.

3. Ready fields

0/8 means 8 profile fields are tracked but none are mature enough yet. A field needs at least 12 baseline buckets and the configured Baseline training days before it is ready.

During learning, treat findings as signals to tune profiles, not as final alerts.

4. Profiles

  • Entity fields define who or what behavior is tracked, such as user, host, source IP, or application actor.
  • Baseline fields define the changing behavior to learn, such as action, event ID, service, URL, status, or counters.
  • Relationships learn pairs such as username to srcip or host to process.
  • Detectors add burst checks for auth failures, DNS queries, and deny actions.

5. Findings

  • Start with Triage Queue and incidents, not raw long tables.
  • Open evidence details before confirming a finding.
  • Use Expected for known behavior, False positive for bad signal, Confirmed for real investigation items.
  • Use expiry when a behavior is expected only temporarily.

6. High EPS / MCP

  • Use aggregate or auto fetch mode for high EPS streams.
  • Keep raw samples small enough for context; aggregate counts represent the full window.
  • Sample capped is normal in aggregate mode. Truncated raw mode means you may miss context.
  • If MCP is stale, the UI shows cached status so you can still inspect previous findings.

7. Ollama

  • Dashboard assessment summarizes current evidence.
  • Profile advisor maps unknown/custom fields and suggests relationships.
  • Ollama advice is constrained to fields discovered from Graylog; unknown fields are rejected.

8. What to fix first

  1. No streams enabled.
  2. Enabled streams with missing profiles.
  3. Enabled streams with zero raw events.
  4. Profiles stuck at 0 ready fields after the training window.
  5. Too many repeated findings without review feedback.
@@ -978,12 +978,12 @@ async function applySuggestedProfile(streamId) { graylog_max_events_per_stream: config.graylog_max_events_per_stream || 5000, graylog_raw_sample_events: config.graylog_raw_sample_events || 5000, graylog_mcp_call_timeout_seconds: config.graylog_mcp_call_timeout_seconds || 8, - graylog_mcp_poll_timeout_seconds: config.graylog_mcp_poll_timeout_seconds || 120, + graylog_mcp_poll_timeout_seconds: config.graylog_mcp_poll_timeout_seconds || 240, graylog_field_mapping: config.graylog_field_mapping || '', baseline_training_days: config.baseline_training_days || 7, - baseline_retention_days: config.baseline_retention_days || 14, - baseline_value_retention_days: config.baseline_value_retention_days || 7, - baseline_max_values_per_field: config.baseline_max_values_per_field || 2000, + baseline_retention_days: config.baseline_retention_days || 7, + baseline_value_retention_days: config.baseline_value_retention_days || 3, + baseline_max_values_per_field: config.baseline_max_values_per_field || 500, llm_enabled: Boolean(config.llm_enabled), llm_model: config.llm_model || '', profile_advisor_enabled: Boolean(config.profile_advisor_enabled), diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index 5b67640..e8363f4 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -392,12 +392,19 @@ def build_status( deviation["score"] = 0 anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations) baseline_events = baseline.ingest(events) if baseline else 0 - profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0 + profile_baseline_fields = ( + baseline.ingest_profile_fields( + events, + stream_profiles, + max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500), + ) + if baseline else 0 + ) baseline_maintenance = ( baseline.maintenance( - retention_days=int(runtime_values.get("baseline_retention_days", 14) or 14), - value_retention_days=int(runtime_values.get("baseline_value_retention_days", 7) or 7), - max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 2000) or 2000), + retention_days=int(runtime_values.get("baseline_retention_days", 7) or 7), + value_retention_days=int(runtime_values.get("baseline_value_retention_days", 3) or 3), + max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 500) or 500), ) if baseline else {} diff --git a/tests/test_baseline.py b/tests/test_baseline.py index d5fec3a..6a77a8b 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -109,6 +109,41 @@ class BaselineTests(unittest.TestCase): self.assertGreaterEqual(result["deleted"]["profile_buckets"], 1) self.assertGreaterEqual(result["deleted"]["seen_events"], 1) self.assertGreaterEqual(result["deleted"]["profile_seen_events"], 1) + self.assertIn("rows", result) + self.assertIn("reclaimable_bytes", result) + + def test_profile_value_storage_skips_noisy_high_cardinality_fields(self): + with tempfile.TemporaryDirectory() as directory: + store = BaselineStore(str(Path(directory) / "baseline.sqlite3")) + profiles = parse_profiles([{"stream_id": "web", "entity_field": "srcip", "categorical_fields": ["url", "action"]}]) + events = [ + parse_log_line(f"fgai_stream_id=web srcip=10.0.0.5 action=allow url=/download/{index}/very/noisy/path event={index}") + for index in range(10) + ] + + store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000) + stats = store.stats() + + self.assertEqual(stats["rows"]["profile_buckets"], 2) + with store._connect() as connection: + fields = {row[0] for row in connection.execute("select distinct field from profile_values").fetchall()} + self.assertIn("action", fields) + self.assertNotIn("url", fields) + + def test_profile_value_storage_caps_new_values_during_ingest(self): + with tempfile.TemporaryDirectory() as directory: + store = BaselineStore(str(Path(directory) / "baseline.sqlite3")) + profiles = parse_profiles([{"stream_id": "dns", "entity_field": "srcip", "categorical_fields": ["query_domain"]}]) + events = [ + parse_log_line(f"fgai_stream_id=dns srcip=10.0.0.5 query_domain=value-{index}.example event={index}") + for index in range(5) + ] + + store.ingest_profile_fields(events, profiles, observed_at=1_700_000_000, max_values_per_field=2) + + with store._connect() as connection: + stored = connection.execute("select count(*) from profile_values").fetchone()[0] + self.assertEqual(stored, 2) def test_rare_values_are_limited_per_entity(self): with tempfile.TemporaryDirectory() as directory: