diff --git a/README.md b/README.md index 8a5644c..a1bb586 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,13 @@ contain them. This lets late-arriving or less frequent fields such as custom `lcs_*` application fields stay visible long enough to be reviewed and appended to an existing profile. +Shared-field discovery is also used as the base for cross-source correlation. +SignalScope groups exact aliases and broader semantic families such as source IP, +user, host, ID, status/result, action, type/category, domain, URL, and custom +namespaces such as `lcs_*`. These shared groups are the foundation for a global +correlation profile and future flow graphs that show how users, hosts, IPs, +applications, IDs, statuses, and destinations relate across streams. + Enabled streams are normalized through the same event model. Stream profiles define the entity, timestamp, categorical, and numeric fields used for baselines. The dashboard and Ollama then correlate behavior across sources, for example a @@ -173,6 +180,29 @@ A stream profile can track multiple entities from the same event, such as each selected entity value, which makes cross-source investigation work even when one source is user-centric and another is IP- or host-centric. +Profiles can also track field relationships as behavior patterns. This is useful +when the suspicious signal is not a single new value, but a new combination such +as a known user logging in successfully from a source IP that has never been seen +for that user before. Add `relationship_fields` to a stream profile, for example: + +```json +[ + { + "stream_id": "windows-security", + "entity_field": "username", + "categorical_fields": ["action", "eventid"], + "relationship_fields": [ + {"left": "username", "right": "srcip", "name": "user source IP"}, + {"left": "username", "right": "hostname", "name": "user host"} + ] + } +] +``` + +After the baseline has learned those relationships, a new `username -> srcip` or +`username -> hostname` pair is reported as `new_relationship` with sample events. +The dashboard profile editor exposes this as `Behavior relationships (JSON)`. + SignalScope keeps a common alias map for fields such as source IP, destination IP, ports, action, severity, service/protocol, DNS query, URL, message, and event type. This lets Related Activity and correlations work with firewall/proxy/DNS diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 090ea59..b7ffed5 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -93,6 +93,10 @@ def _sample_event(event: LogEvent, value: str = "") -> dict[str, object]: } +def _relationship_key(left: str, right: str) -> str: + return f"relationship:{left.lower()}->{right.lower()}" + + class BaselineStore: """Persistent five-minute behavior baseline, implemented with stdlib SQLite.""" @@ -230,6 +234,13 @@ class BaselineStore: for detector in event_detector_categories(event): detector_pending[(stream_id, entity, detector, bucket)] += 1 detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1 + for relation in getattr(profile, "relationship_fields", ()): + left = str(getattr(relation, "left", "")).lower() + right = str(getattr(relation, "right", "")).lower() + left_value = event.fields.get(left) + right_value = event.fields.get(right) + if left_value and right_value: + pending_values[(stream_id, left_value, _relationship_key(left, right), right_value)] += 1 for field in fields: key = (stream_id, entity, str(field).lower(), bucket) value = _number(event.fields.get(key[2])) if key[2] in numeric else 0 @@ -386,6 +397,63 @@ class BaselineStore: samples = [_sample_event(event, detector) for event in matching[:5]] if score >= MIN_REPORTED_DEVIATION_SCORE: output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples}) + # Detect custom behavior relationships, e.g. username -> srcip or host -> process.name. + relationship_counts: Counter[tuple[str, str, str, str, str]] = Counter() + relationship_samples: dict[tuple[str, str, str, str, str], list[LogEvent]] = defaultdict(list) + for event in events: + stream = event.fields.get("fgai_stream_id", "") + profile = profiles.get(stream) + if not profile: + continue + for relation in getattr(profile, "relationship_fields", ()): + left = str(getattr(relation, "left", "")).lower() + right = str(getattr(relation, "right", "")).lower() + left_value = event.fields.get(left) + right_value = event.fields.get(right) + if not left_value or not right_value: + continue + key = (stream, left_value, left, right, right_value) + relationship_counts[key] += 1 + if len(relationship_samples[key]) < 5: + relationship_samples[key].append(event) + relationship_limit: Counter[tuple[str, str]] = Counter() + for (stream, left_value, left, right, right_value), count in relationship_counts.items(): + field = _relationship_key(left, right) + known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, left_value, field, right_value)).fetchone() + known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0] + if known is not None or int(known_total or 0) < 12: + continue + if relationship_limit[(stream, left_value)] >= MAX_RARE_VALUES_PER_ENTITY: + continue + oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=?", (stream, left_value)).fetchone()[0] + if oldest is None: + oldest = connection.execute("select min(last_seen) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0] + baseline_age_days = _age_days(oldest, _event_epoch(relationship_samples[(stream, left_value, left, right, right_value)][0], int(time.time()))) + if baseline_age_days < min_training_days: + continue + profile = profiles.get(stream) + base_score = 28 + score, weight = _weighted_score(base_score, profile, field, "new_relationship") + if score < MIN_REPORTED_DEVIATION_SCORE: + continue + samples = relationship_samples[(stream, left_value, left, right, right_value)] + output[left_value].append({ + "detector": "new_relationship", + "field": field, + "stream_id": stream, + "score": score, + "base_score": base_score, + "weight": weight, + "confidence": "medium", + "baseline_samples": int(known_total), + "baseline_age_days": round(baseline_age_days, 2), + "baseline_scope": "known field relationships", + "reason": f"new {right} value for {left}={left_value}", + "value": right_value, + "sample_values": [f"{left}={left_value}", f"{right}={right_value}"], + "sample_events": [_sample_event(item, f"{left}={left_value} {right}={right_value}") for item in samples], + }) + relationship_limit[(stream, left_value)] += 1 # Detect selected categorical values that have not appeared for this entity in prior data. rare_counts: Counter[tuple[str, str]] = Counter() for event in events: diff --git a/src/fgai/cli.py b/src/fgai/cli.py index 6ef8f7c..71f84a0 100644 --- a/src/fgai/cli.py +++ b/src/fgai/cli.py @@ -257,13 +257,15 @@ def _stream_name(config: dict[str, object], stream_id: str) -> str: def _profile_fields(profile: object | None) -> tuple[str, ...]: if not profile: return () - return tuple( - field for field in ( - str(getattr(profile, "entity_field", "")), - *tuple(str(item) for item in getattr(profile, "entity_fields", ())), - str(getattr(profile, "timestamp_field", "")), + return tuple( + field for field in ( + str(getattr(profile, "entity_field", "")), + *tuple(str(item) for item in getattr(profile, "entity_fields", ())), + str(getattr(profile, "timestamp_field", "")), *tuple(str(item) for item in getattr(profile, "categorical_fields", ())), *tuple(str(item) for item in getattr(profile, "numeric_fields", ())), + *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())), + *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())), ) if field ) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index f9c5c56..ef64cc5 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -128,7 +128,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Correlation Map

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

+

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

How To Use SignalScope

1. Connect Logs

  1. Open Settings and select Graylog MCP.
  2. Save the MCP URL and token.
  3. Click Load streams, enable the streams you want monitored, then save.

2. Create Profiles

  1. Use Recommended Stream Profiles first.
  2. Click Apply profile for missing streams.
  3. Use Edit profile when a stream needs custom entity, time, baseline, detector, or weight fields.

3. Let Baselines Learn

  1. Set Baseline training days to the history window you trust.
  2. Use Diagnostics to see profile readiness per stream and field.
  3. Treat early findings as learning signals until fields are ready.

4. Investigate Findings

  1. Start with Overview incidents and the correlation map.
  2. Use Findings for triage, field deviations, related activity, and reputation.
  3. Open evidence rows to see samples and Graylog query details.

5. Review Noise

  1. Mark deviations as Expected, False positive, or Confirmed.
  2. Use expiry days when expected behavior should be temporary.
  3. Reviewed expected patterns stay labeled and reduce repeat noise for the same scope.

6. Production Settings

  • Use aggregate or auto fetch mode for high EPS streams.
  • Keep the poll window around 300 seconds unless you know Graylog can handle more.
  • Enable Ollama only when local models are installed.
  • Add threat intel API keys only if you want external public-IP reputation checks.