diff --git a/README.md b/README.md index 8a5644c..a1bb586 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,13 @@ contain them. This lets late-arriving or less frequent fields such as custom `lcs_*` application fields stay visible long enough to be reviewed and appended to an existing profile. +Shared-field discovery is also used as the base for cross-source correlation. +SignalScope groups exact aliases and broader semantic families such as source IP, +user, host, ID, status/result, action, type/category, domain, URL, and custom +namespaces such as `lcs_*`. These shared groups are the foundation for a global +correlation profile and future flow graphs that show how users, hosts, IPs, +applications, IDs, statuses, and destinations relate across streams. + Enabled streams are normalized through the same event model. Stream profiles define the entity, timestamp, categorical, and numeric fields used for baselines. The dashboard and Ollama then correlate behavior across sources, for example a @@ -173,6 +180,29 @@ A stream profile can track multiple entities from the same event, such as each selected entity value, which makes cross-source investigation work even when one source is user-centric and another is IP- or host-centric. +Profiles can also track field relationships as behavior patterns. This is useful +when the suspicious signal is not a single new value, but a new combination such +as a known user logging in successfully from a source IP that has never been seen +for that user before. Add `relationship_fields` to a stream profile, for example: + +```json +[ + { + "stream_id": "windows-security", + "entity_field": "username", + "categorical_fields": ["action", "eventid"], + "relationship_fields": [ + {"left": "username", "right": "srcip", "name": "user source IP"}, + {"left": "username", "right": "hostname", "name": "user host"} + ] + } +] +``` + +After the baseline has learned those relationships, a new `username -> srcip` or +`username -> hostname` pair is reported as `new_relationship` with sample events. +The dashboard profile editor exposes this as `Behavior relationships (JSON)`. + SignalScope keeps a common alias map for fields such as source IP, destination IP, ports, action, severity, service/protocol, DNS query, URL, message, and event type. This lets Related Activity and correlations work with firewall/proxy/DNS diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 090ea59..b7ffed5 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -93,6 +93,10 @@ def _sample_event(event: LogEvent, value: str = "") -> dict[str, object]: } +def _relationship_key(left: str, right: str) -> str: + return f"relationship:{left.lower()}->{right.lower()}" + + class BaselineStore: """Persistent five-minute behavior baseline, implemented with stdlib SQLite.""" @@ -230,6 +234,13 @@ class BaselineStore: for detector in event_detector_categories(event): detector_pending[(stream_id, entity, detector, bucket)] += 1 detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1 + for relation in getattr(profile, "relationship_fields", ()): + left = str(getattr(relation, "left", "")).lower() + right = str(getattr(relation, "right", "")).lower() + left_value = event.fields.get(left) + right_value = event.fields.get(right) + if left_value and right_value: + pending_values[(stream_id, left_value, _relationship_key(left, right), right_value)] += 1 for field in fields: key = (stream_id, entity, str(field).lower(), bucket) value = _number(event.fields.get(key[2])) if key[2] in numeric else 0 @@ -386,6 +397,63 @@ class BaselineStore: samples = [_sample_event(event, detector) for event in matching[:5]] if score >= MIN_REPORTED_DEVIATION_SCORE: output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples}) + # Detect custom behavior relationships, e.g. username -> srcip or host -> process.name. + relationship_counts: Counter[tuple[str, str, str, str, str]] = Counter() + relationship_samples: dict[tuple[str, str, str, str, str], list[LogEvent]] = defaultdict(list) + for event in events: + stream = event.fields.get("fgai_stream_id", "") + profile = profiles.get(stream) + if not profile: + continue + for relation in getattr(profile, "relationship_fields", ()): + left = str(getattr(relation, "left", "")).lower() + right = str(getattr(relation, "right", "")).lower() + left_value = event.fields.get(left) + right_value = event.fields.get(right) + if not left_value or not right_value: + continue + key = (stream, left_value, left, right, right_value) + relationship_counts[key] += 1 + if len(relationship_samples[key]) < 5: + relationship_samples[key].append(event) + relationship_limit: Counter[tuple[str, str]] = Counter() + for (stream, left_value, left, right, right_value), count in relationship_counts.items(): + field = _relationship_key(left, right) + known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, left_value, field, right_value)).fetchone() + known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0] + if known is not None or int(known_total or 0) < 12: + continue + if relationship_limit[(stream, left_value)] >= MAX_RARE_VALUES_PER_ENTITY: + continue + oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=?", (stream, left_value)).fetchone()[0] + if oldest is None: + oldest = connection.execute("select min(last_seen) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0] + baseline_age_days = _age_days(oldest, _event_epoch(relationship_samples[(stream, left_value, left, right, right_value)][0], int(time.time()))) + if baseline_age_days < min_training_days: + continue + profile = profiles.get(stream) + base_score = 28 + score, weight = _weighted_score(base_score, profile, field, "new_relationship") + if score < MIN_REPORTED_DEVIATION_SCORE: + continue + samples = relationship_samples[(stream, left_value, left, right, right_value)] + output[left_value].append({ + "detector": "new_relationship", + "field": field, + "stream_id": stream, + "score": score, + "base_score": base_score, + "weight": weight, + "confidence": "medium", + "baseline_samples": int(known_total), + "baseline_age_days": round(baseline_age_days, 2), + "baseline_scope": "known field relationships", + "reason": f"new {right} value for {left}={left_value}", + "value": right_value, + "sample_values": [f"{left}={left_value}", f"{right}={right_value}"], + "sample_events": [_sample_event(item, f"{left}={left_value} {right}={right_value}") for item in samples], + }) + relationship_limit[(stream, left_value)] += 1 # Detect selected categorical values that have not appeared for this entity in prior data. rare_counts: Counter[tuple[str, str]] = Counter() for event in events: diff --git a/src/fgai/cli.py b/src/fgai/cli.py index 6ef8f7c..71f84a0 100644 --- a/src/fgai/cli.py +++ b/src/fgai/cli.py @@ -257,13 +257,15 @@ def _stream_name(config: dict[str, object], stream_id: str) -> str: def _profile_fields(profile: object | None) -> tuple[str, ...]: if not profile: return () - return tuple( - field for field in ( - str(getattr(profile, "entity_field", "")), - *tuple(str(item) for item in getattr(profile, "entity_fields", ())), - str(getattr(profile, "timestamp_field", "")), + return tuple( + field for field in ( + str(getattr(profile, "entity_field", "")), + *tuple(str(item) for item in getattr(profile, "entity_fields", ())), + str(getattr(profile, "timestamp_field", "")), *tuple(str(item) for item in getattr(profile, "categorical_fields", ())), *tuple(str(item) for item in getattr(profile, "numeric_fields", ())), + *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())), + *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())), ) if field ) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index f9c5c56..ef64cc5 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -128,7 +128,7 @@ HTML = """