add relationship fields

src/fgai/baseline.py

@@ -93,6 +93,10 @@ def _sample_event(event: LogEvent, value: str = "") -> dict[str, object]:
     }
 
 
+def _relationship_key(left: str, right: str) -> str:
+    return f"relationship:{left.lower()}->{right.lower()}"
+
+
 class BaselineStore:
     """Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
 
@@ -230,6 +234,13 @@ class BaselineStore:
                     for detector in event_detector_categories(event):
                         detector_pending[(stream_id, entity, detector, bucket)] += 1
                         detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
+                    for relation in getattr(profile, "relationship_fields", ()):
+                        left = str(getattr(relation, "left", "")).lower()
+                        right = str(getattr(relation, "right", "")).lower()
+                        left_value = event.fields.get(left)
+                        right_value = event.fields.get(right)
+                        if left_value and right_value:
+                            pending_values[(stream_id, left_value, _relationship_key(left, right), right_value)] += 1
                     for field in fields:
                         key = (stream_id, entity, str(field).lower(), bucket)
                         value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
@@ -386,6 +397,63 @@ class BaselineStore:
                 samples = [_sample_event(event, detector) for event in matching[:5]]
                 if score >= MIN_REPORTED_DEVIATION_SCORE:
                     output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
+            # Detect custom behavior relationships, e.g. username -> srcip or host -> process.name.
+            relationship_counts: Counter[tuple[str, str, str, str, str]] = Counter()
+            relationship_samples: dict[tuple[str, str, str, str, str], list[LogEvent]] = defaultdict(list)
+            for event in events:
+                stream = event.fields.get("fgai_stream_id", "")
+                profile = profiles.get(stream)
+                if not profile:
+                    continue
+                for relation in getattr(profile, "relationship_fields", ()):
+                    left = str(getattr(relation, "left", "")).lower()
+                    right = str(getattr(relation, "right", "")).lower()
+                    left_value = event.fields.get(left)
+                    right_value = event.fields.get(right)
+                    if not left_value or not right_value:
+                        continue
+                    key = (stream, left_value, left, right, right_value)
+                    relationship_counts[key] += 1
+                    if len(relationship_samples[key]) < 5:
+                        relationship_samples[key].append(event)
+            relationship_limit: Counter[tuple[str, str]] = Counter()
+            for (stream, left_value, left, right, right_value), count in relationship_counts.items():
+                field = _relationship_key(left, right)
+                known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, left_value, field, right_value)).fetchone()
+                known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0]
+                if known is not None or int(known_total or 0) < 12:
+                    continue
+                if relationship_limit[(stream, left_value)] >= MAX_RARE_VALUES_PER_ENTITY:
+                    continue
+                oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=?", (stream, left_value)).fetchone()[0]
+                if oldest is None:
+                    oldest = connection.execute("select min(last_seen) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0]
+                baseline_age_days = _age_days(oldest, _event_epoch(relationship_samples[(stream, left_value, left, right, right_value)][0], int(time.time())))
+                if baseline_age_days < min_training_days:
+                    continue
+                profile = profiles.get(stream)
+                base_score = 28
+                score, weight = _weighted_score(base_score, profile, field, "new_relationship")
+                if score < MIN_REPORTED_DEVIATION_SCORE:
+                    continue
+                samples = relationship_samples[(stream, left_value, left, right, right_value)]
+                output[left_value].append({
+                    "detector": "new_relationship",
+                    "field": field,
+                    "stream_id": stream,
+                    "score": score,
+                    "base_score": base_score,
+                    "weight": weight,
+                    "confidence": "medium",
+                    "baseline_samples": int(known_total),
+                    "baseline_age_days": round(baseline_age_days, 2),
+                    "baseline_scope": "known field relationships",
+                    "reason": f"new {right} value for {left}={left_value}",
+                    "value": right_value,
+                    "sample_values": [f"{left}={left_value}", f"{right}={right_value}"],
+                    "sample_events": [_sample_event(item, f"{left}={left_value} {right}={right_value}") for item in samples],
+                })
+                relationship_limit[(stream, left_value)] += 1
             # Detect selected categorical values that have not appeared for this entity in prior data.
             rare_counts: Counter[tuple[str, str]] = Counter()
             for event in events: