add relationship fields

src/fgai/baseline.py

@@ -93,6 +93,10 @@ def _sample_event(event: LogEvent, value: str = "") -> dict[str, object]:
     }
 
 
+def _relationship_key(left: str, right: str) -> str:
+    return f"relationship:{left.lower()}->{right.lower()}"
+
+
 class BaselineStore:
     """Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
 
@@ -230,6 +234,13 @@ class BaselineStore:
                     for detector in event_detector_categories(event):
                         detector_pending[(stream_id, entity, detector, bucket)] += 1
                         detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
+                    for relation in getattr(profile, "relationship_fields", ()):
+                        left = str(getattr(relation, "left", "")).lower()
+                        right = str(getattr(relation, "right", "")).lower()
+                        left_value = event.fields.get(left)
+                        right_value = event.fields.get(right)
+                        if left_value and right_value:
+                            pending_values[(stream_id, left_value, _relationship_key(left, right), right_value)] += 1
                     for field in fields:
                         key = (stream_id, entity, str(field).lower(), bucket)
                         value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
@@ -386,6 +397,63 @@ class BaselineStore:
                 samples = [_sample_event(event, detector) for event in matching[:5]]
                 if score >= MIN_REPORTED_DEVIATION_SCORE:
                     output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
+            # Detect custom behavior relationships, e.g. username -> srcip or host -> process.name.
+            relationship_counts: Counter[tuple[str, str, str, str, str]] = Counter()
+            relationship_samples: dict[tuple[str, str, str, str, str], list[LogEvent]] = defaultdict(list)
+            for event in events:
+                stream = event.fields.get("fgai_stream_id", "")
+                profile = profiles.get(stream)
+                if not profile:
+                    continue
+                for relation in getattr(profile, "relationship_fields", ()):
+                    left = str(getattr(relation, "left", "")).lower()
+                    right = str(getattr(relation, "right", "")).lower()
+                    left_value = event.fields.get(left)
+                    right_value = event.fields.get(right)
+                    if not left_value or not right_value:
+                        continue
+                    key = (stream, left_value, left, right, right_value)
+                    relationship_counts[key] += 1
+                    if len(relationship_samples[key]) < 5:
+                        relationship_samples[key].append(event)
+            relationship_limit: Counter[tuple[str, str]] = Counter()
+            for (stream, left_value, left, right, right_value), count in relationship_counts.items():
+                field = _relationship_key(left, right)
+                known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, left_value, field, right_value)).fetchone()
+                known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0]
+                if known is not None or int(known_total or 0) < 12:
+                    continue
+                if relationship_limit[(stream, left_value)] >= MAX_RARE_VALUES_PER_ENTITY:
+                    continue
+                oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=?", (stream, left_value)).fetchone()[0]
+                if oldest is None:
+                    oldest = connection.execute("select min(last_seen) from profile_values where stream_id=? and entity=? and field=?", (stream, left_value, field)).fetchone()[0]
+                baseline_age_days = _age_days(oldest, _event_epoch(relationship_samples[(stream, left_value, left, right, right_value)][0], int(time.time())))
+                if baseline_age_days < min_training_days:
+                    continue
+                profile = profiles.get(stream)
+                base_score = 28
+                score, weight = _weighted_score(base_score, profile, field, "new_relationship")
+                if score < MIN_REPORTED_DEVIATION_SCORE:
+                    continue
+                samples = relationship_samples[(stream, left_value, left, right, right_value)]
+                output[left_value].append({
+                    "detector": "new_relationship",
+                    "field": field,
+                    "stream_id": stream,
+                    "score": score,
+                    "base_score": base_score,
+                    "weight": weight,
+                    "confidence": "medium",
+                    "baseline_samples": int(known_total),
+                    "baseline_age_days": round(baseline_age_days, 2),
+                    "baseline_scope": "known field relationships",
+                    "reason": f"new {right} value for {left}={left_value}",
+                    "value": right_value,
+                    "sample_values": [f"{left}={left_value}", f"{right}={right_value}"],
+                    "sample_events": [_sample_event(item, f"{left}={left_value} {right}={right_value}") for item in samples],
+                })
+                relationship_limit[(stream, left_value)] += 1
             # Detect selected categorical values that have not appeared for this entity in prior data.
             rare_counts: Counter[tuple[str, str]] = Counter()
             for event in events:

src/fgai/cli.py

@@ -257,13 +257,15 @@ def _stream_name(config: dict[str, object], stream_id: str) -> str:
 def _profile_fields(profile: object | None) -> tuple[str, ...]:
     if not profile:
         return ()
-        return tuple(
-            field for field in (
-                str(getattr(profile, "entity_field", "")),
-                *tuple(str(item) for item in getattr(profile, "entity_fields", ())),
-                str(getattr(profile, "timestamp_field", "")),
+    return tuple(
+        field for field in (
+            str(getattr(profile, "entity_field", "")),
+            *tuple(str(item) for item in getattr(profile, "entity_fields", ())),
+            str(getattr(profile, "timestamp_field", "")),
             *tuple(str(item) for item in getattr(profile, "categorical_fields", ())),
             *tuple(str(item) for item in getattr(profile, "numeric_fields", ())),
+            *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())),
+            *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())),
         )
         if field
     )

src/fgai/history.py

@@ -75,6 +75,42 @@ class FieldDiscoveryStore:
                     primary key (stream_id, field)
                 )"""
             )
+            connection.execute(
+                """create table if not exists field_catalog (
+                    stream_id text not null, stream_name text not null, field text not null,
+                    field_type text not null, properties text not null, first_seen integer not null, last_seen integer not null,
+                    primary key (stream_id, field)
+                )"""
+            )
+
+    def ingest_catalog(self, stream_id: str, stream_name: str, fields: list[dict[str, object]], *, retention_days: int = 30) -> int:
+        now = int(time.time())
+        rows = []
+        for item in fields:
+            if not isinstance(item, dict):
+                continue
+            field = str(item.get("name") or item.get("field") or "").strip()
+            if not field or field in IGNORED_DISCOVERY_FIELDS:
+                continue
+            type_info = item.get("type", {})
+            field_type = str(type_info.get("type", "") if isinstance(type_info, dict) else type_info)
+            properties = type_info.get("properties", []) if isinstance(type_info, dict) else []
+            if not isinstance(properties, list):
+                properties = []
+            rows.append((stream_id, stream_name, field, field_type, json.dumps([str(prop) for prop in properties]), now, now))
+        with sqlite3.connect(self.path) as connection:
+            for row in rows:
+                connection.execute(
+                    """insert into field_catalog values (?, ?, ?, ?, ?, ?, ?)
+                       on conflict(stream_id, field) do update set
+                       stream_name=excluded.stream_name,
+                       field_type=excluded.field_type,
+                       properties=excluded.properties,
+                       last_seen=excluded.last_seen""",
+                    row,
+                )
+            connection.execute("delete from field_catalog where last_seen < ?", (now - retention_days * 86400,))
+        return len(rows)
 
     def ingest(self, events: list[object], *, max_values: int = 20, retention_days: int = 30) -> int:
         now = int(time.time())
@@ -130,6 +166,7 @@ class FieldDiscoveryStore:
         events: list[LogEvent] = []
         with sqlite3.connect(self.path) as connection:
             rows = connection.execute("select stream_id, stream_name, field, seen_count, numeric_count, sample_values from field_discovery").fetchall()
+            catalog_rows = connection.execute("select stream_id, stream_name, field, field_type, properties from field_catalog").fetchall()
         for stream_id, stream_name, field, seen_count, numeric_count, sample_values in rows:
             try:
                 values = [str(item) for item in json.loads(str(sample_values))]
@@ -142,4 +179,20 @@ class FieldDiscoveryStore:
                 value = values[index % len(values)]
                 fields = {"fgai_stream_id": str(stream_id), "fgai_stream": str(stream_name), str(field): value}
                 events.append(LogEvent(raw=json.dumps(fields, sort_keys=True), fields=fields))
+        existing = {(event.fields.get("fgai_stream_id", ""), field) for event in events for field in event.fields if field not in {"fgai_stream_id", "fgai_stream"}}
+        for stream_id, stream_name, field, field_type, properties in catalog_rows:
+            if (str(stream_id), str(field)) in existing:
+                continue
+            try:
+                props = set(json.loads(str(properties)))
+            except (json.JSONDecodeError, TypeError):
+                props = set()
+            if "numeric" in props or str(field_type).lower() in {"long", "double", "integer", "int", "float"}:
+                value = "1"
+            elif "enumerable" in props:
+                value = "observed"
+            else:
+                value = "catalog"
+            fields = {"fgai_stream_id": str(stream_id), "fgai_stream": str(stream_name), str(field): value}
+            events.append(LogEvent(raw=json.dumps(fields, sort_keys=True), fields=fields))
         return events

src/fgai/llm.py

@@ -109,22 +109,28 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
     )
 
 
+def _compact_profile_suggestion(item: dict[str, object]) -> dict[str, object]:
+    discovery = item.get("discovery", {}) if isinstance(item.get("discovery"), dict) else {}
+    selected = discovery.get("selected_fields", {}) if isinstance(discovery.get("selected_fields"), dict) else {}
+    return {
+        "stream_id": item.get("stream_id"),
+        "stream_name": item.get("stream_name"),
+        "events": item.get("events"),
+        "detected_log_type": item.get("detected_log_type", ""),
+        "common_fields": item.get("common_fields", [])[:25],
+        "shared_fields": item.get("shared_fields", [])[:20],
+        "top_fields": discovery.get("top_fields", [])[:25],
+        "selected_fields": selected,
+        "rejected_fields": discovery.get("rejected_fields", [])[:10],
+        "heuristic_profile": item.get("profile", {}),
+    }
+
+
 def ollama_profile_advice(suggestions: list[dict[str, object]], model: str | None = None, timeout: int | None = None) -> list[dict[str, object]]:
     host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
     selected_model = model or os.getenv("FGAI_PROFILE_ADVISOR_MODEL", "qwen3:8b")
     selected_timeout = timeout or int(os.getenv("FGAI_PROFILE_ADVISOR_TIMEOUT", "120"))
-    compact = [
-        {
-            "stream_id": item.get("stream_id"),
-            "stream_name": item.get("stream_name"),
-            "events": item.get("events"),
-            "common_fields": item.get("common_fields", [])[:20],
-            "shared_fields": item.get("shared_fields", [])[:12],
-            "detected_log_type": item.get("detected_log_type", ""),
-            "heuristic_profile": item.get("profile", {}),
-        }
-        for item in suggestions[:10]
-    ]
+    compact = [_compact_profile_suggestion(item) for item in suggestions[:10]]
     body = json.dumps(
         {
             "model": selected_model,
@@ -136,9 +142,18 @@ def ollama_profile_advice(suggestions: list[dict[str, object]], model: str | Non
                 "Return only valid JSON with this schema: "
                 "{\"profiles\":[{\"stream_id\":\"...\",\"entity_fields\":[\"...\"],\"timestamp_field\":\"...\","
                 "\"categorical_fields\":[\"...\"],\"numeric_fields\":[\"...\"],\"detectors\":{\"auth_failure\":{\"enabled\":true,\"minimum\":5,\"z_threshold\":3}},"
+                "\"relationship_fields\":[{\"left\":\"username\",\"right\":\"srcip\",\"name\":\"user source IP\"}],"
+                "\"field_roles\":{\"field\":\"entity|identity|asset|session|status|action|resource|metric|context\"},"
+                "\"correlation_roles\":{\"entity\":[\"...\"],\"identity\":[\"...\"],\"asset\":[\"...\"],\"session\":[\"...\"],\"activity\":[\"...\"],\"resource\":[\"...\"],\"status\":[\"...\"]},"
                 "\"reason\":\"short reason\"}]}. "
-                "Use only field names present in common_fields, shared_fields, or heuristic_profile. Prefer fields that appear in shared_fields when they are useful categorical or numeric baseline fields. Do not include raw message/full_message fields. "
-                "Allowed detectors are auth_failure, dns_query, deny_action. Prefer canonical fields such as username, hostname, eventid, srcip, dstip when present. "
+                "Use only field names present in common_fields, shared_fields aliases, top_fields, selected_fields, or heuristic_profile. "
+                "Do not invent fields. Do not include raw message/full_message/raw/answer fields. "
+                "Treat shared_fields as deterministic evidence that a field or semantic group appears across streams. "
+                "For custom namespaces such as lcs_* infer roles from the real field names and observed stats, for example tenant/customer/request/session/correlation IDs. "
+                "Prefer profile fields that connect streams: source or client IPs, usernames/accounts, hostnames, event IDs, session/correlation/request IDs, status/result/action/type, service/app, URL/domain/resource, and stable numeric metrics. "
+                "Use categorical_fields for low-cardinality behavior dimensions, numeric_fields for counters/durations/sizes, and entity_fields for fields that identify a user, host, IP, account, device, or application actor. "
+                "Use relationship_fields for behavior pairs where a left identity should learn normal right-side values, such as username->srcip, username->hostname, host->process.name, srcip->dstip, or account->application. "
+                "Allowed detectors are auth_failure, dns_query, deny_action. Prefer canonical fields such as username, hostname, eventid, srcip, dstip when present, but keep useful stream-specific fields too. "
                 f"\n\nObserved streams:\n{json.dumps(compact, sort_keys=True)}"
             ),
         }

src/fgai/monitor.py

@@ -54,6 +54,21 @@ def _range_seconds(value: object) -> int:
         return 300
 
 
+def _graylog_fields_from_result(result: dict[str, object]) -> list[dict[str, object]]:
+    content = result.get("result", {}).get("content", []) if isinstance(result.get("result"), dict) else []
+    text = next((item.get("text", "") for item in content if isinstance(item, dict)), "")
+    if not text:
+        return []
+    try:
+        payload = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+    fields = payload.get("fields", payload) if isinstance(payload, dict) else payload
+    if isinstance(fields, dict) and isinstance(fields.get("fields"), list):
+        fields = fields["fields"]
+    return [item for item in fields if isinstance(item, dict)] if isinstance(fields, list) else []
+
+
 def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[str, object], stream_status: dict[str, object], profile_readiness: list[dict[str, object]], stream_titles: dict[str, str]) -> list[dict[str, object]]:
     configured = [
         item for item in runtime_values.get("graylog_streams", [])
@@ -85,7 +100,7 @@ def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[st
             "profile": _profile_name(stream_id, stream_titles, profile) if profile else "",
             "profile_ready": bool(profile),
             "entity_field": ", ".join(getattr(profile, "entity_fields", ()) or (str(getattr(profile, "entity_field", "")),)) if profile else "",
-            "tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) if profile else 0,
+            "tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) + len(getattr(profile, "relationship_fields", ())) if profile else 0,
             "ready_fields": ready_fields,
             "total_fields": total_fields,
             "readiness": f"{ready_fields}/{total_fields}" if total_fields else "0/0",
@@ -155,6 +170,8 @@ def build_status(
                 aggregate_events_total = 0
                 client = GraylogMcpClient(url, token, verify_tls=verify_tls)
                 probe_status = client.probe()
+                discovery_store = FieldDiscoveryStore(history_path) if history_path else None
+                catalog_fields_total = 0
                 for stream_config in stream_configs:
                     stream_id = str(stream_config["id"])
                     profile = stream_profiles.get(stream_id)
@@ -164,8 +181,15 @@ def build_status(
                         str(getattr(profile, "timestamp_field", "")),
                         *tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
                         *tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
+                        *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())),
+                        *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())),
                     ) if profile else ()
                     stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
+                    if discovery_store:
+                        try:
+                            catalog_fields_total += discovery_store.ingest_catalog(stream_id, stream_name, _graylog_fields_from_result(client.call_tool("list_fields", {"streams": [stream_id]})))
+                        except RuntimeError:
+                            pass
                     aggregate_status: dict[str, object] = {}
                     if use_aggregate:
                         aggregate_status = GraylogAggregateSource(client, stream_id, str(runtime_values.get("graylog_query", "*"))).fetch_count(range_seconds=range_seconds, probe_status=probe_status)
@@ -199,6 +223,7 @@ def build_status(
                     "sample_limited_streams": len(sample_limited_streams),
                     "truncated_streams": len(truncated_streams),
                     "aggregate_error_streams": len(aggregate_errors),
+                    "catalog_fields": catalog_fields_total,
                     "coverage_status": "partial" if partial_streams else "truncated" if truncated_streams else "complete_window",
                     "coverage_warning": " ".join(warnings),
                 }
@@ -338,7 +363,7 @@ def build_status(
         "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "discovery_fields_recorded": discovered_profile_fields, "discovery_cache_events": len(discovery_cache_events), "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
         "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
         "configuration": runtime_config,
-        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
+        "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights, "relationship_fields": [{"left": relation.left, "right": relation.right, "name": relation.name} for relation in item.relationship_fields]} for item in stream_profiles.values()],
         "stream_coverage": stream_coverage,
         "profile_suggestions": profile_suggestions,
         "profile_readiness": profile_readiness,

src/fgai/profile_suggestions.py

@@ -219,7 +219,46 @@ def _looks_like_time_field(field: str) -> bool:
 
 
 def _semantic_field_group(field: str) -> str:
-    return ALIAS_CANONICAL_BY_FIELD.get(field.lower(), field.lower())
+    return _semantic_field_groups(field)[0]
+
+
+def _semantic_field_groups(field: str) -> list[str]:
+    normalized = field.lower()
+    if normalized in ALIAS_CANONICAL_BY_FIELD:
+        return [ALIAS_CANONICAL_BY_FIELD[normalized]]
+    groups = []
+    if normalized.startswith("lcs_"):
+        groups.append("lcs_*")
+    compact = normalized.replace(".", "_").replace("-", "_")
+    parts = [part for part in compact.split("_") if part]
+    if not parts:
+        return [normalized]
+    if parts[-1] in {"ip", "addr", "address"}:
+        groups.append("*_ip")
+    if parts[-1] in {"user", "username", "account", "principal", "actor", "name"} and any(part in {"user", "account", "principal", "actor"} for part in parts):
+        groups.append("*_user")
+    if parts[-1] in {"host", "hostname", "computer", "node", "name"} and any(part in {"host", "hostname", "computer", "node"} for part in parts):
+        groups.append("*_host")
+    if parts[-1] in {"id", "uuid", "guid"}:
+        groups.append("*_id")
+    if parts[-1] in {"name", "label", "title"}:
+        groups.append("*_name")
+    if parts[-1] in {"status", "state", "result", "outcome", "disposition"}:
+        groups.append("*_status")
+    if parts[-1] in {"action", "operation", "event"}:
+        groups.append("*_action")
+    if parts[-1] in {"type", "category", "class", "kind"}:
+        groups.append("*_type")
+    if parts[-1] in {"code", "reason"}:
+        groups.append("*_code")
+    if parts[-1] in {"port"}:
+        groups.append("*_port")
+    if parts[-1] in {"domain", "fqdn"}:
+        groups.append("*_domain")
+    if parts[-1] in {"url", "uri", "path"}:
+        groups.append("*_url")
+    groups.append(normalized)
+    return list(dict.fromkeys(groups))
 
 
 def _shared_field_kind(field: str, *, cardinality: int, total: int, numeric_ratio: float) -> str:
@@ -413,7 +452,9 @@ def _shared_profile_fields(
             "candidate": candidate,
             "semantic_group": _semantic_field_group(field),
         })
-        if kind == "numeric":
+        if kind == "entity":
+            pass
+        elif kind == "numeric":
             numeric.append(field)
         elif kind == "categorical":
             categorical.append(field)
@@ -443,6 +484,14 @@ def _shared_profile_fields(
         else:
             stats_by_key[group].setdefault("aliases", aliases[:10])
             stats_by_key[group]["streams"] = max(int(stats_by_key[group].get("streams", 0) or 0), stream_count)
+        if kind == "numeric":
+            for field in present_aliases:
+                if field not in numeric:
+                    numeric.append(field)
+        elif kind == "categorical":
+            for field in present_aliases:
+                if field not in categorical:
+                    categorical.append(field)
     stats = sorted(
         stats_by_key.values(),
         key=lambda item: (
@@ -458,6 +507,21 @@ def _shared_profile_fields(
 def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
     profile = suggestion.get("profile", {}) if isinstance(suggestion.get("profile"), dict) else {}
     fields = {str(item.get("field", "")) for item in suggestion.get("common_fields", []) if isinstance(item, dict)}
+    discovery = suggestion.get("discovery", {}) if isinstance(suggestion.get("discovery"), dict) else {}
+    for item in discovery.get("top_fields", []) if isinstance(discovery.get("top_fields"), list) else []:
+        if isinstance(item, dict):
+            fields.add(str(item.get("field", "")))
+    for item in suggestion.get("shared_fields", []) if isinstance(suggestion.get("shared_fields"), list) else []:
+        if not isinstance(item, dict):
+            continue
+        fields.add(str(item.get("field", "")))
+        aliases = item.get("aliases", [])
+        if isinstance(aliases, list):
+            fields.update(str(alias) for alias in aliases if alias)
+    selected = discovery.get("selected_fields", {}) if isinstance(discovery.get("selected_fields"), dict) else {}
+    for value in selected.values():
+        if isinstance(value, list):
+            fields.update(str(field) for field in value if field)
     for key in ("entity_fields", "categorical_fields", "numeric_fields"):
         value = profile.get(key, [])
         if isinstance(value, list):
@@ -490,6 +554,19 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
             timestamp = str(heuristic.get("timestamp_field", "timestamp"))
         categorical = valid_fields("categorical_fields", list(heuristic.get("categorical_fields", [])) if isinstance(heuristic.get("categorical_fields"), list) else [])
         numeric = valid_fields("numeric_fields", list(heuristic.get("numeric_fields", [])) if isinstance(heuristic.get("numeric_fields"), list) else [])
+        relationships = []
+        raw_relationships = advisor.get("relationship_fields", [])
+        if isinstance(raw_relationships, list):
+            for relation in raw_relationships:
+                if not isinstance(relation, dict):
+                    continue
+                left = str(relation.get("left", "")).strip()
+                right = str(relation.get("right", "")).strip()
+                if left not in allowed_fields or right not in allowed_fields or left == right:
+                    continue
+                relationships.append({"left": left, "right": right, "name": str(relation.get("name", "") or f"{left}->{right}")})
+        if not relationships and isinstance(heuristic.get("relationship_fields"), list):
+            relationships = [item for item in heuristic.get("relationship_fields", []) if isinstance(item, dict)]
         detectors_raw = advisor.get("detectors", {})
         detectors = {}
         if isinstance(detectors_raw, dict):
@@ -503,6 +580,23 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
         if not detectors:
             detectors = dict(heuristic.get("detectors", {})) if isinstance(heuristic.get("detectors"), dict) else {}
         if entity_fields:
+            raw_field_roles = advisor.get("field_roles", {})
+            field_roles = {}
+            if isinstance(raw_field_roles, dict):
+                field_roles = {
+                    str(field): str(role)
+                    for field, role in raw_field_roles.items()
+                    if str(field) in allowed_fields and str(role)
+                }
+            raw_correlation_roles = advisor.get("correlation_roles", {})
+            correlation_roles = {}
+            if isinstance(raw_correlation_roles, dict):
+                for role, fields in raw_correlation_roles.items():
+                    if not isinstance(fields, list):
+                        continue
+                    valid = [str(field) for field in fields if str(field) in allowed_fields]
+                    if valid:
+                        correlation_roles[str(role)] = list(dict.fromkeys(valid))
             profile = {
                 **heuristic,
                 "entity_field": entity_fields[0],
@@ -511,9 +605,16 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
                 "categorical_fields": categorical,
                 "numeric_fields": numeric,
                 "detectors": detectors,
+                "relationship_fields": relationships,
+            }
+            item.update({"profile": profile, "entity_fields": entity_fields, "timestamp_field": timestamp, "categorical_fields": categorical, "numeric_fields": numeric, "detectors": detectors, "relationship_fields": relationships})
+            item["profile_advisor"] = {
+                "status": "ok",
+                "model": str(advisor.get("model", "")),
+                "reason": str(advisor.get("reason", "")),
+                "field_roles": field_roles,
+                "correlation_roles": correlation_roles,
             }
-            item.update({"profile": profile, "entity_fields": entity_fields, "timestamp_field": timestamp, "categorical_fields": categorical, "numeric_fields": numeric, "detectors": detectors})
-            item["profile_advisor"] = {"status": "ok", "model": str(advisor.get("model", "")), "reason": str(advisor.get("reason", ""))}
         else:
             item["profile_advisor"] = {"status": "invalid", "reason": "advisor returned no valid entity fields"}
         output.append(item)
@@ -539,10 +640,11 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
             if field not in IGNORED_DISCOVERY_FIELDS and _has_discovery_value(value)
         }
         stream_field_counts.update(fields)
-        semantic_groups = {_semantic_field_group(field) for field in fields}
+        semantic_groups = {group for field in fields for group in _semantic_field_groups(field)}
         stream_semantic_counts.update(semantic_groups)
         for field in fields:
-            stream_semantic_fields[_semantic_field_group(field)].add(field)
+            for group in _semantic_field_groups(field):
+                stream_semantic_fields[group].add(field)
 
     suggestions: list[dict[str, object]] = []
     for stream_id, stream_events in grouped.items():
@@ -673,6 +775,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
                 "numeric_fields": numeric,
                 "detectors": detectors,
                 "field_weights": {},
+                "relationship_fields": [],
             },
             "reason": "selected common entity, time, categorical, and numeric fields from observed events; non-priority fields are included when coverage and cardinality look useful",
         })

src/fgai/stream_profiles.py

@@ -3,6 +3,13 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 
 
+@dataclass(frozen=True)
+class RelationshipField:
+    left: str
+    right: str
+    name: str = ""
+
+
 @dataclass(frozen=True)
 class StreamProfile:
     stream_id: str
@@ -14,6 +21,7 @@ class StreamProfile:
     numeric_fields: tuple[str, ...] = ()
     detectors: dict[str, dict[str, object]] = field(default_factory=dict)
     field_weights: dict[str, object] = field(default_factory=dict)
+    relationship_fields: tuple[RelationshipField, ...] = ()
 
 
 def _detectors(value: object) -> dict[str, dict[str, object]]:
@@ -54,6 +62,25 @@ def _field_weights(value: object) -> dict[str, object]:
     return output
 
 
+def _relationship_fields(value: object) -> tuple[RelationshipField, ...]:
+    output: list[RelationshipField] = []
+    seen: set[tuple[str, str]] = set()
+    for item in value if isinstance(value, list) else []:
+        if not isinstance(item, dict):
+            continue
+        left = str(item.get("left") or item.get("entity") or item.get("from") or "").strip()
+        right = str(item.get("right") or item.get("value") or item.get("to") or "").strip()
+        if not left or not right or left == right:
+            continue
+        key = (left.lower(), right.lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        name = str(item.get("name") or f"{left}->{right}").strip()
+        output.append(RelationshipField(left=left, right=right, name=name))
+    return tuple(output)
+
+
 def parse_profiles(value: object) -> dict[str, StreamProfile]:
     profiles: dict[str, StreamProfile] = {}
     for item in value if isinstance(value, list) else []:
@@ -69,11 +96,13 @@ def parse_profiles(value: object) -> dict[str, StreamProfile]:
             continue
         detectors = _detectors(item.get("detectors", {}))
         field_weights = _field_weights(item.get("field_weights", {}))
+        relationship_fields = _relationship_fields(item.get("relationship_fields", []))
         profiles[stream_id] = StreamProfile(
             stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp, entity_fields,
             tuple(str(field) for field in item.get("categorical_fields", []) if field),
             tuple(str(field) for field in item.get("numeric_fields", []) if field),
             detectors,
             field_weights,
+            relationship_fields,
         )
     return profiles