Profile recommendations

src/fgai/detectors.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from .logs import THREAT_ACTIONS
 from .models import LogEvent
+from .normalization import canonical_value
 
 
 DETECTOR_MINIMUMS = {
@@ -14,7 +15,7 @@ DETECTOR_MINIMUMS = {
 def event_detector_categories(event: LogEvent) -> tuple[str, ...]:
     fields = event.fields
     action = event.action
-    event_id = fields.get("eventid", fields.get("event_id", fields.get("winlog_event_id", "")))
+    event_id = canonical_value(fields, "eventid")
     categories: list[str] = []
     if action in {"fail", "failed", "failure", "login_failed", "logon_failed", "authentication_failed"} or event_id == "4625":
         categories.append("auth_failure")

src/fgai/normalization.py

@@ -12,8 +12,11 @@ FIELD_ALIASES: dict[str, tuple[str, ...]] = {
     "type": ("type", "event_type", "log_type", "category", "event_category", "facility"),
     "subtype": ("subtype", "sub_type", "event_subtype", "subcategory"),
     "service": ("service", "service_name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "query_type", "qt"),
+    "username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "user", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"),
+    "hostname": ("hostname", "host", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "winlog_computer_name", "host.name", "agent.name"),
+    "eventid": ("eventid", "event_id", "winlog_event_id", "event_code", "event.code", "windows_event_id"),
     "dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"),
-    "context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message"),
+    "context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original"),
 }
 
 DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases))

src/fgai/profile_suggestions.py

@@ -3,19 +3,35 @@ from __future__ import annotations
 from collections import Counter, defaultdict
 
 from .detectors import event_detector_categories
+from .entities import ENTITY_FIELDS
 from .models import LogEvent
 
 ENTITY_PRIORITY = (
     "srcip",
+    "source_ip",
+    "client_ip",
     "username",
+    "actor_id",
+    "actor",
+    "principal",
     "user",
+    "user_name",
+    "account",
+    "account_name",
+    "targetusername",
+    "subjectusername",
+    "winlog_event_data_targetusername",
+    "winlog_event_data_subjectusername",
     "hostname",
     "host",
+    "computer",
+    "computer_name",
+    "winlog_computer_name",
     "source",
-    "client_ip",
     "ip",
+    "winlog_event_data_ipaddress",
 )
-TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at")
+TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at", "winlog_timecreated", "event_created")
 CATEGORICAL_PRIORITY = (
     "action",
     "severity",
@@ -25,12 +41,35 @@ CATEGORICAL_PRIORITY = (
     "dstport",
     "srcport",
     "policyid",
+    "eventid",
+    "event_id",
+    "winlog_event_id",
+    "event_code",
+    "channel",
+    "provider_name",
+    "logon_type",
     "dns_query",
     "query_domain",
     "url",
+    "username",
+    "hostname",
+    "eventid",
     "context",
 )
-NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "event_id")
+NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
+IGNORED_DISCOVERY_FIELDS = {
+    "message",
+    "msg",
+    "full_message",
+    "raw",
+    "answer",
+    "gl2_message_id",
+    "gl2_source_node",
+    "gl2_source_input",
+    "fgai_stream",
+    "fgai_stream_id",
+    "fgai_stream_name",
+}
 
 
 def _stream_id(event: LogEvent) -> str:
@@ -59,6 +98,59 @@ def _pick_present(priority: tuple[str, ...], coverage: Counter[str], total: int,
     return selected
 
 
+def _looks_like_entity_field(field: str) -> bool:
+    field = field.lower()
+    if any(field in fields for fields in ENTITY_FIELDS.values()):
+        return True
+    tokens = ("user", "account", "actor", "principal", "login", "identity", "subject", "target", "host", "computer", "workstation", "client", "source", "src", "remote", "ipaddress", "ip")
+    return any(token in field for token in tokens)
+
+
+def _looks_like_time_field(field: str) -> bool:
+    field = field.lower()
+    return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
+
+
+def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
+    output = list(selected)
+    for field, count in coverage.most_common():
+        if len(output) >= 4:
+            break
+        if field in output or field in IGNORED_DISCOVERY_FIELDS or not _looks_like_entity_field(field):
+            continue
+        coverage_ratio = count / max(1, total)
+        cardinality = len(unique_values[field])
+        if coverage_ratio >= 0.05 and 1 < cardinality <= min(500, max(10, total)):
+            output.append(field)
+    return output
+
+
+def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
+    output = list(selected)
+    for field, count in coverage.most_common():
+        if len(output) >= 12:
+            break
+        if field in output or field in IGNORED_DISCOVERY_FIELDS:
+            continue
+        coverage_ratio = count / max(1, total)
+        cardinality = len(unique_values[field])
+        if coverage_ratio >= 0.1 and 1 < cardinality <= min(200, max(8, int(total * 0.7))):
+            output.append(field)
+    return output
+
+
+def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str], total: int, selected: list[str]) -> list[str]:
+    output = list(selected)
+    for field, count in coverage.most_common():
+        if len(output) >= 8:
+            break
+        if field in output or field in IGNORED_DISCOVERY_FIELDS:
+            continue
+        if count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95:
+            output.append(field)
+    return output
+
+
 def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]:
     existing_profiles = existing_profiles or {}
     grouped: dict[str, list[LogEvent]] = defaultdict(list)
@@ -86,15 +178,19 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
                     numeric_counts[field] += 1
             detector_counts.update(event_detector_categories(event))
 
-        entity_fields = _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3)
+        entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3))
-        timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "timestamp")
+        timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
-        categorical = _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8)
+        if not timestamp:
-        numeric = [
+            timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
+        categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8))
+        numeric = _generic_numeric_fields(coverage, numeric_counts, total, [
             field for field in NUMERIC_PRIORITY
             if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
-        ][:5]
+        ][:5])
         if not entity_fields:
-            entity_fields = [field for field, _count in coverage.most_common(1)]
+            entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
+        if not entity_fields:
+            entity_fields = ["source"]
         detectors = {
             name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
             for name, count in detector_counts.items()
@@ -132,6 +228,6 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
                 "detectors": detectors,
                 "field_weights": {},
             },
-            "reason": "selected common entity, time, categorical, and numeric fields from observed events",
+            "reason": "selected common entity, time, categorical, and numeric fields from observed events; non-priority fields are included when coverage and cardinality look useful",
         })
     return sorted(suggestions, key=lambda item: (bool(item["profile_exists"]), -int(item["score"]), str(item["stream_name"])))

tests/test_normalization.py

@@ -34,6 +34,16 @@ class NormalizationTests(unittest.TestCase):
         self.assertEqual(row["service"], "tcp")
         self.assertEqual(row["context"], "allowed outbound session")
 
+    def test_identity_and_event_aliases_populate_common_fields(self):
+        event = parse_log_line(
+            "winlog_event_data_targetusername=alice winlog_computer_name=host01 "
+            "winlog_event_id=4625 actor_id=bob"
+        )
+
+        self.assertEqual(event.fields["username"], "alice")
+        self.assertEqual(event.fields["hostname"], "host01")
+        self.assertEqual(event.fields["eventid"], "4625")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_profile_suggestions.py

@@ -22,6 +22,42 @@ class ProfileSuggestionTests(unittest.TestCase):
         self.assertIn("hitcount", profile["numeric_fields"])
         self.assertIn("deny_action", profile["detectors"])
 
+    def test_suggests_windows_event_fields(self):
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=windows fgai_stream=Windows winlog_event_id=4625 "
+                f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} "
+                f"event_provider=Microsoft-Windows-Security-Auditing logon_type=3 "
+                f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure"
+            )
+            for index in range(1, 30)
+        ]
+
+        profile = suggest_stream_profiles(events)[0]["profile"]
+
+        self.assertIn("username", profile["entity_fields"])
+        self.assertIn("hostname", profile["entity_fields"])
+        self.assertIn("eventid", profile["categorical_fields"])
+        self.assertIn("logon_type", profile["categorical_fields"])
+        self.assertIn("auth_failure", profile["detectors"])
+
+    def test_discovers_non_priority_application_fields(self):
+        events = [
+            parse_log_line(
+                f"fgai_stream_id=app fgai_stream=BillingApp tenant_id=t{index % 4} actor_id=user{index % 7} "
+                f"workflow_state={'approved' if index % 2 else 'rejected'} payment_provider=stripe risk_points={index % 9} "
+                f"created_at=2026-06-29T10:00:{index:02d}Z"
+            )
+            for index in range(1, 40)
+        ]
+
+        profile = suggest_stream_profiles(events)[0]["profile"]
+
+        self.assertIn("username", profile["entity_fields"])
+        self.assertIn("tenant_id", profile["categorical_fields"])
+        self.assertIn("workflow_state", profile["categorical_fields"])
+        self.assertIn("risk_points", profile["numeric_fields"])
+
 
 if __name__ == "__main__":
     unittest.main()