From f2385a00a93a79cb72014875b1c8a5803b434a09bc2ec838ca62efc28bebc161 Mon Sep 17 00:00:00 2001 From: larssand Date: Tue, 30 Jun 2026 08:55:15 +0200 Subject: [PATCH] Profile recommendations --- src/fgai/detectors.py | 3 +- src/fgai/normalization.py | 5 +- src/fgai/profile_suggestions.py | 116 +++++++++++++++++++++++++++--- tests/test_normalization.py | 10 +++ tests/test_profile_suggestions.py | 36 ++++++++++ 5 files changed, 158 insertions(+), 12 deletions(-) diff --git a/src/fgai/detectors.py b/src/fgai/detectors.py index 5e808e0..714517a 100644 --- a/src/fgai/detectors.py +++ b/src/fgai/detectors.py @@ -2,6 +2,7 @@ from __future__ import annotations from .logs import THREAT_ACTIONS from .models import LogEvent +from .normalization import canonical_value DETECTOR_MINIMUMS = { @@ -14,7 +15,7 @@ DETECTOR_MINIMUMS = { def event_detector_categories(event: LogEvent) -> tuple[str, ...]: fields = event.fields action = event.action - event_id = fields.get("eventid", fields.get("event_id", fields.get("winlog_event_id", ""))) + event_id = canonical_value(fields, "eventid") categories: list[str] = [] if action in {"fail", "failed", "failure", "login_failed", "logon_failed", "authentication_failed"} or event_id == "4625": categories.append("auth_failure") diff --git a/src/fgai/normalization.py b/src/fgai/normalization.py index 1495b1a..3c170de 100644 --- a/src/fgai/normalization.py +++ b/src/fgai/normalization.py @@ -12,8 +12,11 @@ FIELD_ALIASES: dict[str, tuple[str, ...]] = { "type": ("type", "event_type", "log_type", "category", "event_category", "facility"), "subtype": ("subtype", "sub_type", "event_subtype", "subcategory"), "service": ("service", "service_name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "query_type", "qt"), + "username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "user", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"), + "hostname": ("hostname", "host", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "winlog_computer_name", "host.name", "agent.name"), + "eventid": ("eventid", "event_id", "winlog_event_id", "event_code", "event.code", "windows_event_id"), "dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"), - "context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message"), + "context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original"), } DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases)) diff --git a/src/fgai/profile_suggestions.py b/src/fgai/profile_suggestions.py index 57c56f5..ef0d0ea 100644 --- a/src/fgai/profile_suggestions.py +++ b/src/fgai/profile_suggestions.py @@ -3,19 +3,35 @@ from __future__ import annotations from collections import Counter, defaultdict from .detectors import event_detector_categories +from .entities import ENTITY_FIELDS from .models import LogEvent ENTITY_PRIORITY = ( "srcip", + "source_ip", + "client_ip", "username", + "actor_id", + "actor", + "principal", "user", + "user_name", + "account", + "account_name", + "targetusername", + "subjectusername", + "winlog_event_data_targetusername", + "winlog_event_data_subjectusername", "hostname", "host", + "computer", + "computer_name", + "winlog_computer_name", "source", - "client_ip", "ip", + "winlog_event_data_ipaddress", ) -TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at") +TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at", "winlog_timecreated", "event_created") CATEGORICAL_PRIORITY = ( "action", "severity", @@ -25,12 +41,35 @@ CATEGORICAL_PRIORITY = ( "dstport", "srcport", "policyid", + "eventid", + "event_id", + "winlog_event_id", + "event_code", + "channel", + "provider_name", + "logon_type", "dns_query", "query_domain", "url", + "username", + "hostname", + "eventid", "context", ) -NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "event_id") +NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code") +IGNORED_DISCOVERY_FIELDS = { + "message", + "msg", + "full_message", + "raw", + "answer", + "gl2_message_id", + "gl2_source_node", + "gl2_source_input", + "fgai_stream", + "fgai_stream_id", + "fgai_stream_name", +} def _stream_id(event: LogEvent) -> str: @@ -59,6 +98,59 @@ def _pick_present(priority: tuple[str, ...], coverage: Counter[str], total: int, return selected +def _looks_like_entity_field(field: str) -> bool: + field = field.lower() + if any(field in fields for fields in ENTITY_FIELDS.values()): + return True + tokens = ("user", "account", "actor", "principal", "login", "identity", "subject", "target", "host", "computer", "workstation", "client", "source", "src", "remote", "ipaddress", "ip") + return any(token in field for token in tokens) + + +def _looks_like_time_field(field: str) -> bool: + field = field.lower() + return any(token in field for token in ("time", "timestamp", "created", "@timestamp")) + + +def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]: + output = list(selected) + for field, count in coverage.most_common(): + if len(output) >= 4: + break + if field in output or field in IGNORED_DISCOVERY_FIELDS or not _looks_like_entity_field(field): + continue + coverage_ratio = count / max(1, total) + cardinality = len(unique_values[field]) + if coverage_ratio >= 0.05 and 1 < cardinality <= min(500, max(10, total)): + output.append(field) + return output + + +def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]: + output = list(selected) + for field, count in coverage.most_common(): + if len(output) >= 12: + break + if field in output or field in IGNORED_DISCOVERY_FIELDS: + continue + coverage_ratio = count / max(1, total) + cardinality = len(unique_values[field]) + if coverage_ratio >= 0.1 and 1 < cardinality <= min(200, max(8, int(total * 0.7))): + output.append(field) + return output + + +def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str], total: int, selected: list[str]) -> list[str]: + output = list(selected) + for field, count in coverage.most_common(): + if len(output) >= 8: + break + if field in output or field in IGNORED_DISCOVERY_FIELDS: + continue + if count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95: + output.append(field) + return output + + def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]: existing_profiles = existing_profiles or {} grouped: dict[str, list[LogEvent]] = defaultdict(list) @@ -86,15 +178,19 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s numeric_counts[field] += 1 detector_counts.update(event_detector_categories(event)) - entity_fields = _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3) - timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "timestamp") - categorical = _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8) - numeric = [ + entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3)) + timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "") + if not timestamp: + timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp") + categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8)) + numeric = _generic_numeric_fields(coverage, numeric_counts, total, [ field for field in NUMERIC_PRIORITY if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8 - ][:5] + ][:5]) if not entity_fields: - entity_fields = [field for field, _count in coverage.most_common(1)] + entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1] + if not entity_fields: + entity_fields = ["source"] detectors = { name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0} for name, count in detector_counts.items() @@ -132,6 +228,6 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s "detectors": detectors, "field_weights": {}, }, - "reason": "selected common entity, time, categorical, and numeric fields from observed events", + "reason": "selected common entity, time, categorical, and numeric fields from observed events; non-priority fields are included when coverage and cardinality look useful", }) return sorted(suggestions, key=lambda item: (bool(item["profile_exists"]), -int(item["score"]), str(item["stream_name"]))) diff --git a/tests/test_normalization.py b/tests/test_normalization.py index 3f77684..d2f30e7 100644 --- a/tests/test_normalization.py +++ b/tests/test_normalization.py @@ -34,6 +34,16 @@ class NormalizationTests(unittest.TestCase): self.assertEqual(row["service"], "tcp") self.assertEqual(row["context"], "allowed outbound session") + def test_identity_and_event_aliases_populate_common_fields(self): + event = parse_log_line( + "winlog_event_data_targetusername=alice winlog_computer_name=host01 " + "winlog_event_id=4625 actor_id=bob" + ) + + self.assertEqual(event.fields["username"], "alice") + self.assertEqual(event.fields["hostname"], "host01") + self.assertEqual(event.fields["eventid"], "4625") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_profile_suggestions.py b/tests/test_profile_suggestions.py index 421309a..0fd21ce 100644 --- a/tests/test_profile_suggestions.py +++ b/tests/test_profile_suggestions.py @@ -22,6 +22,42 @@ class ProfileSuggestionTests(unittest.TestCase): self.assertIn("hitcount", profile["numeric_fields"]) self.assertIn("deny_action", profile["detectors"]) + def test_suggests_windows_event_fields(self): + events = [ + parse_log_line( + f"fgai_stream_id=windows fgai_stream=Windows winlog_event_id=4625 " + f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} " + f"event_provider=Microsoft-Windows-Security-Auditing logon_type=3 " + f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure" + ) + for index in range(1, 30) + ] + + profile = suggest_stream_profiles(events)[0]["profile"] + + self.assertIn("username", profile["entity_fields"]) + self.assertIn("hostname", profile["entity_fields"]) + self.assertIn("eventid", profile["categorical_fields"]) + self.assertIn("logon_type", profile["categorical_fields"]) + self.assertIn("auth_failure", profile["detectors"]) + + def test_discovers_non_priority_application_fields(self): + events = [ + parse_log_line( + f"fgai_stream_id=app fgai_stream=BillingApp tenant_id=t{index % 4} actor_id=user{index % 7} " + f"workflow_state={'approved' if index % 2 else 'rejected'} payment_provider=stripe risk_points={index % 9} " + f"created_at=2026-06-29T10:00:{index:02d}Z" + ) + for index in range(1, 40) + ] + + profile = suggest_stream_profiles(events)[0]["profile"] + + self.assertIn("username", profile["entity_fields"]) + self.assertIn("tenant_id", profile["categorical_fields"]) + self.assertIn("workflow_state", profile["categorical_fields"]) + self.assertIn("risk_points", profile["numeric_fields"]) + if __name__ == "__main__": unittest.main()