Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Profile recommendations

Browse Source
This commit is contained in:
larssand
2026-06-30 08:55:15 +02:00
parent d99fe24b84
commit f2385a00a9
5 changed files with 158 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/fgai/detectors.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
from .logs import THREAT_ACTIONS
from .models import LogEvent
from .normalization import canonical_value
DETECTOR_MINIMUMS = {
@@ -14,7 +15,7 @@ DETECTOR_MINIMUMS = {
def event_detector_categories(event: LogEvent) -> tuple[str, ...]:
fields = event.fields
action = event.action
event_id = fields.get("eventid", fields.get("event_id", fields.get("winlog_event_id", "")))
event_id = canonical_value(fields, "eventid")
categories: list[str] = []
if action in {"fail", "failed", "failure", "login_failed", "logon_failed", "authentication_failed"} or event_id == "4625":
categories.append("auth_failure")

5
src/fgai/normalization.py
View File

@@ -12,8 +12,11 @@ FIELD_ALIASES: dict[str, tuple[str, ...]] = {
"type": ("type", "event_type", "log_type", "category", "event_category", "facility"),
"subtype": ("subtype", "sub_type", "event_subtype", "subcategory"),
"service": ("service", "service_name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "query_type", "qt"),
"username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "user", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"),
"hostname": ("hostname", "host", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "winlog_computer_name", "host.name", "agent.name"),
"eventid": ("eventid", "event_id", "winlog_event_id", "event_code", "event.code", "windows_event_id"),
"dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"),
"context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message"),
"context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original"),
}
DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases))

116
src/fgai/profile_suggestions.py
View File

@@ -3,19 +3,35 @@ from __future__ import annotations
from collections import Counter, defaultdict
from .detectors import event_detector_categories
from .entities import ENTITY_FIELDS
from .models import LogEvent
ENTITY_PRIORITY = (
"srcip",
"source_ip",
"client_ip",
"username",
"actor_id",
"actor",
"principal",
"user",
"user_name",
"account",
"account_name",
"targetusername",
"subjectusername",
"winlog_event_data_targetusername",
"winlog_event_data_subjectusername",
"hostname",
"host",
"computer",
"computer_name",
"winlog_computer_name",
"source",
"client_ip",
"ip",
"winlog_event_data_ipaddress",
)
TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at")
TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at", "winlog_timecreated", "event_created")
CATEGORICAL_PRIORITY = (
"action",
"severity",
@@ -25,12 +41,35 @@ CATEGORICAL_PRIORITY = (
"dstport",
"srcport",
"policyid",
"eventid",
"event_id",
"winlog_event_id",
"event_code",
"channel",
"provider_name",
"logon_type",
"dns_query",
"query_domain",
"url",
"username",
"hostname",
"eventid",
"context",
)
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "event_id")
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
IGNORED_DISCOVERY_FIELDS = {
"message",
"msg",
"full_message",
"raw",
"answer",
"gl2_message_id",
"gl2_source_node",
"gl2_source_input",
"fgai_stream",
"fgai_stream_id",
"fgai_stream_name",
}
def _stream_id(event: LogEvent) -> str:
@@ -59,6 +98,59 @@ def _pick_present(priority: tuple[str, ...], coverage: Counter[str], total: int,
return selected
def _looks_like_entity_field(field: str) -> bool:
field = field.lower()
if any(field in fields for fields in ENTITY_FIELDS.values()):
return True
tokens = ("user", "account", "actor", "principal", "login", "identity", "subject", "target", "host", "computer", "workstation", "client", "source", "src", "remote", "ipaddress", "ip")
return any(token in field for token in tokens)
def _looks_like_time_field(field: str) -> bool:
field = field.lower()
return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
output = list(selected)
for field, count in coverage.most_common():
if len(output) >= 4:
break
if field in output or field in IGNORED_DISCOVERY_FIELDS or not _looks_like_entity_field(field):
continue
coverage_ratio = count / max(1, total)
cardinality = len(unique_values[field])
if coverage_ratio >= 0.05 and 1 < cardinality <= min(500, max(10, total)):
output.append(field)
return output
def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
output = list(selected)
for field, count in coverage.most_common():
if len(output) >= 12:
break
if field in output or field in IGNORED_DISCOVERY_FIELDS:
continue
coverage_ratio = count / max(1, total)
cardinality = len(unique_values[field])
if coverage_ratio >= 0.1 and 1 < cardinality <= min(200, max(8, int(total * 0.7))):
output.append(field)
return output
def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str], total: int, selected: list[str]) -> list[str]:
output = list(selected)
for field, count in coverage.most_common():
if len(output) >= 8:
break
if field in output or field in IGNORED_DISCOVERY_FIELDS:
continue
if count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95:
output.append(field)
return output
def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]:
existing_profiles = existing_profiles or {}
grouped: dict[str, list[LogEvent]] = defaultdict(list)
@@ -86,15 +178,19 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
numeric_counts[field] += 1
detector_counts.update(event_detector_categories(event))
entity_fields = _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3)
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "timestamp")
categorical = _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8)
numeric = [
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3))
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
if not timestamp:
timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8))
numeric = _generic_numeric_fields(coverage, numeric_counts, total, [
field for field in NUMERIC_PRIORITY
if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
][:5]
][:5])
if not entity_fields:
entity_fields = [field for field, _count in coverage.most_common(1)]
entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
if not entity_fields:
entity_fields = ["source"]
detectors = {
name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
for name, count in detector_counts.items()
@@ -132,6 +228,6 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
"detectors": detectors,
"field_weights": {},
},
"reason": "selected common entity, time, categorical, and numeric fields from observed events",
"reason": "selected common entity, time, categorical, and numeric fields from observed events; non-priority fields are included when coverage and cardinality look useful",
})
return sorted(suggestions, key=lambda item: (bool(item["profile_exists"]), -int(item["score"]), str(item["stream_name"])))

10
tests/test_normalization.py
View File

@@ -34,6 +34,16 @@ class NormalizationTests(unittest.TestCase):
self.assertEqual(row["service"], "tcp")
self.assertEqual(row["context"], "allowed outbound session")
def test_identity_and_event_aliases_populate_common_fields(self):
event = parse_log_line(
"winlog_event_data_targetusername=alice winlog_computer_name=host01 "
"winlog_event_id=4625 actor_id=bob"
)
self.assertEqual(event.fields["username"], "alice")
self.assertEqual(event.fields["hostname"], "host01")
self.assertEqual(event.fields["eventid"], "4625")
if __name__ == "__main__":
unittest.main()

36
tests/test_profile_suggestions.py
View File

@@ -22,6 +22,42 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("hitcount", profile["numeric_fields"])
self.assertIn("deny_action", profile["detectors"])
def test_suggests_windows_event_fields(self):
events = [
parse_log_line(
f"fgai_stream_id=windows fgai_stream=Windows winlog_event_id=4625 "
f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} "
f"event_provider=Microsoft-Windows-Security-Auditing logon_type=3 "
f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure"
)
for index in range(1, 30)
]
profile = suggest_stream_profiles(events)[0]["profile"]
self.assertIn("username", profile["entity_fields"])
self.assertIn("hostname", profile["entity_fields"])
self.assertIn("eventid", profile["categorical_fields"])
self.assertIn("logon_type", profile["categorical_fields"])
self.assertIn("auth_failure", profile["detectors"])
def test_discovers_non_priority_application_fields(self):
events = [
parse_log_line(
f"fgai_stream_id=app fgai_stream=BillingApp tenant_id=t{index % 4} actor_id=user{index % 7} "
f"workflow_state={'approved' if index % 2 else 'rejected'} payment_provider=stripe risk_points={index % 9} "
f"created_at=2026-06-29T10:00:{index:02d}Z"
)
for index in range(1, 40)
]
profile = suggest_stream_profiles(events)[0]["profile"]
self.assertIn("username", profile["entity_fields"])
self.assertIn("tenant_id", profile["categorical_fields"])
self.assertIn("workflow_state", profile["categorical_fields"])
self.assertIn("risk_points", profile["numeric_fields"])
if __name__ == "__main__":
unittest.main()