MCP sample fetch now asks for common Winlogbeat/ECS fields before

This commit is contained in:
larssand
2026-06-30 15:10:47 +02:00
parent b8fe738aa1
commit 4a441bf443
8 changed files with 307 additions and 26 deletions

View File

@@ -72,6 +72,19 @@ validated against fields actually seen in the stream before it can be applied.
Unknown fields, raw message fields, internal `fgai_*` fields, and unknown Unknown fields, raw message fields, internal `fgai_*` fields, and unknown
detectors are rejected. detectors are rejected.
Windows-like streams are recognized from stream names such as `Windows`,
`Winlog`, `Security Event Log`, `Sysmon`, and `Powershell`, or from Windows event
fields. The MCP search asks for common Winlogbeat/ECS names such as
`winlog.event_id`, `event.code`, `user.name`, `host.name`, `source.ip`,
`winlog.channel`, and `process.name` before a profile exists. Their default
recommendation favors normalized `username`, `hostname`, and `srcip` as
entities, then uses the observed Windows fields such as event ID,
action/outcome, channel/provider, `logon_type`, process/service fields, and
event category/type as categorical baseline fields. The authentication-failure
detector is enabled by default. You can still edit the applied profile per
stream when your Windows parser uses different field names or when a stream
contains a narrower log type.
Settings also lists locally installed Ollama models from `http://127.0.0.1:11434/api/tags`. Settings also lists locally installed Ollama models from `http://127.0.0.1:11434/api/tags`.
Click a model name to fill both the dashboard analyst model and profile advisor Click a model name to fill both the dashboard analyst model and profile advisor
model fields. model fields.
@@ -445,9 +458,9 @@ filebeat -e -c examples/filebeat-windows-synthetic.yml
Update the absolute JSONL path and Graylog host in the Filebeat template first. Update the absolute JSONL path and Graylog host in the Filebeat template first.
Route `stream_hint: Windows` to a dedicated Graylog stream, then enable that Route `stream_hint: Windows` to a dedicated Graylog stream, then enable that
stream in SignalScope and configure a profile such as entity `user` or stream in SignalScope. The Settings page should recommend a Windows profile once
`source_ip`, categorical `event_id`, `status`, `logon_type`, and numeric fields sample events have been fetched; apply it and adjust the entity/categorical
when present. Filebeat uses its Logstash output to communicate with Graylog's fields if your parser uses different names. Filebeat uses its Logstash output to communicate with Graylog's
Beats input on TCP `5044`. [Graylog Beats input documentation](https://go2docs.graylog.org/current/getting_in_log_data/beats_input.html) Beats input on TCP `5044`. [Graylog Beats input documentation](https://go2docs.graylog.org/current/getting_in_log_data/beats_input.html)
For logs, configure FortiGate syslog to write into a local file such as `logs/fg_syslog.jsonl`. The parser supports common key/value syslog lines and JSONL. For logs, configure FortiGate syslog to write into a local file such as `logs/fg_syslog.jsonl`. The parser supports common key/value syslog lines and JSONL.

View File

@@ -9,9 +9,9 @@ from .query_details import event_query_details
ENTITY_FIELDS: dict[str, tuple[str, ...]] = { ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
"ip": ("srcip", "src_ip", "source_ip", "client_ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "event_data_ipaddress"), "ip": ("srcip", "src_ip", "source_ip", "source.ip", "client_ip", "client.ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "event_data_ipaddress"),
"user": ("username", "user", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog_event_data_subjectusername"), "user": ("username", "user", "user.name", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername"),
"host": ("hostname", "host", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "agent_name"), "host": ("hostname", "host", "host.name", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname", "agent_name"),
} }

View File

@@ -8,7 +8,30 @@ from .models import LogEvent
from .normalization import DEFAULT_SEARCH_FIELDS, normalize_fields from .normalization import DEFAULT_SEARCH_FIELDS, normalize_fields
DEFAULT_FIELDS = list(dict.fromkeys([*DEFAULT_SEARCH_FIELDS, "source", "eventid", "event_id", "winlog_event_id", "policyid", "policy_id", "policy", "rule", "rule_name", "sentbyte", "rcvdbyte", "bytes_in", "bytes_out", "hitcount", "elapsed"])) WINDOWS_DISCOVERY_FIELDS = (
"winlog.channel",
"winlog.provider_name",
"winlog.record_id",
"winlog.process.pid",
"winlog.process.thread.id",
"winlog.task",
"winlog.opcode",
"winlog.keywords",
"winlog.event_data.logontype",
"winlog.event_data.logon_type",
"winlog.event_data.authenticationpackagename",
"winlog.event_data.logonprocessname",
"winlog.event_data.failurereason",
"winlog.event_data.status",
"winlog.event_data.substatus",
"process.name",
"process.executable",
"process.pid",
"process.command_line",
"service.name",
)
DEFAULT_FIELDS = list(dict.fromkeys([*DEFAULT_SEARCH_FIELDS, *WINDOWS_DISCOVERY_FIELDS, "source", "eventid", "event_id", "winlog_event_id", "policyid", "policy_id", "policy", "rule", "rule_name", "sentbyte", "rcvdbyte", "bytes_in", "bytes_out", "hitcount", "elapsed"]))
def _records(value: object) -> Iterable[dict[str, object]]: def _records(value: object) -> Iterable[dict[str, object]]:

View File

@@ -2,21 +2,21 @@ from __future__ import annotations
FIELD_ALIASES: dict[str, tuple[str, ...]] = { FIELD_ALIASES: dict[str, tuple[str, ...]] = {
"srcip": ("srcip", "src_ip", "source_ip", "source.ip", "src", "src_addr", "srcaddr", "sourceaddress", "client_ip", "client.ip", "clientip", "client", "remote_addr", "remote_ip", "ip", "ipaddress", "gl2_remote_ip", "winlog_event_data_ipaddress", "event_data_ipaddress"), "srcip": ("srcip", "src_ip", "source_ip", "source.ip", "source.address", "src", "src_addr", "srcaddr", "sourceaddress", "client_ip", "client.ip", "client.address", "clientip", "client", "remote_addr", "remote_ip", "ip", "ipaddress", "related.ip", "gl2_remote_ip", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "event_data_ipaddress"),
"dstip": ("dstip", "dst_ip", "destination_ip", "destination.ip", "dst", "dst_addr", "dstaddr", "destinationaddress", "dest_ip", "dest", "server_ip", "server.ip", "server", "upstream"), "dstip": ("dstip", "dst_ip", "destination_ip", "destination.ip", "destination.address", "dst", "dst_addr", "dstaddr", "destinationaddress", "dest_ip", "dest", "server_ip", "server.ip", "server.address", "server", "upstream"),
"srcport": ("srcport", "src_port", "source_port", "source.port", "sport", "client_port", "client.port", "clientport"), "srcport": ("srcport", "src_port", "source_port", "source.port", "sport", "client_port", "client.port", "clientport"),
"dstport": ("dstport", "dst_port", "destination_port", "destination.port", "dest_port", "dport", "server_port", "server.port", "serverport"), "dstport": ("dstport", "dst_port", "destination_port", "destination.port", "dest_port", "dport", "server_port", "server.port", "serverport"),
"eventtime": ("eventtime", "timestamp", "time", "event_time", "eventtime_ms", "created_at"), "eventtime": ("eventtime", "timestamp", "@timestamp", "time", "event_time", "eventtime_ms", "created_at", "event.created", "event.ingested", "winlog.time_created"),
"severity": ("severity", "level", "priority", "sev", "loglevel", "log_level", "event_severity"), "severity": ("severity", "level", "priority", "sev", "loglevel", "log_level", "log.level", "event_severity", "event.severity", "winlog.level", "winlog.level_name"),
"action": ("action", "act", "fw_action", "rule_action", "policy_action", "event_action", "disposition", "outcome", "result", "status", "event_outcome"), "action": ("action", "act", "fw_action", "rule_action", "policy_action", "event_action", "event.action", "disposition", "outcome", "result", "status", "event_outcome", "event.outcome"),
"type": ("type", "event_type", "log_type", "category", "event_category", "facility"), "type": ("type", "event_type", "log_type", "category", "event_category", "event.category", "event.kind", "event.module", "event.dataset", "facility"),
"subtype": ("subtype", "sub_type", "event_subtype", "subcategory"), "subtype": ("subtype", "sub_type", "event_subtype", "subcategory", "event.type"),
"service": ("service", "service_name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "query_type", "qt"), "service": ("service", "service_name", "service.name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "network.protocol", "query_type", "qt"),
"username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "user", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"), "username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog.event_data.target_user_name", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername", "winlog.event_data.subject_user_name", "user", "user.name", "user.id", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"),
"hostname": ("hostname", "host", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "winlog_computer_name", "host.name", "agent.name"), "hostname": ("hostname", "host", "host.name", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "agent.hostname", "agent.name", "observer.hostname", "winlog_computer_name", "winlog.computer_name"),
"eventid": ("eventid", "event_id", "winlog_event_id", "event_code", "event.code", "windows_event_id"), "eventid": ("eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code", "windows_event_id"),
"dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"), "dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"),
"context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original"), "context": ("query_domain", "qh", "dns_query", "url", "url.full", "url.path", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original", "event.original", "process.command_line", "powershell.command.value"),
} }
DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases)) DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases))

View File

@@ -57,6 +57,99 @@ CATEGORICAL_PRIORITY = (
"context", "context",
) )
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code") NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
WINDOWS_ENTITY_PRIORITY = (
"username",
"user.name",
"user.id",
"hostname",
"host.name",
"srcip",
"source.ip",
"client.ip",
"source",
"targetusername",
"winlog_event_data_targetusername",
"winlog.event_data.targetusername",
"winlog.event_data.target_user_name",
"subjectusername",
"winlog_event_data_subjectusername",
"winlog.event_data.subjectusername",
"winlog.event_data.subject_user_name",
"user",
"account_name",
"computer",
"computer_name",
"winlog_computer_name",
"winlog.computer_name",
"agent.hostname",
"source_ip",
"client_ip",
"winlog_event_data_ipaddress",
"winlog.event_data.ipaddress",
"winlog.event_data.ip_address",
"ipaddress",
)
WINDOWS_CATEGORICAL_PRIORITY = (
"eventid",
"event_id",
"winlog_event_id",
"winlog.event_id",
"event_code",
"event.code",
"action",
"event.action",
"event.outcome",
"severity",
"channel",
"winlog.channel",
"provider_name",
"event_provider",
"event.provider",
"winlog_provider_name",
"winlog.provider_name",
"logon_type",
"winlog.event_data.logontype",
"winlog.event_data.logon_type",
"event.category",
"event.type",
"event.kind",
"event.module",
"event.dataset",
"task",
"winlog.task",
"opcode",
"winlog.opcode",
"keywords",
"winlog.keywords",
"process_name",
"process.name",
"process.executable",
"image",
"service_name",
"service.name",
"authentication_package_name",
"winlog.event_data.authenticationpackagename",
"logon_process_name",
"winlog.event_data.logonprocessname",
"failure_reason",
"winlog.event_data.failurereason",
"sub_status",
"winlog.event_data.substatus",
"status",
"winlog.event_data.status",
)
WINDOWS_NUMERIC_PRIORITY = (
"event_record_id",
"record_id",
"winlog_record_id",
"winlog.record_id",
"process_id",
"process.pid",
"winlog_process_pid",
"winlog.process.pid",
"thread_id",
"winlog.process.thread.id",
)
ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"} ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"}
IGNORED_DISCOVERY_FIELDS = { IGNORED_DISCOVERY_FIELDS = {
"message", "message",
@@ -112,6 +205,43 @@ def _looks_like_time_field(field: str) -> bool:
return any(token in field for token in ("time", "timestamp", "created", "@timestamp")) return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
def _looks_like_windows_stream(stream_id: str, stream_name: str, coverage: Counter[str]) -> bool:
name = f"{stream_id} {stream_name}".lower()
if any(token in name for token in ("windows", "winlog", "event log", "security event", "sysmon", "powershell")):
return True
windows_fields = {
"winlog_event_id",
"winlog.event_id",
"winlog_computer_name",
"winlog.computer_name",
"winlog_event_data_targetusername",
"winlog.event_data.targetusername",
"winlog_event_data_subjectusername",
"winlog.event_data.subjectusername",
"eventid",
"event_code",
"event.code",
"logon_type",
"winlog.channel",
}
return sum(1 for field in windows_fields if coverage[field]) >= 2
def _compact_windows_fields(fields: list[str]) -> list[str]:
output = list(dict.fromkeys(fields))
alias_groups = (
("username", "user.name", "user.id", "targetusername", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog.event_data.target_user_name", "subjectusername", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername", "winlog.event_data.subject_user_name", "user", "account_name"),
("hostname", "host.name", "computer", "computer_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname"),
("srcip", "source.ip", "source_ip", "client.ip", "client_ip", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "ipaddress"),
("eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code"),
("action", "event.action", "event.outcome"),
)
for canonical, *aliases in alias_groups:
if canonical in output:
output = [field for field in output if field == canonical or field not in aliases]
return output
def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]: def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
output = list(selected) output = list(selected)
for field, count in coverage.most_common(): for field, count in coverage.most_common():
@@ -244,24 +374,59 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
numeric_counts[field] += 1 numeric_counts[field] += 1
detector_counts.update(event_detector_categories(event)) detector_counts.update(event_detector_categories(event))
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3)) stream_name = names.get(stream_id, stream_id)
is_windows = _looks_like_windows_stream(stream_id, stream_name, coverage)
entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3))
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "") timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
if not timestamp: if not timestamp:
timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp") timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8)) categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(categorical_priority, coverage, total, min_ratio=0.02, limit=10 if is_windows else 8))
numeric = _generic_numeric_fields(coverage, numeric_counts, total, [ numeric_seed = [
field for field in NUMERIC_PRIORITY field for field in numeric_priority
if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8 if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
][:5]) ][:5]
numeric = numeric_seed if is_windows else _generic_numeric_fields(coverage, numeric_counts, total, numeric_seed)
if not entity_fields: if not entity_fields:
entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1] entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
if not entity_fields: if not entity_fields:
entity_fields = ["source"] entity_fields = ["source"]
if is_windows:
entity_fields = _compact_windows_fields(entity_fields)[:4]
categorical = _compact_windows_fields(categorical)
entity_set = set(entity_fields)
categorical = [
field for field in categorical
if field not in entity_set and field not in {
"srcip", "source.ip", "source_ip", "client.ip", "client_ip",
"hostname", "host.name", "computer", "username", "user.name",
"source", "context",
}
][:10]
for field in WINDOWS_CATEGORICAL_PRIORITY:
if len(categorical) >= 10:
break
compacted = _compact_windows_fields([*categorical, field])
if field not in compacted or field in categorical:
continue
if not coverage[field] or len(unique_values[field]) <= 1:
continue
if field in entity_set or field in {"srcip", "source.ip", "client.ip", "hostname", "host.name", "username", "user.name"}:
continue
categorical = compacted
numeric = [
field for field in _compact_windows_fields(numeric)
if field not in set(categorical) and field not in {"eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code", "logon_type"}
][:5]
detectors = { detectors = {
name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0} name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
for name, count in detector_counts.items() for name, count in detector_counts.items()
if count > 0 if count > 0
} }
if is_windows and coverage["eventid"]:
detectors.setdefault("auth_failure", {"enabled": True, "minimum": 5, "z_threshold": 3.0})
if not detectors: if not detectors:
detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}} detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
high_coverage = [ high_coverage = [

View File

@@ -88,6 +88,17 @@ class GraylogSourceTests(unittest.TestCase):
self.assertIn("fw_action", client.arguments["fields"]) self.assertIn("fw_action", client.arguments["fields"])
self.assertIn("full_message", client.arguments["fields"]) self.assertIn("full_message", client.arguments["fields"])
def test_requests_winlogbeat_discovery_fields_before_profile_exists(self):
client = _Client()
GraylogStreamSource(client, "windows").fetch()
self.assertIn("winlog.event_id", client.arguments["fields"])
self.assertIn("event.code", client.arguments["fields"])
self.assertIn("user.name", client.arguments["fields"])
self.assertIn("host.name", client.arguments["fields"])
self.assertIn("source.ip", client.arguments["fields"])
self.assertIn("winlog.channel", client.arguments["fields"])
self.assertIn("process.name", client.arguments["fields"])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View File

@@ -44,6 +44,22 @@ class NormalizationTests(unittest.TestCase):
self.assertEqual(event.fields["hostname"], "host01") self.assertEqual(event.fields["hostname"], "host01")
self.assertEqual(event.fields["eventid"], "4625") self.assertEqual(event.fields["eventid"], "4625")
def test_winlogbeat_ecs_aliases_populate_common_fields(self):
event = parse_log_line(
"@timestamp=2026-06-30T10:00:00Z event.code=4625 event.action=logon-failed "
"event.outcome=failure log.level=warning user.name=alice host.name=win01 "
"source.ip=10.0.0.5 winlog.channel=Security winlog.provider_name=Microsoft-Windows-Security-Auditing "
"winlog.event_data.LogonType=3 process.name=lsass.exe"
)
self.assertEqual(event.fields["eventtime"], "2026-06-30T10:00:00Z")
self.assertEqual(event.fields["eventid"], "4625")
self.assertEqual(event.fields["action"], "logon-failed")
self.assertEqual(event.fields["severity"], "warning")
self.assertEqual(event.fields["username"], "alice")
self.assertEqual(event.fields["hostname"], "win01")
self.assertEqual(event.fields["srcip"], "10.0.0.5")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View File

@@ -25,9 +25,10 @@ class ProfileSuggestionTests(unittest.TestCase):
def test_suggests_windows_event_fields(self): def test_suggests_windows_event_fields(self):
events = [ events = [
parse_log_line( parse_log_line(
f"fgai_stream_id=windows fgai_stream=Windows winlog_event_id=4625 " f"fgai_stream_id=windows fgai_stream='Illuminate:Windows Security Event Log Messages' winlog_event_id=4625 "
f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} " f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} "
f"event_provider=Microsoft-Windows-Security-Auditing logon_type=3 " f"event_provider=Microsoft-Windows-Security-Auditing channel=Security logon_type=3 "
f"winlog_event_data_ipaddress=10.0.0.{index % 5} "
f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure" f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure"
) )
for index in range(1, 30) for index in range(1, 30)
@@ -37,8 +38,60 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("username", profile["entity_fields"]) self.assertIn("username", profile["entity_fields"])
self.assertIn("hostname", profile["entity_fields"]) self.assertIn("hostname", profile["entity_fields"])
self.assertIn("srcip", profile["entity_fields"])
self.assertNotIn("winlog_event_data_targetusername", profile["entity_fields"])
self.assertIn("eventid", profile["categorical_fields"]) self.assertIn("eventid", profile["categorical_fields"])
self.assertIn("logon_type", profile["categorical_fields"]) self.assertIn("logon_type", profile["categorical_fields"])
self.assertIn("event_provider", profile["categorical_fields"])
self.assertNotIn("policyid", profile["categorical_fields"])
self.assertNotIn("dstport", profile["categorical_fields"])
self.assertNotIn("srcip", profile["categorical_fields"])
self.assertNotIn("eventid", profile["numeric_fields"])
self.assertIn("auth_failure", profile["detectors"])
def test_windows_named_stream_gets_auth_profile_even_before_failures(self):
events = [
parse_log_line(
f"fgai_stream_id=winsec fgai_stream='Illuminate:Windows Security Event Log Messages' "
f"eventid=4624 username=user{index % 4} hostname=host{index % 2} channel=Security "
f"eventtime=2026-06-29T10:00:{index:02d}Z action=success"
)
for index in range(1, 30)
]
suggestion = suggest_stream_profiles(events)[0]
profile = suggestion["profile"]
self.assertEqual(suggestion["stream_name"], "Illuminate:Windows Security Event Log Messages")
self.assertIn("username", profile["entity_fields"])
self.assertIn("hostname", profile["entity_fields"])
self.assertIn("eventid", profile["categorical_fields"])
self.assertIn("auth_failure", profile["detectors"])
def test_winlogbeat_ecs_fields_build_windows_profile(self):
events = [
parse_log_line(
f"fgai_stream_id=winbeat fgai_stream='Winlogbeat Security' "
f"@timestamp=2026-06-30T10:00:{index:02d}Z event.code=4625 event.action=logon-failed "
f"event.outcome=failure event.category=authentication event.type=start "
f"user.name=user{index % 4} host.name=win{index % 3} source.ip=10.0.0.{index % 5} "
f"winlog.channel=Security winlog.provider_name=Microsoft-Windows-Security-Auditing "
f"winlog.event_data.LogonType=3 process.name=proc{index % 3}.exe"
)
for index in range(1, 30)
]
profile = suggest_stream_profiles(events)[0]["profile"]
self.assertEqual(profile["timestamp_field"], "eventtime")
self.assertEqual(profile["entity_fields"][:3], ["username", "hostname", "srcip"])
self.assertIn("eventid", profile["categorical_fields"])
self.assertIn("action", profile["categorical_fields"])
self.assertIn("event.category", profile["categorical_fields"])
self.assertIn("winlog.channel", profile["categorical_fields"])
self.assertIn("process.name", profile["categorical_fields"])
self.assertNotIn("user.name", profile["categorical_fields"])
self.assertNotIn("source.ip", profile["categorical_fields"])
self.assertIn("auth_failure", profile["detectors"]) self.assertIn("auth_failure", profile["detectors"])
def test_discovers_non_priority_application_fields(self): def test_discovers_non_priority_application_fields(self):