MCP sample fetch now asks for common Winlogbeat/ECS fields before
This commit is contained in:
19
README.md
19
README.md
@@ -72,6 +72,19 @@ validated against fields actually seen in the stream before it can be applied.
|
|||||||
Unknown fields, raw message fields, internal `fgai_*` fields, and unknown
|
Unknown fields, raw message fields, internal `fgai_*` fields, and unknown
|
||||||
detectors are rejected.
|
detectors are rejected.
|
||||||
|
|
||||||
|
Windows-like streams are recognized from stream names such as `Windows`,
|
||||||
|
`Winlog`, `Security Event Log`, `Sysmon`, and `Powershell`, or from Windows event
|
||||||
|
fields. The MCP search asks for common Winlogbeat/ECS names such as
|
||||||
|
`winlog.event_id`, `event.code`, `user.name`, `host.name`, `source.ip`,
|
||||||
|
`winlog.channel`, and `process.name` before a profile exists. Their default
|
||||||
|
recommendation favors normalized `username`, `hostname`, and `srcip` as
|
||||||
|
entities, then uses the observed Windows fields such as event ID,
|
||||||
|
action/outcome, channel/provider, `logon_type`, process/service fields, and
|
||||||
|
event category/type as categorical baseline fields. The authentication-failure
|
||||||
|
detector is enabled by default. You can still edit the applied profile per
|
||||||
|
stream when your Windows parser uses different field names or when a stream
|
||||||
|
contains a narrower log type.
|
||||||
|
|
||||||
Settings also lists locally installed Ollama models from `http://127.0.0.1:11434/api/tags`.
|
Settings also lists locally installed Ollama models from `http://127.0.0.1:11434/api/tags`.
|
||||||
Click a model name to fill both the dashboard analyst model and profile advisor
|
Click a model name to fill both the dashboard analyst model and profile advisor
|
||||||
model fields.
|
model fields.
|
||||||
@@ -445,9 +458,9 @@ filebeat -e -c examples/filebeat-windows-synthetic.yml
|
|||||||
|
|
||||||
Update the absolute JSONL path and Graylog host in the Filebeat template first.
|
Update the absolute JSONL path and Graylog host in the Filebeat template first.
|
||||||
Route `stream_hint: Windows` to a dedicated Graylog stream, then enable that
|
Route `stream_hint: Windows` to a dedicated Graylog stream, then enable that
|
||||||
stream in SignalScope and configure a profile such as entity `user` or
|
stream in SignalScope. The Settings page should recommend a Windows profile once
|
||||||
`source_ip`, categorical `event_id`, `status`, `logon_type`, and numeric fields
|
sample events have been fetched; apply it and adjust the entity/categorical
|
||||||
when present. Filebeat uses its Logstash output to communicate with Graylog's
|
fields if your parser uses different names. Filebeat uses its Logstash output to communicate with Graylog's
|
||||||
Beats input on TCP `5044`. [Graylog Beats input documentation](https://go2docs.graylog.org/current/getting_in_log_data/beats_input.html)
|
Beats input on TCP `5044`. [Graylog Beats input documentation](https://go2docs.graylog.org/current/getting_in_log_data/beats_input.html)
|
||||||
|
|
||||||
For logs, configure FortiGate syslog to write into a local file such as `logs/fg_syslog.jsonl`. The parser supports common key/value syslog lines and JSONL.
|
For logs, configure FortiGate syslog to write into a local file such as `logs/fg_syslog.jsonl`. The parser supports common key/value syslog lines and JSONL.
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ from .query_details import event_query_details
|
|||||||
|
|
||||||
|
|
||||||
ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
|
ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
|
||||||
"ip": ("srcip", "src_ip", "source_ip", "client_ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "event_data_ipaddress"),
|
"ip": ("srcip", "src_ip", "source_ip", "source.ip", "client_ip", "client.ip", "remote_addr", "remote_ip", "ip", "ipaddress", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "event_data_ipaddress"),
|
||||||
"user": ("username", "user", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog_event_data_subjectusername"),
|
"user": ("username", "user", "user.name", "user_name", "account", "account_name", "targetusername", "subjectusername", "xauthuser", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername"),
|
||||||
"host": ("hostname", "host", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "agent_name"),
|
"host": ("hostname", "host", "host.name", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname", "agent_name"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,30 @@ from .models import LogEvent
|
|||||||
from .normalization import DEFAULT_SEARCH_FIELDS, normalize_fields
|
from .normalization import DEFAULT_SEARCH_FIELDS, normalize_fields
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_FIELDS = list(dict.fromkeys([*DEFAULT_SEARCH_FIELDS, "source", "eventid", "event_id", "winlog_event_id", "policyid", "policy_id", "policy", "rule", "rule_name", "sentbyte", "rcvdbyte", "bytes_in", "bytes_out", "hitcount", "elapsed"]))
|
WINDOWS_DISCOVERY_FIELDS = (
|
||||||
|
"winlog.channel",
|
||||||
|
"winlog.provider_name",
|
||||||
|
"winlog.record_id",
|
||||||
|
"winlog.process.pid",
|
||||||
|
"winlog.process.thread.id",
|
||||||
|
"winlog.task",
|
||||||
|
"winlog.opcode",
|
||||||
|
"winlog.keywords",
|
||||||
|
"winlog.event_data.logontype",
|
||||||
|
"winlog.event_data.logon_type",
|
||||||
|
"winlog.event_data.authenticationpackagename",
|
||||||
|
"winlog.event_data.logonprocessname",
|
||||||
|
"winlog.event_data.failurereason",
|
||||||
|
"winlog.event_data.status",
|
||||||
|
"winlog.event_data.substatus",
|
||||||
|
"process.name",
|
||||||
|
"process.executable",
|
||||||
|
"process.pid",
|
||||||
|
"process.command_line",
|
||||||
|
"service.name",
|
||||||
|
)
|
||||||
|
|
||||||
|
DEFAULT_FIELDS = list(dict.fromkeys([*DEFAULT_SEARCH_FIELDS, *WINDOWS_DISCOVERY_FIELDS, "source", "eventid", "event_id", "winlog_event_id", "policyid", "policy_id", "policy", "rule", "rule_name", "sentbyte", "rcvdbyte", "bytes_in", "bytes_out", "hitcount", "elapsed"]))
|
||||||
|
|
||||||
|
|
||||||
def _records(value: object) -> Iterable[dict[str, object]]:
|
def _records(value: object) -> Iterable[dict[str, object]]:
|
||||||
|
|||||||
@@ -2,21 +2,21 @@ from __future__ import annotations
|
|||||||
|
|
||||||
|
|
||||||
FIELD_ALIASES: dict[str, tuple[str, ...]] = {
|
FIELD_ALIASES: dict[str, tuple[str, ...]] = {
|
||||||
"srcip": ("srcip", "src_ip", "source_ip", "source.ip", "src", "src_addr", "srcaddr", "sourceaddress", "client_ip", "client.ip", "clientip", "client", "remote_addr", "remote_ip", "ip", "ipaddress", "gl2_remote_ip", "winlog_event_data_ipaddress", "event_data_ipaddress"),
|
"srcip": ("srcip", "src_ip", "source_ip", "source.ip", "source.address", "src", "src_addr", "srcaddr", "sourceaddress", "client_ip", "client.ip", "client.address", "clientip", "client", "remote_addr", "remote_ip", "ip", "ipaddress", "related.ip", "gl2_remote_ip", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "event_data_ipaddress"),
|
||||||
"dstip": ("dstip", "dst_ip", "destination_ip", "destination.ip", "dst", "dst_addr", "dstaddr", "destinationaddress", "dest_ip", "dest", "server_ip", "server.ip", "server", "upstream"),
|
"dstip": ("dstip", "dst_ip", "destination_ip", "destination.ip", "destination.address", "dst", "dst_addr", "dstaddr", "destinationaddress", "dest_ip", "dest", "server_ip", "server.ip", "server.address", "server", "upstream"),
|
||||||
"srcport": ("srcport", "src_port", "source_port", "source.port", "sport", "client_port", "client.port", "clientport"),
|
"srcport": ("srcport", "src_port", "source_port", "source.port", "sport", "client_port", "client.port", "clientport"),
|
||||||
"dstport": ("dstport", "dst_port", "destination_port", "destination.port", "dest_port", "dport", "server_port", "server.port", "serverport"),
|
"dstport": ("dstport", "dst_port", "destination_port", "destination.port", "dest_port", "dport", "server_port", "server.port", "serverport"),
|
||||||
"eventtime": ("eventtime", "timestamp", "time", "event_time", "eventtime_ms", "created_at"),
|
"eventtime": ("eventtime", "timestamp", "@timestamp", "time", "event_time", "eventtime_ms", "created_at", "event.created", "event.ingested", "winlog.time_created"),
|
||||||
"severity": ("severity", "level", "priority", "sev", "loglevel", "log_level", "event_severity"),
|
"severity": ("severity", "level", "priority", "sev", "loglevel", "log_level", "log.level", "event_severity", "event.severity", "winlog.level", "winlog.level_name"),
|
||||||
"action": ("action", "act", "fw_action", "rule_action", "policy_action", "event_action", "disposition", "outcome", "result", "status", "event_outcome"),
|
"action": ("action", "act", "fw_action", "rule_action", "policy_action", "event_action", "event.action", "disposition", "outcome", "result", "status", "event_outcome", "event.outcome"),
|
||||||
"type": ("type", "event_type", "log_type", "category", "event_category", "facility"),
|
"type": ("type", "event_type", "log_type", "category", "event_category", "event.category", "event.kind", "event.module", "event.dataset", "facility"),
|
||||||
"subtype": ("subtype", "sub_type", "event_subtype", "subcategory"),
|
"subtype": ("subtype", "sub_type", "event_subtype", "subcategory", "event.type"),
|
||||||
"service": ("service", "service_name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "query_type", "qt"),
|
"service": ("service", "service_name", "service.name", "dst_service", "app", "appname", "application", "application_name", "proto", "protocol", "transport", "network.transport", "network.protocol", "query_type", "qt"),
|
||||||
"username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "user", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"),
|
"username": ("username", "targetusername", "target_user_name", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog.event_data.target_user_name", "subjectusername", "subject_user_name", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername", "winlog.event_data.subject_user_name", "user", "user.name", "user.id", "user_name", "account", "account_name", "xauthuser", "actor", "actor_id", "principal", "principal_name", "login", "login_name"),
|
||||||
"hostname": ("hostname", "host", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "winlog_computer_name", "host.name", "agent.name"),
|
"hostname": ("hostname", "host", "host.name", "computer", "computer_name", "workstation", "workstation_name", "device_name", "agent_name", "agent.hostname", "agent.name", "observer.hostname", "winlog_computer_name", "winlog.computer_name"),
|
||||||
"eventid": ("eventid", "event_id", "winlog_event_id", "event_code", "event.code", "windows_event_id"),
|
"eventid": ("eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code", "windows_event_id"),
|
||||||
"dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"),
|
"dns_query": ("dns_query", "query_domain", "qh", "dns_question", "question", "queried_domain"),
|
||||||
"context": ("query_domain", "qh", "dns_query", "url", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original"),
|
"context": ("query_domain", "qh", "dns_query", "url", "url.full", "url.path", "uri", "request", "request_uri", "path", "domain", "hostname", "message", "msg", "full_message", "event_message", "event_original", "event.original", "process.command_line", "powershell.command.value"),
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases))
|
DEFAULT_SEARCH_FIELDS = tuple(dict.fromkeys(field for aliases in FIELD_ALIASES.values() for field in aliases))
|
||||||
|
|||||||
@@ -57,6 +57,99 @@ CATEGORICAL_PRIORITY = (
|
|||||||
"context",
|
"context",
|
||||||
)
|
)
|
||||||
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
|
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
|
||||||
|
WINDOWS_ENTITY_PRIORITY = (
|
||||||
|
"username",
|
||||||
|
"user.name",
|
||||||
|
"user.id",
|
||||||
|
"hostname",
|
||||||
|
"host.name",
|
||||||
|
"srcip",
|
||||||
|
"source.ip",
|
||||||
|
"client.ip",
|
||||||
|
"source",
|
||||||
|
"targetusername",
|
||||||
|
"winlog_event_data_targetusername",
|
||||||
|
"winlog.event_data.targetusername",
|
||||||
|
"winlog.event_data.target_user_name",
|
||||||
|
"subjectusername",
|
||||||
|
"winlog_event_data_subjectusername",
|
||||||
|
"winlog.event_data.subjectusername",
|
||||||
|
"winlog.event_data.subject_user_name",
|
||||||
|
"user",
|
||||||
|
"account_name",
|
||||||
|
"computer",
|
||||||
|
"computer_name",
|
||||||
|
"winlog_computer_name",
|
||||||
|
"winlog.computer_name",
|
||||||
|
"agent.hostname",
|
||||||
|
"source_ip",
|
||||||
|
"client_ip",
|
||||||
|
"winlog_event_data_ipaddress",
|
||||||
|
"winlog.event_data.ipaddress",
|
||||||
|
"winlog.event_data.ip_address",
|
||||||
|
"ipaddress",
|
||||||
|
)
|
||||||
|
WINDOWS_CATEGORICAL_PRIORITY = (
|
||||||
|
"eventid",
|
||||||
|
"event_id",
|
||||||
|
"winlog_event_id",
|
||||||
|
"winlog.event_id",
|
||||||
|
"event_code",
|
||||||
|
"event.code",
|
||||||
|
"action",
|
||||||
|
"event.action",
|
||||||
|
"event.outcome",
|
||||||
|
"severity",
|
||||||
|
"channel",
|
||||||
|
"winlog.channel",
|
||||||
|
"provider_name",
|
||||||
|
"event_provider",
|
||||||
|
"event.provider",
|
||||||
|
"winlog_provider_name",
|
||||||
|
"winlog.provider_name",
|
||||||
|
"logon_type",
|
||||||
|
"winlog.event_data.logontype",
|
||||||
|
"winlog.event_data.logon_type",
|
||||||
|
"event.category",
|
||||||
|
"event.type",
|
||||||
|
"event.kind",
|
||||||
|
"event.module",
|
||||||
|
"event.dataset",
|
||||||
|
"task",
|
||||||
|
"winlog.task",
|
||||||
|
"opcode",
|
||||||
|
"winlog.opcode",
|
||||||
|
"keywords",
|
||||||
|
"winlog.keywords",
|
||||||
|
"process_name",
|
||||||
|
"process.name",
|
||||||
|
"process.executable",
|
||||||
|
"image",
|
||||||
|
"service_name",
|
||||||
|
"service.name",
|
||||||
|
"authentication_package_name",
|
||||||
|
"winlog.event_data.authenticationpackagename",
|
||||||
|
"logon_process_name",
|
||||||
|
"winlog.event_data.logonprocessname",
|
||||||
|
"failure_reason",
|
||||||
|
"winlog.event_data.failurereason",
|
||||||
|
"sub_status",
|
||||||
|
"winlog.event_data.substatus",
|
||||||
|
"status",
|
||||||
|
"winlog.event_data.status",
|
||||||
|
)
|
||||||
|
WINDOWS_NUMERIC_PRIORITY = (
|
||||||
|
"event_record_id",
|
||||||
|
"record_id",
|
||||||
|
"winlog_record_id",
|
||||||
|
"winlog.record_id",
|
||||||
|
"process_id",
|
||||||
|
"process.pid",
|
||||||
|
"winlog_process_pid",
|
||||||
|
"winlog.process.pid",
|
||||||
|
"thread_id",
|
||||||
|
"winlog.process.thread.id",
|
||||||
|
)
|
||||||
ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"}
|
ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"}
|
||||||
IGNORED_DISCOVERY_FIELDS = {
|
IGNORED_DISCOVERY_FIELDS = {
|
||||||
"message",
|
"message",
|
||||||
@@ -112,6 +205,43 @@ def _looks_like_time_field(field: str) -> bool:
|
|||||||
return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
|
return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
|
||||||
|
|
||||||
|
|
||||||
|
def _looks_like_windows_stream(stream_id: str, stream_name: str, coverage: Counter[str]) -> bool:
|
||||||
|
name = f"{stream_id} {stream_name}".lower()
|
||||||
|
if any(token in name for token in ("windows", "winlog", "event log", "security event", "sysmon", "powershell")):
|
||||||
|
return True
|
||||||
|
windows_fields = {
|
||||||
|
"winlog_event_id",
|
||||||
|
"winlog.event_id",
|
||||||
|
"winlog_computer_name",
|
||||||
|
"winlog.computer_name",
|
||||||
|
"winlog_event_data_targetusername",
|
||||||
|
"winlog.event_data.targetusername",
|
||||||
|
"winlog_event_data_subjectusername",
|
||||||
|
"winlog.event_data.subjectusername",
|
||||||
|
"eventid",
|
||||||
|
"event_code",
|
||||||
|
"event.code",
|
||||||
|
"logon_type",
|
||||||
|
"winlog.channel",
|
||||||
|
}
|
||||||
|
return sum(1 for field in windows_fields if coverage[field]) >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def _compact_windows_fields(fields: list[str]) -> list[str]:
|
||||||
|
output = list(dict.fromkeys(fields))
|
||||||
|
alias_groups = (
|
||||||
|
("username", "user.name", "user.id", "targetusername", "winlog_event_data_targetusername", "winlog.event_data.targetusername", "winlog.event_data.target_user_name", "subjectusername", "winlog_event_data_subjectusername", "winlog.event_data.subjectusername", "winlog.event_data.subject_user_name", "user", "account_name"),
|
||||||
|
("hostname", "host.name", "computer", "computer_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname"),
|
||||||
|
("srcip", "source.ip", "source_ip", "client.ip", "client_ip", "winlog_event_data_ipaddress", "winlog.event_data.ipaddress", "winlog.event_data.ip_address", "ipaddress"),
|
||||||
|
("eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code"),
|
||||||
|
("action", "event.action", "event.outcome"),
|
||||||
|
)
|
||||||
|
for canonical, *aliases in alias_groups:
|
||||||
|
if canonical in output:
|
||||||
|
output = [field for field in output if field == canonical or field not in aliases]
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
|
def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
|
||||||
output = list(selected)
|
output = list(selected)
|
||||||
for field, count in coverage.most_common():
|
for field, count in coverage.most_common():
|
||||||
@@ -244,24 +374,59 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
|
|||||||
numeric_counts[field] += 1
|
numeric_counts[field] += 1
|
||||||
detector_counts.update(event_detector_categories(event))
|
detector_counts.update(event_detector_categories(event))
|
||||||
|
|
||||||
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3))
|
stream_name = names.get(stream_id, stream_id)
|
||||||
|
is_windows = _looks_like_windows_stream(stream_id, stream_name, coverage)
|
||||||
|
entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
|
||||||
|
categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
|
||||||
|
numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
|
||||||
|
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3))
|
||||||
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
|
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
|
||||||
if not timestamp:
|
if not timestamp:
|
||||||
timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
|
timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
|
||||||
categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8))
|
categorical = _generic_categorical_fields(coverage, unique_values, total, _pick_present(categorical_priority, coverage, total, min_ratio=0.02, limit=10 if is_windows else 8))
|
||||||
numeric = _generic_numeric_fields(coverage, numeric_counts, total, [
|
numeric_seed = [
|
||||||
field for field in NUMERIC_PRIORITY
|
field for field in numeric_priority
|
||||||
if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
|
if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
|
||||||
][:5])
|
][:5]
|
||||||
|
numeric = numeric_seed if is_windows else _generic_numeric_fields(coverage, numeric_counts, total, numeric_seed)
|
||||||
if not entity_fields:
|
if not entity_fields:
|
||||||
entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
|
entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
|
||||||
if not entity_fields:
|
if not entity_fields:
|
||||||
entity_fields = ["source"]
|
entity_fields = ["source"]
|
||||||
|
if is_windows:
|
||||||
|
entity_fields = _compact_windows_fields(entity_fields)[:4]
|
||||||
|
categorical = _compact_windows_fields(categorical)
|
||||||
|
entity_set = set(entity_fields)
|
||||||
|
categorical = [
|
||||||
|
field for field in categorical
|
||||||
|
if field not in entity_set and field not in {
|
||||||
|
"srcip", "source.ip", "source_ip", "client.ip", "client_ip",
|
||||||
|
"hostname", "host.name", "computer", "username", "user.name",
|
||||||
|
"source", "context",
|
||||||
|
}
|
||||||
|
][:10]
|
||||||
|
for field in WINDOWS_CATEGORICAL_PRIORITY:
|
||||||
|
if len(categorical) >= 10:
|
||||||
|
break
|
||||||
|
compacted = _compact_windows_fields([*categorical, field])
|
||||||
|
if field not in compacted or field in categorical:
|
||||||
|
continue
|
||||||
|
if not coverage[field] or len(unique_values[field]) <= 1:
|
||||||
|
continue
|
||||||
|
if field in entity_set or field in {"srcip", "source.ip", "client.ip", "hostname", "host.name", "username", "user.name"}:
|
||||||
|
continue
|
||||||
|
categorical = compacted
|
||||||
|
numeric = [
|
||||||
|
field for field in _compact_windows_fields(numeric)
|
||||||
|
if field not in set(categorical) and field not in {"eventid", "event_id", "winlog_event_id", "winlog.event_id", "event_code", "event.code", "logon_type"}
|
||||||
|
][:5]
|
||||||
detectors = {
|
detectors = {
|
||||||
name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
|
name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
|
||||||
for name, count in detector_counts.items()
|
for name, count in detector_counts.items()
|
||||||
if count > 0
|
if count > 0
|
||||||
}
|
}
|
||||||
|
if is_windows and coverage["eventid"]:
|
||||||
|
detectors.setdefault("auth_failure", {"enabled": True, "minimum": 5, "z_threshold": 3.0})
|
||||||
if not detectors:
|
if not detectors:
|
||||||
detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
|
detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
|
||||||
high_coverage = [
|
high_coverage = [
|
||||||
|
|||||||
@@ -88,6 +88,17 @@ class GraylogSourceTests(unittest.TestCase):
|
|||||||
self.assertIn("fw_action", client.arguments["fields"])
|
self.assertIn("fw_action", client.arguments["fields"])
|
||||||
self.assertIn("full_message", client.arguments["fields"])
|
self.assertIn("full_message", client.arguments["fields"])
|
||||||
|
|
||||||
|
def test_requests_winlogbeat_discovery_fields_before_profile_exists(self):
|
||||||
|
client = _Client()
|
||||||
|
GraylogStreamSource(client, "windows").fetch()
|
||||||
|
self.assertIn("winlog.event_id", client.arguments["fields"])
|
||||||
|
self.assertIn("event.code", client.arguments["fields"])
|
||||||
|
self.assertIn("user.name", client.arguments["fields"])
|
||||||
|
self.assertIn("host.name", client.arguments["fields"])
|
||||||
|
self.assertIn("source.ip", client.arguments["fields"])
|
||||||
|
self.assertIn("winlog.channel", client.arguments["fields"])
|
||||||
|
self.assertIn("process.name", client.arguments["fields"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -44,6 +44,22 @@ class NormalizationTests(unittest.TestCase):
|
|||||||
self.assertEqual(event.fields["hostname"], "host01")
|
self.assertEqual(event.fields["hostname"], "host01")
|
||||||
self.assertEqual(event.fields["eventid"], "4625")
|
self.assertEqual(event.fields["eventid"], "4625")
|
||||||
|
|
||||||
|
def test_winlogbeat_ecs_aliases_populate_common_fields(self):
|
||||||
|
event = parse_log_line(
|
||||||
|
"@timestamp=2026-06-30T10:00:00Z event.code=4625 event.action=logon-failed "
|
||||||
|
"event.outcome=failure log.level=warning user.name=alice host.name=win01 "
|
||||||
|
"source.ip=10.0.0.5 winlog.channel=Security winlog.provider_name=Microsoft-Windows-Security-Auditing "
|
||||||
|
"winlog.event_data.LogonType=3 process.name=lsass.exe"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(event.fields["eventtime"], "2026-06-30T10:00:00Z")
|
||||||
|
self.assertEqual(event.fields["eventid"], "4625")
|
||||||
|
self.assertEqual(event.fields["action"], "logon-failed")
|
||||||
|
self.assertEqual(event.fields["severity"], "warning")
|
||||||
|
self.assertEqual(event.fields["username"], "alice")
|
||||||
|
self.assertEqual(event.fields["hostname"], "win01")
|
||||||
|
self.assertEqual(event.fields["srcip"], "10.0.0.5")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -25,9 +25,10 @@ class ProfileSuggestionTests(unittest.TestCase):
|
|||||||
def test_suggests_windows_event_fields(self):
|
def test_suggests_windows_event_fields(self):
|
||||||
events = [
|
events = [
|
||||||
parse_log_line(
|
parse_log_line(
|
||||||
f"fgai_stream_id=windows fgai_stream=Windows winlog_event_id=4625 "
|
f"fgai_stream_id=windows fgai_stream='Illuminate:Windows Security Event Log Messages' winlog_event_id=4625 "
|
||||||
f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} "
|
f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} "
|
||||||
f"event_provider=Microsoft-Windows-Security-Auditing logon_type=3 "
|
f"event_provider=Microsoft-Windows-Security-Auditing channel=Security logon_type=3 "
|
||||||
|
f"winlog_event_data_ipaddress=10.0.0.{index % 5} "
|
||||||
f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure"
|
f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure"
|
||||||
)
|
)
|
||||||
for index in range(1, 30)
|
for index in range(1, 30)
|
||||||
@@ -37,8 +38,60 @@ class ProfileSuggestionTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertIn("username", profile["entity_fields"])
|
self.assertIn("username", profile["entity_fields"])
|
||||||
self.assertIn("hostname", profile["entity_fields"])
|
self.assertIn("hostname", profile["entity_fields"])
|
||||||
|
self.assertIn("srcip", profile["entity_fields"])
|
||||||
|
self.assertNotIn("winlog_event_data_targetusername", profile["entity_fields"])
|
||||||
self.assertIn("eventid", profile["categorical_fields"])
|
self.assertIn("eventid", profile["categorical_fields"])
|
||||||
self.assertIn("logon_type", profile["categorical_fields"])
|
self.assertIn("logon_type", profile["categorical_fields"])
|
||||||
|
self.assertIn("event_provider", profile["categorical_fields"])
|
||||||
|
self.assertNotIn("policyid", profile["categorical_fields"])
|
||||||
|
self.assertNotIn("dstport", profile["categorical_fields"])
|
||||||
|
self.assertNotIn("srcip", profile["categorical_fields"])
|
||||||
|
self.assertNotIn("eventid", profile["numeric_fields"])
|
||||||
|
self.assertIn("auth_failure", profile["detectors"])
|
||||||
|
|
||||||
|
def test_windows_named_stream_gets_auth_profile_even_before_failures(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line(
|
||||||
|
f"fgai_stream_id=winsec fgai_stream='Illuminate:Windows Security Event Log Messages' "
|
||||||
|
f"eventid=4624 username=user{index % 4} hostname=host{index % 2} channel=Security "
|
||||||
|
f"eventtime=2026-06-29T10:00:{index:02d}Z action=success"
|
||||||
|
)
|
||||||
|
for index in range(1, 30)
|
||||||
|
]
|
||||||
|
|
||||||
|
suggestion = suggest_stream_profiles(events)[0]
|
||||||
|
profile = suggestion["profile"]
|
||||||
|
|
||||||
|
self.assertEqual(suggestion["stream_name"], "Illuminate:Windows Security Event Log Messages")
|
||||||
|
self.assertIn("username", profile["entity_fields"])
|
||||||
|
self.assertIn("hostname", profile["entity_fields"])
|
||||||
|
self.assertIn("eventid", profile["categorical_fields"])
|
||||||
|
self.assertIn("auth_failure", profile["detectors"])
|
||||||
|
|
||||||
|
def test_winlogbeat_ecs_fields_build_windows_profile(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line(
|
||||||
|
f"fgai_stream_id=winbeat fgai_stream='Winlogbeat Security' "
|
||||||
|
f"@timestamp=2026-06-30T10:00:{index:02d}Z event.code=4625 event.action=logon-failed "
|
||||||
|
f"event.outcome=failure event.category=authentication event.type=start "
|
||||||
|
f"user.name=user{index % 4} host.name=win{index % 3} source.ip=10.0.0.{index % 5} "
|
||||||
|
f"winlog.channel=Security winlog.provider_name=Microsoft-Windows-Security-Auditing "
|
||||||
|
f"winlog.event_data.LogonType=3 process.name=proc{index % 3}.exe"
|
||||||
|
)
|
||||||
|
for index in range(1, 30)
|
||||||
|
]
|
||||||
|
|
||||||
|
profile = suggest_stream_profiles(events)[0]["profile"]
|
||||||
|
|
||||||
|
self.assertEqual(profile["timestamp_field"], "eventtime")
|
||||||
|
self.assertEqual(profile["entity_fields"][:3], ["username", "hostname", "srcip"])
|
||||||
|
self.assertIn("eventid", profile["categorical_fields"])
|
||||||
|
self.assertIn("action", profile["categorical_fields"])
|
||||||
|
self.assertIn("event.category", profile["categorical_fields"])
|
||||||
|
self.assertIn("winlog.channel", profile["categorical_fields"])
|
||||||
|
self.assertIn("process.name", profile["categorical_fields"])
|
||||||
|
self.assertNotIn("user.name", profile["categorical_fields"])
|
||||||
|
self.assertNotIn("source.ip", profile["categorical_fields"])
|
||||||
self.assertIn("auth_failure", profile["detectors"])
|
self.assertIn("auth_failure", profile["detectors"])
|
||||||
|
|
||||||
def test_discovers_non_priority_application_fields(self):
|
def test_discovers_non_priority_application_fields(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user