Entity labels now resolve like this for IP-based entities:
This commit is contained in:
@@ -2,39 +2,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from .entities import entity_display
|
||||||
from .entities import event_entities, sample_timeline
|
from .entities import event_entities, sample_timeline
|
||||||
from .entities import ENTITY_FIELDS
|
|
||||||
from .logs import THREAT_ACTIONS, is_utm_event
|
from .logs import THREAT_ACTIONS, is_utm_event
|
||||||
from .models import LogEvent
|
from .models import LogEvent
|
||||||
|
|
||||||
|
|
||||||
def _best_related_value(events: list[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str:
|
|
||||||
counts: dict[str, int] = defaultdict(int)
|
|
||||||
excluded = exclude.lower()
|
|
||||||
for event in events:
|
|
||||||
for field in fields:
|
|
||||||
value = str(event.fields.get(field, "")).strip()
|
|
||||||
if not value or value in {"-", "unknown", "n/a"}:
|
|
||||||
continue
|
|
||||||
if excluded and value.lower() == excluded:
|
|
||||||
continue
|
|
||||||
counts[value] += 1
|
|
||||||
if not counts:
|
|
||||||
return ""
|
|
||||||
return sorted(counts.items(), key=lambda item: (item[1], len(item[0]) <= 64), reverse=True)[0][0]
|
|
||||||
|
|
||||||
|
|
||||||
def _entity_display(kind: str, entity: str, events: list[LogEvent]) -> dict[str, str]:
|
|
||||||
if kind == "ip":
|
|
||||||
hostname = _best_related_value(events, ENTITY_FIELDS["host"], exclude=entity)
|
|
||||||
username = _best_related_value(events, ENTITY_FIELDS["user"], exclude=entity)
|
|
||||||
if hostname:
|
|
||||||
return {"entity_display": hostname, "entity_label": f"{hostname} ({entity})", "entity_detail": entity}
|
|
||||||
if username:
|
|
||||||
return {"entity_display": username, "entity_label": f"{username} ({entity})", "entity_detail": entity}
|
|
||||||
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
|
|
||||||
|
|
||||||
|
|
||||||
def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
|
def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
|
||||||
"""Correlate IPs, users, and hosts across independently configured Graylog streams."""
|
"""Correlate IPs, users, and hosts across independently configured Graylog streams."""
|
||||||
grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
|
grouped: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
|
||||||
@@ -51,7 +24,7 @@ def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dic
|
|||||||
correlations.append({
|
correlations.append({
|
||||||
"entity": entity,
|
"entity": entity,
|
||||||
"entity_type": kind,
|
"entity_type": kind,
|
||||||
**_entity_display(kind, entity, source_events),
|
**entity_display(kind, entity, source_events),
|
||||||
"source_ip": entity if kind == "ip" else "",
|
"source_ip": entity if kind == "ip" else "",
|
||||||
"streams": streams,
|
"streams": streams,
|
||||||
"events": len(source_events),
|
"events": len(source_events),
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import ipaddress
|
import ipaddress
|
||||||
|
from collections import Counter
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from .models import LogEvent
|
from .models import LogEvent
|
||||||
@@ -14,6 +15,26 @@ ENTITY_FIELDS: dict[str, tuple[str, ...]] = {
|
|||||||
"host": ("hostname", "host", "host.name", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname", "agent_name"),
|
"host": ("hostname", "host", "host.name", "computer", "computer_name", "workstation", "device_name", "winlog_computer_name", "winlog.computer_name", "agent.hostname", "agent_name"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DISPLAY_FIELD_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||||
|
("host", ENTITY_FIELDS["host"]),
|
||||||
|
("user", ENTITY_FIELDS["user"]),
|
||||||
|
(
|
||||||
|
"service",
|
||||||
|
(
|
||||||
|
"service.name",
|
||||||
|
"service_name",
|
||||||
|
"process.name",
|
||||||
|
"process.executable",
|
||||||
|
"winlog.provider_name",
|
||||||
|
"event.provider",
|
||||||
|
"application",
|
||||||
|
"application_name",
|
||||||
|
"appname",
|
||||||
|
"app",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def entity_type(value: str) -> str:
|
def entity_type(value: str) -> str:
|
||||||
try:
|
try:
|
||||||
@@ -23,6 +44,34 @@ def entity_type(value: str) -> str:
|
|||||||
return "entity"
|
return "entity"
|
||||||
|
|
||||||
|
|
||||||
|
def _related_value(events: Iterable[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str:
|
||||||
|
counts: Counter[str] = Counter()
|
||||||
|
excluded = exclude.lower()
|
||||||
|
for event in events:
|
||||||
|
for field in fields:
|
||||||
|
value = str(event.fields.get(field, "")).strip()
|
||||||
|
if not value or value.lower() in {"-", "unknown", "n/a", "none", "null"}:
|
||||||
|
continue
|
||||||
|
if excluded and value.lower() == excluded:
|
||||||
|
continue
|
||||||
|
counts[value] += 1
|
||||||
|
if not counts:
|
||||||
|
return ""
|
||||||
|
return sorted(counts.items(), key=lambda item: (item[1], len(item[0]) <= 64), reverse=True)[0][0]
|
||||||
|
|
||||||
|
|
||||||
|
def entity_display(kind: str, entity: str, events: Iterable[LogEvent]) -> dict[str, str]:
|
||||||
|
"""Return a human label for an entity without changing its stable correlation key."""
|
||||||
|
if kind != "ip":
|
||||||
|
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
|
||||||
|
event_list = list(events)
|
||||||
|
for _group, fields in DISPLAY_FIELD_GROUPS:
|
||||||
|
value = _related_value(event_list, fields, exclude=entity)
|
||||||
|
if value:
|
||||||
|
return {"entity_display": value, "entity_label": f"{value} ({entity})", "entity_detail": entity}
|
||||||
|
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
|
||||||
|
|
||||||
|
|
||||||
def event_entities(event: LogEvent) -> list[dict[str, str]]:
|
def event_entities(event: LogEvent) -> list[dict[str, str]]:
|
||||||
"""Return normalized identities shared across network, endpoint, DNS, and web logs."""
|
"""Return normalized identities shared across network, endpoint, DNS, and web logs."""
|
||||||
identities: list[dict[str, str]] = []
|
identities: list[dict[str, str]] = []
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
import ipaddress
|
import ipaddress
|
||||||
from collections import Counter, defaultdict
|
from collections import Counter, defaultdict
|
||||||
|
|
||||||
from .entities import ENTITY_FIELDS
|
from .entities import entity_display
|
||||||
from .logs import THREAT_ACTIONS, is_utm_event
|
from .logs import THREAT_ACTIONS, is_utm_event
|
||||||
from .models import LogEvent
|
from .models import LogEvent
|
||||||
from .normalization import canonical_value
|
from .normalization import canonical_value
|
||||||
@@ -13,30 +13,12 @@ def _entity(event: LogEvent) -> str:
|
|||||||
return event.src_ip or event.fields.get("user") or event.fields.get("username") or event.fields.get("hostname") or event.fields.get("source") or "unknown"
|
return event.src_ip or event.fields.get("user") or event.fields.get("username") or event.fields.get("hostname") or event.fields.get("source") or "unknown"
|
||||||
|
|
||||||
|
|
||||||
def _related_value(events: list[LogEvent], fields: tuple[str, ...], *, exclude: str = "") -> str:
|
|
||||||
counts: Counter[str] = Counter()
|
|
||||||
excluded = exclude.lower()
|
|
||||||
for event in events:
|
|
||||||
for field in fields:
|
|
||||||
value = str(event.fields.get(field, "")).strip()
|
|
||||||
if not value or value in {"-", "unknown", "n/a"}:
|
|
||||||
continue
|
|
||||||
if excluded and value.lower() == excluded:
|
|
||||||
continue
|
|
||||||
counts[value] += 1
|
|
||||||
return counts.most_common(1)[0][0] if counts else ""
|
|
||||||
|
|
||||||
|
|
||||||
def _display_label(entity: str, events: list[LogEvent]) -> dict[str, str]:
|
def _display_label(entity: str, events: list[LogEvent]) -> dict[str, str]:
|
||||||
try:
|
try:
|
||||||
ipaddress.ip_address(entity)
|
ipaddress.ip_address(entity)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
|
return {"entity_display": entity, "entity_label": entity, "entity_detail": ""}
|
||||||
hostname = _related_value(events, ENTITY_FIELDS["host"], exclude=entity)
|
return entity_display("ip", entity, events)
|
||||||
username = _related_value(events, ENTITY_FIELDS["user"], exclude=entity)
|
|
||||||
display = hostname or username or entity
|
|
||||||
label = f"{display} ({entity})" if display != entity else entity
|
|
||||||
return {"entity_display": display, "entity_label": label, "entity_detail": entity if display != entity else ""}
|
|
||||||
|
|
||||||
|
|
||||||
def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sample_limit: int = 15) -> dict[str, object]:
|
def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sample_limit: int = 15) -> dict[str, object]:
|
||||||
|
|||||||
@@ -34,3 +34,21 @@ class CorrelationTests(unittest.TestCase):
|
|||||||
self.assertEqual(result[0]["entity_display"], "win01")
|
self.assertEqual(result[0]["entity_display"], "win01")
|
||||||
self.assertEqual(result[0]["entity_label"], "win01 (10.0.0.5)")
|
self.assertEqual(result[0]["entity_label"], "win01 (10.0.0.5)")
|
||||||
self.assertEqual(result[0]["entity_detail"], "10.0.0.5")
|
self.assertEqual(result[0]["entity_detail"], "10.0.0.5")
|
||||||
|
|
||||||
|
def test_ip_entity_falls_back_to_username_display_label(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line("srcip=10.0.0.6 username=alice fgai_stream=Windows action=login"),
|
||||||
|
parse_log_line("srcip=10.0.0.6 username=alice fgai_stream=VPN action=accept"),
|
||||||
|
]
|
||||||
|
result = correlate_source_ips(events)
|
||||||
|
self.assertEqual(result[0]["entity"], "10.0.0.6")
|
||||||
|
self.assertEqual(result[0]["entity_label"], "alice (10.0.0.6)")
|
||||||
|
|
||||||
|
def test_ip_entity_falls_back_to_service_display_label(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line("srcip=10.0.0.7 process.name=spoolsv.exe fgai_stream=Windows action=started"),
|
||||||
|
parse_log_line("srcip=10.0.0.7 process.name=spoolsv.exe fgai_stream=Sysmon action=connect"),
|
||||||
|
]
|
||||||
|
result = correlate_source_ips(events)
|
||||||
|
self.assertEqual(result[0]["entity"], "10.0.0.7")
|
||||||
|
self.assertEqual(result[0]["entity_label"], "spoolsv.exe (10.0.0.7)")
|
||||||
|
|||||||
@@ -24,6 +24,13 @@ class EventContextTests(unittest.TestCase):
|
|||||||
self.assertEqual(context["source_profiles"][0]["entity_label"], "win01 (10.0.0.2)")
|
self.assertEqual(context["source_profiles"][0]["entity_label"], "win01 (10.0.0.2)")
|
||||||
self.assertEqual(context["security_event_samples"][0]["entity_label"], "win01 (10.0.0.2)")
|
self.assertEqual(context["security_event_samples"][0]["entity_label"], "win01 (10.0.0.2)")
|
||||||
|
|
||||||
|
def test_ip_entity_falls_back_to_username_display_label(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line("srcip=10.0.0.3 username=alice type=utm action=blocked severity=high"),
|
||||||
|
]
|
||||||
|
context = build_event_context(events)
|
||||||
|
self.assertEqual(context["source_profiles"][0]["entity_label"], "alice (10.0.0.3)")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user