148 lines
4.1 KiB
Python
148 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import shlex
|
|
from collections import Counter
|
|
from collections.abc import Iterable
|
|
from pathlib import Path
|
|
|
|
from .models import LogEvent
|
|
from .normalization import canonical_value, normalize_fields
|
|
|
|
UTM_SUBTYPES = {
|
|
"ips",
|
|
"virus",
|
|
"av",
|
|
"anomaly",
|
|
"ddos",
|
|
"dos",
|
|
"webfilter",
|
|
"app-ctrl",
|
|
"application",
|
|
"waf",
|
|
"dns",
|
|
}
|
|
|
|
THREAT_ACTIONS = {
|
|
"blocked",
|
|
"block",
|
|
"dropped",
|
|
"drop",
|
|
"reset",
|
|
"reset-client",
|
|
"reset-server",
|
|
"detected",
|
|
"deny",
|
|
"quarantine",
|
|
}
|
|
|
|
SEVERITY_SCORE = {
|
|
"emergency": 6,
|
|
"alert": 6,
|
|
"critical": 5,
|
|
"high": 4,
|
|
"warning": 3,
|
|
"medium": 3,
|
|
"notice": 2,
|
|
"low": 1,
|
|
"information": 1,
|
|
"info": 1,
|
|
}
|
|
|
|
|
|
def parse_log_line(line: str) -> LogEvent:
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
return LogEvent(raw=line, fields={})
|
|
|
|
if stripped.startswith("{"):
|
|
try:
|
|
data = json.loads(stripped)
|
|
except json.JSONDecodeError:
|
|
data = {}
|
|
else:
|
|
return LogEvent(raw=line, fields=normalize_fields({str(k).lower(): str(v) for k, v in data.items()}))
|
|
|
|
fields: dict[str, str] = {}
|
|
try:
|
|
parts = shlex.split(stripped)
|
|
except ValueError:
|
|
parts = stripped.split()
|
|
|
|
for part in parts:
|
|
if "=" not in part:
|
|
continue
|
|
key, value = part.split("=", 1)
|
|
fields[key.strip().lower()] = value.strip().strip('"')
|
|
|
|
return LogEvent(raw=line, fields=normalize_fields(fields))
|
|
|
|
|
|
def read_events(path: str | Path) -> list[LogEvent]:
|
|
with Path(path).open("r", encoding="utf-8", errors="replace") as handle:
|
|
return [event for line in handle if (event := parse_log_line(line)).fields]
|
|
|
|
|
|
def is_utm_event(event: LogEvent) -> bool:
|
|
subtype = event.subtype
|
|
event_type = event.fields.get("type", "").lower()
|
|
return event_type == "utm" or subtype in UTM_SUBTYPES
|
|
|
|
|
|
def event_score(event: LogEvent) -> int:
|
|
score = SEVERITY_SCORE.get(event.severity, 0)
|
|
if is_utm_event(event):
|
|
score += 2
|
|
if event.action in THREAT_ACTIONS:
|
|
score += 2
|
|
if event.fields.get("attack") or event.fields.get("attackid") or event.fields.get("signature"):
|
|
score += 1
|
|
return score
|
|
|
|
|
|
def summarize_events(events: Iterable[LogEvent]) -> dict[str, int]:
|
|
summary = {
|
|
"total": 0,
|
|
"utm": 0,
|
|
"threat_actions": 0,
|
|
"with_src_ip": 0,
|
|
"critical_or_high": 0,
|
|
}
|
|
for event in events:
|
|
summary["total"] += 1
|
|
if is_utm_event(event):
|
|
summary["utm"] += 1
|
|
if event.action in THREAT_ACTIONS:
|
|
summary["threat_actions"] += 1
|
|
if event.src_ip:
|
|
summary["with_src_ip"] += 1
|
|
if event.severity in {"critical", "high", "alert", "emergency"}:
|
|
summary["critical_or_high"] += 1
|
|
return summary
|
|
|
|
|
|
def top_field_values(events: Iterable[LogEvent], field: str, *, limit: int = 10) -> list[dict[str, int | str]]:
|
|
counter: Counter[str] = Counter()
|
|
for event in events:
|
|
value = event.fields.get(field)
|
|
if value:
|
|
counter[value] += 1
|
|
return [{"value": value, "count": count} for value, count in counter.most_common(limit)]
|
|
|
|
|
|
def local_in_failures(events: Iterable[LogEvent], *, limit: int = 10) -> list[dict[str, int | str]]:
|
|
counter: Counter[tuple[str, str, str]] = Counter()
|
|
for event in events:
|
|
policy_type = event.fields.get("policytype", event.fields.get("type", "")).lower()
|
|
msg = event.fields.get("msg", "").lower()
|
|
if not (policy_type.startswith("local-in") or "local-in" in policy_type or msg == "connection failed"):
|
|
continue
|
|
src_ip = event.src_ip or "unknown"
|
|
service = canonical_value(event.fields, "service") or "unknown"
|
|
policy_id = event.fields.get("policyid", event.fields.get("poluuid", "unknown"))
|
|
counter[(src_ip, service, policy_id)] += 1
|
|
return [
|
|
{"src_ip": src_ip, "service": service, "policy": policy_id, "count": count}
|
|
for (src_ip, service, policy_id), count in counter.most_common(limit)
|
|
]
|