Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add more fields

Browse Source
This commit is contained in:
larssand
2026-06-21 14:59:47 +02:00
parent 5e257b7d47
commit 4594d080c3
4 changed files with 100 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
src/fgai/anomaly.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import ipaddress
from collections import Counter, defaultdict
from datetime import datetime
from statistics import mean, pstdev
from .logs import THREAT_ACTIONS, event_score, is_utm_event
@@ -42,6 +43,37 @@ def _is_public_ip(value: str) -> bool:
return False
def _event_timestamp(event: LogEvent) -> float | None:
"""Return the FortiGate event time in seconds when present."""
eventtime = event.fields.get("eventtime")
if eventtime:
try:
value = float(eventtime)
# Exports can use seconds, milliseconds, microseconds, or nanoseconds.
while value > 10_000_000_000:
value /= 1_000
return value
except ValueError:
pass
date = event.fields.get("date")
clock = event.fields.get("time")
if date and clock:
try:
return datetime.fromisoformat(f"{date}T{clock}").timestamp()
except ValueError:
return None
return None
def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]:
timestamps = [timestamp for event in events if (timestamp := _event_timestamp(event)) is not None]
if len(timestamps) < 2:
return None, 0.0
duration_seconds = max(timestamps) - min(timestamps)
return len(timestamps) * 60 / max(1.0, duration_seconds), duration_seconds
def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[AnomalyFinding]:
by_src: dict[str, list[LogEvent]] = defaultdict(list)
for event in events:
@@ -60,19 +92,30 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events)
for src_events in by_src.values()
]
hitcount_totals = [sum(_as_int(event.fields.get("hitcount")) for event in src_events) for src_events in by_src.values()]
source_rates = [rate for src_events in by_src.values() if (rate := _rate_per_minute(src_events)[0]) is not None]
avg_events = mean(event_counts)
std_events = pstdev(event_counts) or 1.0
avg_dst = mean(distinct_dst_counts)
std_dst = pstdev(distinct_dst_counts) or 1.0
avg_bytes = mean(byte_totals)
std_bytes = pstdev(byte_totals) or 1.0
avg_hitcount = mean(hitcount_totals)
std_hitcount = pstdev(hitcount_totals) or 1.0
avg_rate = mean(source_rates) if source_rates else 0.0
std_rate = (pstdev(source_rates) or 1.0) if source_rates else 1.0
findings: list[AnomalyFinding] = []
for src_ip, src_events in by_src.items():
event_count = len(src_events)
distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")})
distinct_src_ports = len({event.fields.get("srcport") for event in src_events if event.fields.get("srcport")})
distinct_dst_ports = len({event.fields.get("dstport") for event in src_events if event.fields.get("dstport")})
total_bytes = sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events)
total_hitcount = sum(_as_int(event.fields.get("hitcount")) for event in src_events)
max_hitcount = max((_as_int(event.fields.get("hitcount")) for event in src_events), default=0)
event_rate, observed_duration = _rate_per_minute(src_events)
deny_count = sum(1 for event in src_events if event.action in THREAT_ACTIONS)
utm_count = sum(1 for event in src_events if is_utm_event(event))
high_severity_count = sum(1 for event in src_events if event.severity in {"critical", "high", "alert", "emergency"})
@@ -88,6 +131,13 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
score += points
reasons.append(f"unusually high event volume for source ({event_count} events, z={event_z:.1f})")
if event_rate is not None:
rate_z = (event_rate - avg_rate) / std_rate
if event_rate >= 20 and rate_z >= 2:
points = min(25, 10 + int(rate_z * 5))
score += points
reasons.append(f"unusually high log rate ({event_rate:.1f} events/min, z={rate_z:.1f})")
dst_z = (distinct_dst - avg_dst) / std_dst
if distinct_dst >= 10 and dst_z >= 2:
points = min(25, 10 + int(dst_z * 5))
@@ -100,6 +150,12 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
score += points
reasons.append(f"unusually high byte volume ({total_bytes} bytes, z={byte_z:.1f})")
hitcount_z = (total_hitcount - avg_hitcount) / std_hitcount
if total_hitcount >= 1_000 and hitcount_z >= 2:
points = min(15, 5 + int(hitcount_z * 3))
score += points
reasons.append(f"unusually high policy hitcount ({total_hitcount}, max event value {max_hitcount})")
if event_count >= 5:
deny_rate = deny_count / event_count
if deny_count >= 10 and deny_rate >= 0.5:
@@ -124,6 +180,10 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
score += min(15, distinct_services)
reasons.append(f"many distinct services used ({distinct_services})")
if distinct_dst_ports >= 10 and event_count >= 10:
score += min(15, distinct_dst_ports)
reasons.append(f"many destination ports contacted ({distinct_dst_ports})")
if _is_public_ip(src_ip) and (utm_count or deny_count >= 10):
score += 10
reasons.append("public source with repeated security-relevant events")
@@ -147,6 +207,13 @@ def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[
"utm_events": utm_count,
"high_severity_events": high_severity_count,
"total_bytes": total_bytes,
"events_per_minute": round(event_rate, 2) if event_rate is not None else 0.0,
"observed_duration_seconds": round(observed_duration, 2),
"timed_events": sum(1 for event in src_events if _event_timestamp(event) is not None),
"hitcount_total": total_hitcount,
"hitcount_max": max_hitcount,
"distinct_src_ports": distinct_src_ports,
"distinct_dst_ports": distinct_dst_ports,
"policy_count": len(policies),
"implicit_deny_events": implicit_deny_count,
},

7
src/fgai/dashboard.py
View File

@@ -93,6 +93,11 @@ async function refresh() {
{label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Confidence', key:'confidence'},
{label:'Rate / ports / hits', render:r => {
const e = r.evidence || {};
const rate = e.timed_events > 1 ? `${e.events_per_minute} events/min` : 'no timestamps';
return esc(`${rate}; dst ports: ${e.distinct_dst_ports || 0}; src ports: ${e.distinct_src_ports || 0}; hitcount: ${e.hitcount_total || 0}`);
}},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [
@@ -127,6 +132,8 @@ async function refresh() {
const d = data.diagnostics || {};
document.getElementById('diagnostics').innerHTML =
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Destination Ports</h3>' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Source Ports</h3>' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Services</h3>' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Local-in Failures</h3>' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip'}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]);
}

2
src/fgai/monitor.py
View File

@@ -56,6 +56,8 @@ def build_status(
"anomaly_summary": anomaly_summary(anomalies),
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),
"top_destination_ports": top_field_values(events, "dstport", limit=10),
"top_source_ports": top_field_values(events, "srcport", limit=10),
"top_services": top_field_values(events, "service", limit=10),
"top_actions": top_field_values(events, "action", limit=10),
"top_subtypes": top_field_values(events, "subtype", limit=10),