Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ui

Browse Source
This commit is contained in:
larssand
2026-06-18 22:38:47 +02:00
parent b4ad932c91
commit 5650973e50
9 changed files with 700 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
src/fgai/anomaly.py Normal file
View File

@@ -0,0 +1,161 @@
from __future__ import annotations
import ipaddress
from collections import Counter, defaultdict
from statistics import mean, pstdev
from .logs import THREAT_ACTIONS, event_score, is_utm_event
from .models import AnomalyFinding, LogEvent
def _as_int(value: str | None) -> int:
if not value:
return 0
try:
return int(float(value))
except ValueError:
return 0
def _severity(score: int) -> str:
if score >= 80:
return "critical"
if score >= 60:
return "high"
if score >= 35:
return "medium"
return "low"
def _confidence(event_count: int, reason_count: int) -> str:
if event_count >= 20 and reason_count >= 3:
return "high"
if event_count >= 5 and reason_count >= 2:
return "medium"
return "low"
def _is_public_ip(value: str) -> bool:
try:
return ipaddress.ip_address(value).is_global
except ValueError:
return False
def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[AnomalyFinding]:
by_src: dict[str, list[LogEvent]] = defaultdict(list)
for event in events:
if event.src_ip:
by_src[event.src_ip].append(event)
if not by_src:
return []
event_counts = [len(src_events) for src_events in by_src.values()]
distinct_dst_counts = [
len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
for src_events in by_src.values()
]
byte_totals = [
sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events)
for src_events in by_src.values()
]
avg_events = mean(event_counts)
std_events = pstdev(event_counts) or 1.0
avg_dst = mean(distinct_dst_counts)
std_dst = pstdev(distinct_dst_counts) or 1.0
avg_bytes = mean(byte_totals)
std_bytes = pstdev(byte_totals) or 1.0
findings: list[AnomalyFinding] = []
for src_ip, src_events in by_src.items():
event_count = len(src_events)
distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")})
total_bytes = sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events)
deny_count = sum(1 for event in src_events if event.action in THREAT_ACTIONS)
utm_count = sum(1 for event in src_events if is_utm_event(event))
high_severity_count = sum(1 for event in src_events if event.severity in {"critical", "high", "alert", "emergency"})
policies = {event.fields.get("policyid") for event in src_events if event.fields.get("policyid")}
reasons: list[str] = []
score = 0
event_z = (event_count - avg_events) / std_events
if event_count >= 25 and event_z >= 2:
points = min(25, 10 + int(event_z * 5))
score += points
reasons.append(f"unusually high event volume for source ({event_count} events, z={event_z:.1f})")
dst_z = (distinct_dst - avg_dst) / std_dst
if distinct_dst >= 10 and dst_z >= 2:
points = min(25, 10 + int(dst_z * 5))
score += points
reasons.append(f"source contacted unusually many destinations ({distinct_dst}, z={dst_z:.1f})")
byte_z = (total_bytes - avg_bytes) / std_bytes
if total_bytes >= 50_000_000 and byte_z >= 2:
points = min(20, 8 + int(byte_z * 4))
score += points
reasons.append(f"unusually high byte volume ({total_bytes} bytes, z={byte_z:.1f})")
if event_count >= 5:
deny_rate = deny_count / event_count
if deny_count >= 10 and deny_rate >= 0.5:
score += min(20, 8 + int(deny_rate * 12))
reasons.append(f"high deny/threat-action rate ({deny_count}/{event_count})")
if utm_count:
utm_score = sum(event_score(event) for event in src_events if is_utm_event(event))
points = min(35, 5 + utm_score)
score += points
reasons.append(f"UTM/security detections observed ({utm_count} events)")
if high_severity_count:
score += min(20, high_severity_count * 8)
reasons.append(f"high or critical severity events observed ({high_severity_count})")
if distinct_services >= 8 and event_count >= 10:
score += min(15, distinct_services)
reasons.append(f"many distinct services used ({distinct_services})")
if _is_public_ip(src_ip) and (utm_count or deny_count >= 10):
score += 10
reasons.append("public source with repeated security-relevant events")
if not reasons:
continue
score = min(score, 100)
findings.append(
AnomalyFinding(
subject=src_ip,
score=score,
severity=_severity(score),
confidence=_confidence(event_count, len(reasons)),
reasons=reasons,
evidence={
"events": event_count,
"distinct_destinations": distinct_dst,
"distinct_services": distinct_services,
"deny_or_threat_actions": deny_count,
"utm_events": utm_count,
"high_severity_events": high_severity_count,
"total_bytes": total_bytes,
"policy_count": len(policies),
},
)
)
return sorted(findings, key=lambda finding: finding.score, reverse=True)[:limit]
def anomaly_summary(findings: list[AnomalyFinding]) -> dict[str, int]:
counts = Counter(finding.severity for finding in findings)
return {
"total": len(findings),
"critical": counts["critical"],
"high": counts["high"],
"medium": counts["medium"],
"low": counts["low"],
}

82
src/fgai/cli.py
View File

@@ -5,11 +5,14 @@ import json
import sys
from pathlib import Path
from .anomaly import anomaly_summary, detect_source_anomalies
from .dashboard import serve_dashboard
from .llm import ollama_summary
from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import FortiGateClient, parse_allowlist, suggest_block_candidates
from .policies import audit_policies, read_policies
from .syslog_server import listen_udp_syslog
from .monitor import monitor_loop
def _print_json(data: object) -> None:
@@ -24,8 +27,10 @@ def analyze_logs(args: argparse.Namespace) -> int:
min_score=args.min_score,
allowlist=parse_allowlist(args.allowlist),
)
anomalies = detect_source_anomalies(events, limit=args.anomaly_limit)
analysis = {
"summary": summarize_events(events),
"anomaly_summary": anomaly_summary(anomalies),
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),
"top_services": top_field_values(events, "service", limit=10),
@@ -37,6 +42,17 @@ def analyze_logs(args: argparse.Namespace) -> int:
{"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons}
for candidate in candidates
],
"anomalies": [
{
"subject": finding.subject,
"score": finding.score,
"severity": finding.severity,
"confidence": finding.confidence,
"reasons": finding.reasons,
"evidence": finding.evidence,
}
for finding in anomalies
],
}
_print_json(analysis)
if args.llm:
@@ -78,6 +94,31 @@ def suggest_blocks(args: argparse.Namespace) -> int:
return 0
def detect_anomalies(args: argparse.Namespace) -> int:
events = read_events(args.logs)
anomalies = detect_source_anomalies(events, limit=args.limit)
output = {
"summary": anomaly_summary(anomalies),
"anomalies": [
{
"subject": finding.subject,
"score": finding.score,
"severity": finding.severity,
"confidence": finding.confidence,
"reasons": finding.reasons,
"evidence": finding.evidence,
}
for finding in anomalies
if finding.score >= args.min_score
],
}
_print_json(output)
if args.llm:
print("\nLLM summary:")
print(ollama_summary([], [], args.model, analysis=output, timeout=args.llm_timeout))
return 0
def test_connection(args: argparse.Namespace) -> int:
client = FortiGateClient.from_env()
status = client.system_status()
@@ -105,6 +146,22 @@ def listen_syslog(args: argparse.Namespace) -> int:
return 0
def run_monitor(args: argparse.Namespace) -> int:
monitor_loop(
args.logs,
args.output,
policy_path=args.policies,
interval=args.interval,
anomaly_limit=args.anomaly_limit,
)
return 0
def run_dashboard(args: argparse.Namespace) -> int:
serve_dashboard(args.host, args.port, args.status_file, image_dir=args.image_dir)
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Local FortiGate AI/ML inspection tool")
subparsers = parser.add_subparsers(required=True)
@@ -117,8 +174,18 @@ def build_parser() -> argparse.ArgumentParser:
logs.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results")
logs.add_argument("--model", default=None, help="Ollama model name")
logs.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds")
logs.add_argument("--anomaly-limit", type=int, default=20, help="Maximum anomaly findings to include")
logs.set_defaults(func=analyze_logs)
anomalies = subparsers.add_parser("detect-anomalies", help="Score likely traffic anomalies by source IP")
anomalies.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file")
anomalies.add_argument("--limit", type=int, default=20, help="Maximum anomaly findings to include")
anomalies.add_argument("--min-score", type=int, default=1, help="Minimum anomaly score to output")
anomalies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results")
anomalies.add_argument("--model", default=None, help="Ollama model name")
anomalies.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds")
anomalies.set_defaults(func=detect_anomalies)
policies = subparsers.add_parser("audit-policies", help="Audit FortiOS firewall policy config")
policies.add_argument("--config", required=True, help="Path to FortiOS config backup")
policies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results")
@@ -149,6 +216,21 @@ def build_parser() -> argparse.ArgumentParser:
listener.add_argument("--quiet", action="store_true", help="Do not print each received syslog message")
listener.set_defaults(func=listen_syslog)
monitor = subparsers.add_parser("monitor", help="Continuously analyze logs and write dashboard status JSON")
monitor.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file")
monitor.add_argument("--output", default="state/fgai-status.json", help="Status JSON written for dashboard")
monitor.add_argument("--policies", default=None, help="Optional FortiGate policy JSON/config file to audit continuously")
monitor.add_argument("--interval", type=int, default=10, help="Seconds between analysis runs")
monitor.add_argument("--anomaly-limit", type=int, default=20, help="Maximum anomaly findings to include")
monitor.set_defaults(func=run_monitor)
dashboard = subparsers.add_parser("dashboard", help="Serve local fgAI dashboard")
dashboard.add_argument("--host", default="127.0.0.1", help="Dashboard bind address")
dashboard.add_argument("--port", type=int, default=8088, help="Dashboard TCP port")
dashboard.add_argument("--status-file", default="state/fgai-status.json", help="Status JSON produced by monitor")
dashboard.add_argument("--image-dir", default="images", help="Directory containing dashboard images")
dashboard.set_defaults(func=run_dashboard)
return parser

154
src/fgai/dashboard.py Normal file
View File

@@ -0,0 +1,154 @@
from __future__ import annotations
import json
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
HTML = """<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>fgAI Monitor</title>
<style>
:root { color-scheme: light; font-family: Arial, sans-serif; background: #f5f7f9; color: #16202a; }
body { margin: 0; }
header { background: #102032; color: white; padding: 18px 24px; }
h1 { margin: 0; font-size: 22px; }
main { padding: 18px; max-width: 1320px; margin: 0 auto; }
.hero { display: grid; grid-template-columns: minmax(280px, 0.9fr) minmax(360px, 1.1fr); gap: 14px; align-items: stretch; }
.hero img { width: 100%; height: 100%; max-height: 360px; object-fit: cover; border-radius: 6px; border: 1px solid #1f3b57; background: #061322; }
.hero .panel { margin-bottom: 0; }
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 12px; }
.panel { background: white; border: 1px solid #d9e0e7; border-radius: 6px; padding: 14px; margin-bottom: 14px; }
.metric { font-size: 28px; font-weight: 700; }
.label { color: #536170; font-size: 13px; margin-top: 4px; }
table { width: 100%; border-collapse: collapse; font-size: 14px; }
th, td { border-bottom: 1px solid #e4e9ee; padding: 8px; text-align: left; vertical-align: top; }
th { color: #536170; font-weight: 600; }
.sev-critical { color: #b00020; font-weight: 700; }
.sev-high { color: #b54708; font-weight: 700; }
.sev-medium { color: #8a6d00; font-weight: 700; }
.sev-low { color: #345995; font-weight: 700; }
.muted { color: #697789; }
code { background: #eef2f6; padding: 2px 4px; border-radius: 4px; }
@media (max-width: 860px) { .hero { grid-template-columns: 1fr; } .hero img { max-height: 240px; } }
</style>
</head>
<body>
<header><h1>fgAI Monitor</h1><div id="stamp" class="muted"></div></header>
<main>
<section class="hero">
<img src="/images/FGinspectionagent.png" alt="FortiGate AI/ML Analyzer">
<div>
<section class="grid" id="metrics"></section>
<section class="panel"><h2>Live Status</h2><div id="liveStatus" class="muted">Waiting for monitor data.</div></section>
</div>
</section>
<section class="panel"><h2>Anomalies</h2><div id="anomalies"></div></section>
<section class="panel"><h2>Block Candidates</h2><div id="blocks"></div></section>
<section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section>
<section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section>
</main>
<script>
function esc(value) {
return String(value ?? "").replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
}
function metric(label, value) {
return `<div class="panel"><div class="metric">${esc(value)}</div><div class="label">${esc(label)}</div></div>`;
}
function table(rows, columns) {
if (!rows || rows.length === 0) return '<p class="muted">No data.</p>';
const head = columns.map(c => `<th>${esc(c.label)}</th>`).join('');
const body = rows.map(row => `<tr>${columns.map(c => `<td>${c.render ? c.render(row) : esc(row[c.key])}</td>`).join('')}</tr>`).join('');
return `<table><thead><tr>${head}</tr></thead><tbody>${body}</tbody></table>`;
}
async function refresh() {
const res = await fetch('/api/status', {cache: 'no-store'});
const data = await res.json();
const s = data.summary || {};
const a = data.anomaly_summary || {};
document.getElementById('stamp').textContent = data.generated_at ? `Updated ${new Date(data.generated_at * 1000).toLocaleString()}` : 'Waiting for monitor data';
document.getElementById('metrics').innerHTML = [
metric('Total events', s.total || 0),
metric('UTM events', s.utm || 0),
metric('Threat actions', s.threat_actions || 0),
metric('Anomalies high+', (a.high || 0) + (a.critical || 0))
].join('');
document.getElementById('liveStatus').innerHTML = [
`Log file: <code>${esc(data.log_path || '')}</code>`,
`Policy file: <code>${esc(data.policy_path || 'none')}</code>`,
`Critical anomalies: ${esc((a.critical || 0))}`,
`High anomalies: ${esc((a.high || 0))}`
].join('<br>');
document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [
{label:'Source', key:'subject'},
{label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Confidence', key:'confidence'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
{label:'Source', key:'src_ip'},
{label:'Score', key:'score'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
document.getElementById('policies').innerHTML = table(data.policy_findings || [], [
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Reference', key:'reference'},
{label:'Title', key:'title'},
{label:'Detail', key:'detail'}
]);
const d = data.diagnostics || {};
document.getElementById('diagnostics').innerHTML =
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Services</h3>' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Local-in Failures</h3>' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip'}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]);
}
refresh();
setInterval(refresh, 5000);
</script>
</body>
</html>
"""
def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | None = None) -> None:
status_path = Path(status_file)
image_root = Path(image_dir) if image_dir else None
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
if self.path == "/":
self._send(200, "text/html; charset=utf-8", HTML.encode("utf-8"))
return
if self.path == "/api/status":
if status_path.exists():
body = status_path.read_bytes()
else:
body = json.dumps({"summary": {}, "anomalies": [], "block_candidates": []}).encode("utf-8")
self._send(200, "application/json", body)
return
if self.path.startswith("/images/") and image_root:
image_path = image_root / Path(self.path).name
if image_path.exists() and image_path.is_file():
content_type = "image/png" if image_path.suffix.lower() == ".png" else "application/octet-stream"
self._send(200, content_type, image_path.read_bytes())
return
self._send(404, "text/plain; charset=utf-8", b"not found")
def log_message(self, format: str, *args: object) -> None:
return
def _send(self, status: int, content_type: str, body: bytes) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
server = ThreadingHTTPServer((host, port), Handler)
print(f"Dashboard listening on http://{host}:{port}")
print(f"Reading status from {status_path}")
server.serve_forever()

10
src/fgai/models.py
View File

@@ -52,3 +52,13 @@ class BlockCandidate:
score: int
reasons: list[str]
sample_events: list[LogEvent]
@dataclass(frozen=True)
class AnomalyFinding:
subject: str
score: int
severity: str
confidence: str
reasons: list[str]
evidence: dict[str, int | str | float]

92
src/fgai/monitor.py Normal file
View File

@@ -0,0 +1,92 @@
from __future__ import annotations
import json
import time
from pathlib import Path
from .anomaly import anomaly_summary, detect_source_anomalies
from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import parse_allowlist, suggest_block_candidates
from .policies import audit_policies, read_policies
def build_status(
log_path: str,
*,
policy_path: str | None = None,
min_block_events: int = 3,
min_block_score: int = 7,
anomaly_limit: int = 20,
) -> dict[str, object]:
events = read_events(log_path) if Path(log_path).exists() else []
anomalies = detect_source_anomalies(events, limit=anomaly_limit)
block_candidates = suggest_block_candidates(
events,
min_events=min_block_events,
min_score=min_block_score,
allowlist=parse_allowlist(),
)
policy_findings: list[dict[str, str | None]] = []
policy_error: str | None = None
if policy_path and Path(policy_path).exists():
try:
policy_findings = [finding.__dict__ for finding in audit_policies(read_policies(policy_path))]
except Exception as exc:
policy_error = str(exc)
return {
"generated_at": int(time.time()),
"log_path": log_path,
"policy_path": policy_path,
"summary": summarize_events(events),
"anomaly_summary": anomaly_summary(anomalies),
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),
"top_services": top_field_values(events, "service", limit=10),
"top_actions": top_field_values(events, "action", limit=10),
"top_subtypes": top_field_values(events, "subtype", limit=10),
"local_in_failures": local_in_failures(events, limit=10),
},
"anomalies": [
{
"subject": finding.subject,
"score": finding.score,
"severity": finding.severity,
"confidence": finding.confidence,
"reasons": finding.reasons,
"evidence": finding.evidence,
}
for finding in anomalies
],
"block_candidates": [
{"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons}
for candidate in block_candidates
],
"policy_findings": policy_findings,
"policy_error": policy_error,
}
def write_status(status: dict[str, object], output: str) -> None:
output_path = Path(output)
output_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = output_path.with_suffix(f"{output_path.suffix}.tmp")
tmp_path.write_text(json.dumps(status, indent=2, sort_keys=True), encoding="utf-8")
tmp_path.replace(output_path)
def monitor_loop(
log_path: str,
output: str,
*,
policy_path: str | None = None,
interval: int = 10,
anomaly_limit: int = 20,
) -> None:
print(f"Monitoring {log_path}")
print(f"Writing status to {output}")
while True:
status = build_status(log_path, policy_path=policy_path, anomaly_limit=anomaly_limit)
write_status(status, output)
time.sleep(interval)