Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ad VS

Browse Source
This commit is contained in:
larssand
2026-06-18 22:51:54 +02:00
parent 5650973e50
commit f7611ef2b3
8 changed files with 374 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/fgai/cli.py
View File

@@ -11,8 +11,10 @@ from .llm import ollama_summary
from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import FortiGateClient, parse_allowlist, suggest_block_candidates
from .policies import audit_policies, read_policies
from .recommendations import build_recommendations
from .syslog_server import listen_udp_syslog
from .monitor import monitor_loop
from .threat_intel import enrich_ips, is_public_ip
def _print_json(data: object) -> None:
@@ -119,6 +121,43 @@ def detect_anomalies(args: argparse.Namespace) -> int:
return 0
def recommend(args: argparse.Namespace) -> int:
events = read_events(args.logs)
anomalies = detect_source_anomalies(events, limit=args.limit)
intel_ips = sorted(
{
ip
for event in events
for ip in (event.src_ip, event.dst_ip)
if is_public_ip(ip)
}
)
reputation = enrich_ips(intel_ips, limit=args.intel_limit) if args.threat_intel else {}
recommendations = build_recommendations(events, anomalies, reputation)
output = {
"recommendations": [
{
"subject": item.subject,
"score": item.score,
"severity": item.severity,
"title": item.title,
"recommendation": item.recommendation,
"reasons": item.reasons,
"related_policy_ids": item.related_policy_ids,
"related_services": item.related_services,
}
for item in recommendations
if item.score >= args.min_score
],
"reputation": reputation,
}
_print_json(output)
if args.llm:
print("\nLLM summary:")
print(ollama_summary([], [], args.model, analysis=output, timeout=args.llm_timeout))
return 0
def test_connection(args: argparse.Namespace) -> int:
client = FortiGateClient.from_env()
status = client.system_status()
@@ -186,6 +225,17 @@ def build_parser() -> argparse.ArgumentParser:
anomalies.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds")
anomalies.set_defaults(func=detect_anomalies)
recommendations = subparsers.add_parser("recommend", help="Generate policy and response recommendations from anomalies")
recommendations.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file")
recommendations.add_argument("--limit", type=int, default=20, help="Maximum anomaly findings to evaluate")
recommendations.add_argument("--min-score", type=int, default=35, help="Minimum recommendation score to output")
recommendations.add_argument("--threat-intel", action="store_true", help="Use enabled external threat intelligence lookups")
recommendations.add_argument("--intel-limit", type=int, default=25, help="Maximum public IPs to enrich")
recommendations.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results")
recommendations.add_argument("--model", default=None, help="Ollama model name")
recommendations.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds")
recommendations.set_defaults(func=recommend)
policies = subparsers.add_parser("audit-policies", help="Audit FortiOS firewall policy config")
policies.add_argument("--config", required=True, help="Path to FortiOS config backup")
policies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results")

20
src/fgai/dashboard.py
View File

@@ -47,7 +47,9 @@ HTML = """<!doctype html>
</div>
</section>
<section class="panel"><h2>Anomalies</h2><div id="anomalies"></div></section>
<section class="panel"><h2>Recommendations</h2><div id="recommendations"></div></section>
<section class="panel"><h2>Block Candidates</h2><div id="blocks"></div></section>
<section class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></section>
<section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section>
<section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section>
</main>
@@ -89,11 +91,29 @@ async function refresh() {
{label:'Confidence', key:'confidence'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [
{label:'Subject', key:'subject'},
{label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Title', key:'title'},
{label:'Recommendation', key:'recommendation'},
{label:'Policies', render:r => esc((r.related_policy_ids || []).join(', '))},
{label:'Services', render:r => esc((r.related_services || []).join(', '))}
]);
document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
{label:'Source', key:'src_ip'},
{label:'Score', key:'score'},
{label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
]);
const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
document.getElementById('reputation').innerHTML = table(reputationRows, [
{label:'IP', key:'ip'},
{label:'Provider', key:'provider'},
{label:'Status', key:'status'},
{label:'Score', key:'score'},
{label:'Malicious', key:'malicious'},
{label:'Suspicious', key:'suspicious'}
]);
document.getElementById('policies').innerHTML = table(data.policy_findings || [], [
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'Reference', key:'reference'},

16
src/fgai/models.py
View File

@@ -12,6 +12,10 @@ class LogEvent:
def src_ip(self) -> str | None:
return self.fields.get("srcip") or self.fields.get("src_ip") or self.fields.get("source_ip")
@property
def dst_ip(self) -> str | None:
return self.fields.get("dstip") or self.fields.get("dst_ip") or self.fields.get("destination_ip")
@property
def subtype(self) -> str:
return self.fields.get("subtype", "").lower()
@@ -62,3 +66,15 @@ class AnomalyFinding:
confidence: str
reasons: list[str]
evidence: dict[str, int | str | float]
@dataclass(frozen=True)
class Recommendation:
subject: str
score: int
severity: str
title: str
recommendation: str
reasons: list[str]
related_policy_ids: list[str] = field(default_factory=list)
related_services: list[str] = field(default_factory=list)

26
src/fgai/monitor.py
View File

@@ -8,6 +8,8 @@ from .anomaly import anomaly_summary, detect_source_anomalies
from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import parse_allowlist, suggest_block_candidates
from .policies import audit_policies, read_policies
from .recommendations import build_recommendations
from .threat_intel import enrich_ips, is_public_ip
def build_status(
@@ -20,6 +22,16 @@ def build_status(
) -> dict[str, object]:
events = read_events(log_path) if Path(log_path).exists() else []
anomalies = detect_source_anomalies(events, limit=anomaly_limit)
intel_ips = sorted(
{
ip
for event in events
for ip in (event.src_ip, event.dst_ip)
if is_public_ip(ip)
}
)
reputation = enrich_ips(intel_ips, limit=25)
recommendations = build_recommendations(events, anomalies, reputation)
block_candidates = suggest_block_candidates(
events,
min_events=min_block_events,
@@ -59,6 +71,20 @@ def build_status(
}
for finding in anomalies
],
"recommendations": [
{
"subject": item.subject,
"score": item.score,
"severity": item.severity,
"title": item.title,
"recommendation": item.recommendation,
"reasons": item.reasons,
"related_policy_ids": item.related_policy_ids,
"related_services": item.related_services,
}
for item in recommendations
],
"reputation": reputation,
"block_candidates": [
{"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons}
for candidate in block_candidates

114
src/fgai/recommendations.py Normal file
View File

@@ -0,0 +1,114 @@
from __future__ import annotations
from collections import Counter, defaultdict
from .models import AnomalyFinding, LogEvent, Recommendation
from .threat_intel import is_public_ip
def _severity(score: int) -> str:
if score >= 80:
return "critical"
if score >= 60:
return "high"
if score >= 35:
return "medium"
return "low"
def _top_values(events: list[LogEvent], field: str, limit: int = 5) -> list[str]:
counter: Counter[str] = Counter()
for event in events:
value = event.fields.get(field)
if value:
counter[value] += 1
return [value for value, _ in counter.most_common(limit)]
def _events_by_src(events: list[LogEvent]) -> dict[str, list[LogEvent]]:
grouped: dict[str, list[LogEvent]] = defaultdict(list)
for event in events:
if event.src_ip:
grouped[event.src_ip].append(event)
return grouped
def build_recommendations(
events: list[LogEvent],
anomalies: list[AnomalyFinding],
reputation: dict[str, dict[str, object]] | None = None,
) -> list[Recommendation]:
reputation = reputation or {}
grouped = _events_by_src(events)
recommendations: list[Recommendation] = []
for anomaly in anomalies:
src_events = grouped.get(anomaly.subject, [])
if not src_events:
continue
policy_ids = _top_values(src_events, "policyid")
services = _top_values(src_events, "service")
dst_ips = [event.dst_ip for event in src_events if event.dst_ip]
public_dst = [ip for ip in _top_values(src_events, "dstip", limit=10) if is_public_ip(ip)]
bad_reputation = [
f"{ip}:score={intel.get('score')} status={intel.get('status')}"
for ip, intel in reputation.items()
if int(intel.get("score", 0) or 0) >= 50 and (ip == anomaly.subject or ip in dst_ips)
]
score = anomaly.score
reasons = list(anomaly.reasons)
if bad_reputation:
score = min(100, score + 20)
reasons.append(f"threat intelligence hit ({'; '.join(bad_reputation[:3])})")
if is_public_ip(anomaly.subject) and anomaly.score >= 60:
title = "Quarantine or block suspicious public source"
action = (
"Inspect the matching FortiGate logs and policy IDs, then quarantine the source IP "
"temporarily if the traffic is unsolicited or UTM-confirmed. Convert to a permanent "
"address object/block only after reputation and business impact are verified."
)
elif public_dst and bad_reputation:
title = "Investigate risky destination reputation"
action = (
"Inspect the affected internal client, DNS history, and policy path. Consider blocking "
"the destination with an address object or ISDB/category control if reputation remains malicious."
)
elif anomaly.evidence.get("distinct_destinations", 0) >= 10:
title = "Investigate scan-like or fan-out traffic"
action = (
"Inspect the source host and the matching policies. If this is not expected discovery or monitoring, "
"limit allowed destinations/services and add IPS/application control on the policy."
)
elif anomaly.evidence.get("distinct_services", 0) >= 8:
title = "Restrict broad service usage"
action = (
"Review the matching policy services. Replace broad service objects such as ALL with the observed "
"business-required services only."
)
elif anomaly.evidence.get("utm_events", 0):
title = "Review UTM-triggering traffic"
action = (
"Inspect the IPS/AV/WebFilter event details and policy. Keep or enable UTM profiles on this policy, "
"and tighten source/destination scope if the traffic is not expected."
)
else:
title = "Review traffic anomaly"
action = "Inspect the related policy and host behavior before changing enforcement."
recommendations.append(
Recommendation(
subject=anomaly.subject,
score=min(100, score),
severity=_severity(min(100, score)),
title=title,
recommendation=action,
reasons=reasons,
related_policy_ids=policy_ids,
related_services=services,
)
)
return sorted(recommendations, key=lambda item: item.score, reverse=True)

94
src/fgai/threat_intel.py Normal file
View File

@@ -0,0 +1,94 @@
from __future__ import annotations
import ipaddress
import json
import os
import time
from pathlib import Path
from urllib import error, request
def is_public_ip(value: str | None) -> bool:
if not value:
return False
try:
return ipaddress.ip_address(value).is_global
except ValueError:
return False
class ThreatIntelClient:
def __init__(self, *, cache_file: str = "state/threat-intel-cache.json", ttl_seconds: int = 86400) -> None:
self.enabled = os.getenv("FGAI_THREAT_INTEL", "").lower() in {"1", "true", "yes", "on"}
self.virustotal_key = os.getenv("VIRUSTOTAL_API_KEY")
self.cache_path = Path(cache_file)
self.ttl_seconds = ttl_seconds
self.cache = self._read_cache()
def _read_cache(self) -> dict[str, dict[str, object]]:
if not self.cache_path.exists():
return {}
try:
return json.loads(self.cache_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return {}
def _write_cache(self) -> None:
self.cache_path.parent.mkdir(parents=True, exist_ok=True)
self.cache_path.write_text(json.dumps(self.cache, indent=2, sort_keys=True), encoding="utf-8")
def lookup_ip(self, ip: str) -> dict[str, object]:
if not is_public_ip(ip):
return {"ip": ip, "provider": "local", "status": "skipped", "reason": "not_public_ip", "score": 0}
cached = self.cache.get(ip)
now = int(time.time())
if cached and now - int(cached.get("cached_at", 0)) < self.ttl_seconds:
return cached
if not self.enabled:
return {"ip": ip, "provider": "none", "status": "disabled", "score": 0}
if not self.virustotal_key:
return {"ip": ip, "provider": "virustotal", "status": "missing_api_key", "score": 0}
result = self._lookup_virustotal_ip(ip)
result["cached_at"] = now
self.cache[ip] = result
self._write_cache()
return result
def _lookup_virustotal_ip(self, ip: str) -> dict[str, object]:
req = request.Request(
f"https://www.virustotal.com/api/v3/ip_addresses/{ip}",
headers={"x-apikey": self.virustotal_key or "", "accept": "application/json"},
)
try:
with request.urlopen(req, timeout=20) as response:
data = json.loads(response.read().decode("utf-8"))
except error.HTTPError as exc:
return {"ip": ip, "provider": "virustotal", "status": f"http_{exc.code}", "score": 0}
except Exception as exc:
return {"ip": ip, "provider": "virustotal", "status": "error", "error": str(exc), "score": 0}
stats = data.get("data", {}).get("attributes", {}).get("last_analysis_stats", {})
malicious = int(stats.get("malicious", 0) or 0)
suspicious = int(stats.get("suspicious", 0) or 0)
harmless = int(stats.get("harmless", 0) or 0)
undetected = int(stats.get("undetected", 0) or 0)
score = min(100, malicious * 20 + suspicious * 10)
return {
"ip": ip,
"provider": "virustotal",
"status": "ok",
"score": score,
"malicious": malicious,
"suspicious": suspicious,
"harmless": harmless,
"undetected": undetected,
}
def enrich_ips(ips: list[str], *, cache_file: str = "state/threat-intel-cache.json", limit: int = 25) -> dict[str, dict[str, object]]:
client = ThreatIntelClient(cache_file=cache_file)
enriched: dict[str, dict[str, object]] = {}
for ip in ips[:limit]:
enriched[ip] = client.lookup_ip(ip)
return enriched