from __future__ import annotations import argparse import json import sys from pathlib import Path from .anomaly import anomaly_summary, detect_source_anomalies from .dashboard import serve_dashboard from .llm import ollama_summary from .logs import local_in_failures, read_events, summarize_events, top_field_values from .mitigation import FortiGateClient, parse_allowlist, suggest_block_candidates from .policies import audit_policies, read_policies from .recommendations import build_recommendations from .syslog_server import listen_udp_syslog from .monitor import monitor_loop from .threat_intel import enrich_ips, is_public_ip from .config import ConfigStore from .baseline import BaselineStore, DEFAULT_MAX_VALUES_PER_FIELD, DEFAULT_RETENTION_DAYS, DEFAULT_VALUE_RETENTION_DAYS from .exports import investigation_report, investigation_report_markdown from .graylog_mcp import GraylogMcpClient from .graylog_source import GraylogStreamSource from .replay import replay_comparison, replay_events from .stream_profiles import parse_profiles def _print_json(data: object) -> None: print(json.dumps(data, indent=2, sort_keys=True)) def analyze_logs(args: argparse.Namespace) -> int: events = read_events(args.logs) candidates = suggest_block_candidates( events, min_events=args.min_events, min_score=args.min_score, allowlist=parse_allowlist(args.allowlist), ) anomalies = detect_source_anomalies(events, limit=args.anomaly_limit) analysis = { "summary": summarize_events(events), "anomaly_summary": anomaly_summary(anomalies), "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), "top_services": top_field_values(events, "service", limit=10), "top_actions": top_field_values(events, "action", limit=10), "top_subtypes": top_field_values(events, "subtype", limit=10), "local_in_failures": local_in_failures(events, limit=10), }, "block_candidates": [ {"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons} for candidate in candidates ], "anomalies": [ { "subject": finding.subject, "score": finding.score, "severity": finding.severity, "confidence": finding.confidence, "reasons": finding.reasons, "evidence": finding.evidence, } for finding in anomalies ], } _print_json(analysis) if args.llm: print("\nLLM summary:") print(ollama_summary([], candidates, args.model, analysis=analysis, timeout=args.llm_timeout)) return 0 def audit_policy_file(args: argparse.Namespace) -> int: findings = audit_policies(read_policies(args.config)) _print_json([finding.__dict__ for finding in findings]) if args.llm: print("\nLLM summary:") print(ollama_summary(findings, [], args.model, timeout=args.llm_timeout)) return 0 def suggest_blocks(args: argparse.Namespace) -> int: events = read_events(args.logs) candidates = suggest_block_candidates( events, min_events=args.min_events, min_score=args.min_score, allowlist=parse_allowlist(args.allowlist), ) if not candidates: print("No block candidates met the current thresholds.") return 0 for candidate in candidates: print(f"{candidate.src_ip} score={candidate.score} reasons={', '.join(candidate.reasons)}") if not args.execute: print(" dry-run: not blocked") continue client = FortiGateClient.from_env() reason = f"fgAI UTM mitigation score={candidate.score}" response = client.quarantine_ip(candidate.src_ip, args.expiry_minutes, reason) print(f" blocked: {response}") return 0 def detect_anomalies(args: argparse.Namespace) -> int: events = read_events(args.logs) anomalies = detect_source_anomalies(events, limit=args.limit) output = { "summary": anomaly_summary(anomalies), "anomalies": [ { "subject": finding.subject, "score": finding.score, "severity": finding.severity, "confidence": finding.confidence, "reasons": finding.reasons, "evidence": finding.evidence, } for finding in anomalies if finding.score >= args.min_score ], } _print_json(output) if args.llm: print("\nLLM summary:") print(ollama_summary([], [], args.model, analysis=output, timeout=args.llm_timeout)) return 0 def recommend(args: argparse.Namespace) -> int: events = read_events(args.logs) anomalies = detect_source_anomalies(events, limit=args.limit) intel_ips = sorted( { ip for event in events for ip in (event.src_ip, event.dst_ip) if is_public_ip(ip) } ) reputation = enrich_ips(intel_ips, limit=args.intel_limit) if args.threat_intel else {} recommendations = build_recommendations(events, anomalies, reputation) output = { "recommendations": [ { "subject": item.subject, "score": item.score, "severity": item.severity, "title": item.title, "recommendation": item.recommendation, "reasons": item.reasons, "related_policy_ids": item.related_policy_ids, "related_services": item.related_services, } for item in recommendations if item.score >= args.min_score ], "reputation": reputation, } _print_json(output) if args.llm: print("\nLLM summary:") print(ollama_summary([], [], args.model, analysis=output, timeout=args.llm_timeout)) return 0 def test_connection(args: argparse.Namespace) -> int: client = FortiGateClient.from_env() status = client.system_status() results = status.get("results", status) print("FortiGate API connection OK") _print_json(results) return 0 def fetch_policies(args: argparse.Namespace) -> int: client = FortiGateClient.from_env() policies = client.firewall_policies() if args.output: output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(policies, indent=2, sort_keys=True), encoding="utf-8") print(f"Wrote {output}") else: _print_json(policies) return 0 def listen_syslog(args: argparse.Namespace) -> int: listen_udp_syslog( args.host, args.port, args.output, rotate_bytes=args.rotate_bytes, rotate_count=args.rotate_count, quiet=args.quiet, ) return 0 def run_monitor(args: argparse.Namespace) -> int: monitor_loop( args.logs, args.output, policy_path=args.policies, interval=args.interval, anomaly_limit=args.anomaly_limit, llm=args.llm, llm_interval=args.llm_interval, llm_model=args.model, llm_timeout=args.llm_timeout, baseline_path=args.baseline_db, config_path=args.config_file, history_path=args.history_db, status_cache_path=args.status_cache_db, ) return 0 def run_dashboard(args: argparse.Namespace) -> int: serve_dashboard(args.host, args.port, args.status_file, image_dir=args.image_dir, config_file=args.config_file) return 0 def export_investigation(args: argparse.Namespace) -> int: status_path = Path(args.status_file) status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {} report = investigation_report(status, incident_id=args.incident_id or None) fmt = "markdown" if args.format == "md" else args.format output = investigation_report_markdown(report) if fmt == "markdown" else json.dumps(report, indent=2, sort_keys=True) if args.output: path = Path(args.output) path.parent.mkdir(parents=True, exist_ok=True) path.write_text(output, encoding="utf-8") print(f"Wrote {path}") else: print(output, end="" if output.endswith("\n") else "\n") return 0 def baseline_maintenance(args: argparse.Namespace) -> int: result = BaselineStore(args.baseline_db).maintenance( retention_days=args.retention_days, value_retention_days=args.value_retention_days, max_values_per_field=args.max_values_per_field, vacuum=args.vacuum, include_rows=True, ) _print_json(result) return 0 def _stream_name(config: dict[str, object], stream_id: str) -> str: return next((str(item.get("title", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") == stream_id), stream_id) def _profile_fields(profile: object | None) -> tuple[str, ...]: if not profile: return () return tuple( field for field in ( str(getattr(profile, "entity_field", "")), *tuple(str(item) for item in getattr(profile, "entity_fields", ())), str(getattr(profile, "timestamp_field", "")), *tuple(str(item) for item in getattr(profile, "categorical_fields", ())), *tuple(str(item) for item in getattr(profile, "numeric_fields", ())), *tuple(str(getattr(relation, "left", "")) for relation in getattr(profile, "relationship_fields", ())), *tuple(str(getattr(relation, "right", "")) for relation in getattr(profile, "relationship_fields", ())), ) if field ) def _configured_streams(config: dict[str, object], selected: list[str] | None = None) -> list[dict[str, str]]: selected_order = list(dict.fromkeys(item for item in selected or [] if item)) selected_set = set(selected_order) configured = [ {"id": str(item.get("id", "")), "title": str(item.get("title", ""))} for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") and (item.get("enabled") or selected_set) ] if selected_set: configured = [item for item in configured if item["id"] in selected_set] by_id = {item["id"]: item for item in configured} configured = [by_id.get(stream_id, {"id": stream_id, "title": _stream_name(config, stream_id)}) for stream_id in selected_order] if not configured and str(config.get("graylog_stream", "")): stream_id = str(config.get("graylog_stream", "")) configured = [{"id": stream_id, "title": _stream_name(config, stream_id)}] return configured def replay_history(args: argparse.Namespace) -> int: config = ConfigStore(args.config_file).read() profiles = parse_profiles(config.get("graylog_stream_profiles", [])) stream_name = _stream_name(config, args.stream_id) events = read_events(args.logs) result = replay_events(events, profiles, stream_id=args.stream_id, stream_name=stream_name, bucket_seconds=args.bucket_seconds) if args.compare_config_file: candidate_config = ConfigStore(args.compare_config_file).read() candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", [])) candidate = replay_events(events, candidate_profiles, stream_id=args.stream_id, stream_name=_stream_name(candidate_config, args.stream_id), bucket_seconds=args.bucket_seconds) result = {"current": result, "candidate": candidate, "comparison": replay_comparison(result, candidate), "candidate_config_file": args.compare_config_file} _print_json(result) return 0 def replay_graylog(args: argparse.Namespace) -> int: config = ConfigStore(args.config_file).read() url, token = str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")) if not url or not token: raise RuntimeError("graylog_mcp_not_configured") profiles = parse_profiles(config.get("graylog_stream_profiles", [])) candidate_profiles = {} if args.compare_config_file: candidate_config = ConfigStore(args.compare_config_file).read() candidate_profiles = parse_profiles(candidate_config.get("graylog_stream_profiles", [])) streams = _configured_streams(config, args.stream_id) if not streams: raise RuntimeError("no_graylog_streams_selected") events = [] stream_statuses = [] for stream in streams: stream_id = stream["id"] profile_fields = tuple(dict.fromkeys([*_profile_fields(profiles.get(stream_id)), *_profile_fields(candidate_profiles.get(stream_id))])) stream_events, status = GraylogStreamSource( GraylogMcpClient(url, token), stream_id, str(config.get("graylog_query", "*")), str(config.get("graylog_field_mapping", "")), stream.get("title") or stream_id, profile_fields, ).fetch(max_events=args.max_events_per_stream, range_seconds=args.range_seconds) events.extend(stream_events) stream_statuses.append({"stream_id": stream_id, "stream_name": stream.get("title") or stream_id, **status}) result = { "source": "graylog_mcp", "range_seconds": args.range_seconds, "streams": stream_statuses, "replay": replay_events(events, profiles, bucket_seconds=args.bucket_seconds), } if args.compare_config_file: candidate = replay_events(events, candidate_profiles, bucket_seconds=args.bucket_seconds) current = result["replay"] result["candidate_config_file"] = args.compare_config_file result["candidate_replay"] = candidate result["comparison"] = replay_comparison(current, candidate) _print_json(result) return 0 def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Local FortiGate AI/ML inspection tool") subparsers = parser.add_subparsers(required=True) logs = subparsers.add_parser("analyze-logs", help="Analyze local FortiGate logs") logs.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") logs.add_argument("--min-events", type=int, default=3) logs.add_argument("--min-score", type=int, default=7) logs.add_argument("--allowlist", default=None, help="Comma-separated IPs/CIDRs never to block") logs.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") logs.add_argument("--model", default=None, help="Ollama model name") logs.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") logs.add_argument("--anomaly-limit", type=int, default=20, help="Maximum anomaly findings to include") logs.set_defaults(func=analyze_logs) anomalies = subparsers.add_parser("detect-anomalies", help="Score likely traffic anomalies by source IP") anomalies.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") anomalies.add_argument("--limit", type=int, default=20, help="Maximum anomaly findings to include") anomalies.add_argument("--min-score", type=int, default=1, help="Minimum anomaly score to output") anomalies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") anomalies.add_argument("--model", default=None, help="Ollama model name") anomalies.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") anomalies.set_defaults(func=detect_anomalies) recommendations = subparsers.add_parser("recommend", help="Generate policy and response recommendations from anomalies") recommendations.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") recommendations.add_argument("--limit", type=int, default=20, help="Maximum anomaly findings to evaluate") recommendations.add_argument("--min-score", type=int, default=35, help="Minimum recommendation score to output") recommendations.add_argument("--threat-intel", action="store_true", help="Use enabled external threat intelligence lookups") recommendations.add_argument("--intel-limit", type=int, default=25, help="Maximum public IPs to enrich") recommendations.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") recommendations.add_argument("--model", default=None, help="Ollama model name") recommendations.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") recommendations.set_defaults(func=recommend) policies = subparsers.add_parser("audit-policies", help="Audit FortiOS firewall policy config") policies.add_argument("--config", required=True, help="Path to FortiOS config backup") policies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") policies.add_argument("--model", default=None, help="Ollama model name") policies.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") policies.set_defaults(func=audit_policy_file) blocks = subparsers.add_parser("suggest-blocks", help="Suggest or execute guarded source IP blocks") blocks.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") blocks.add_argument("--min-events", type=int, default=3) blocks.add_argument("--min-score", type=int, default=7) blocks.add_argument("--allowlist", default=None, help="Comma-separated IPs/CIDRs never to block") blocks.add_argument("--execute", action="store_true", help="Actually call the FortiGate quarantine API") blocks.add_argument("--expiry-minutes", type=int, default=60) blocks.set_defaults(func=suggest_blocks) connection = subparsers.add_parser("test-connection", help="Test FortiGate REST API credentials") connection.set_defaults(func=test_connection) fetch = subparsers.add_parser("fetch-policies", help="Fetch firewall policies through the FortiGate REST API") fetch.add_argument("--output", help="Write JSON response to this file") fetch.set_defaults(func=fetch_policies) listener = subparsers.add_parser("listen-syslog", help="Listen for UDP syslog and append to a local log file") listener.add_argument("--host", default="0.0.0.0", help="Bind address") listener.add_argument("--port", type=int, default=5514, help="UDP port. Use 514 only with sudo/capability.") listener.add_argument("--output", default="logs/fg_syslog.jsonl", help="File to append received logs to") listener.add_argument("--rotate-bytes", type=int, default=25 * 1024 * 1024, help="Rotate active log at this size; 0 disables rotation") listener.add_argument("--rotate-count", type=int, default=14, help="Number of compressed log archives to retain") listener.add_argument("--quiet", action="store_true", help="Do not print each received syslog message") listener.set_defaults(func=listen_syslog) monitor = subparsers.add_parser("monitor", help="Continuously analyze logs and write dashboard status JSON") monitor.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") monitor.add_argument("--output", default="state/fgai-status.json", help="Status JSON written for dashboard") monitor.add_argument("--policies", default=None, help="Optional FortiGate policy JSON/config file to audit continuously") monitor.add_argument("--interval", type=int, default=10, help="Seconds between analysis runs") monitor.add_argument("--anomaly-limit", type=int, default=20, help="Maximum anomaly findings to include") monitor.add_argument("--baseline-db", default="state/fgai-baseline.sqlite3", help="SQLite database for historical behavior baselines") monitor.add_argument("--config-file", default="state/fgai-config.json", help="Runtime configuration written by dashboard") monitor.add_argument("--history-db", default="state/signalscope-history.sqlite3", help="Aggregate history database for dashboard trends") monitor.add_argument("--status-cache-db", default="state/signalscope-status-cache.sqlite3", help="SQLite cache for last successful dashboard status") monitor.add_argument("--llm", action="store_true", help="Generate cached Ollama analyst note for dashboard") monitor.add_argument("--llm-interval", type=int, default=300, help="Seconds between Ollama dashboard assessments") monitor.add_argument("--model", default=None, help="Ollama model name") monitor.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") monitor.set_defaults(func=run_monitor) dashboard = subparsers.add_parser("dashboard", help="Serve local fgAI dashboard") dashboard.add_argument("--host", default="127.0.0.1", help="Dashboard bind address") dashboard.add_argument("--port", type=int, default=8088, help="Dashboard TCP port") dashboard.add_argument("--status-file", default="state/fgai-status.json", help="Status JSON produced by monitor") dashboard.add_argument("--image-dir", default="images", help="Directory containing dashboard images") dashboard.add_argument("--config-file", default="state/fgai-config.json", help="Local runtime configuration JSON") dashboard.set_defaults(func=run_dashboard) export = subparsers.add_parser("export-investigation", help="Export current investigation incidents as Markdown or JSON") export.add_argument("--status-file", default="state/fgai-status.json", help="Status JSON produced by monitor") export.add_argument("--incident-id", default="", help="Export only one incident ID") export.add_argument("--format", choices=["markdown", "md", "json"], default="markdown", help="Report format") export.add_argument("--output", default="", help="Write the report to this file instead of stdout") export.set_defaults(func=export_investigation) maintenance = subparsers.add_parser("baseline-maintenance", help="Prune and optionally compact the local SQLite baseline") maintenance.add_argument("--baseline-db", default="state/fgai-baseline.sqlite3", help="SQLite baseline database") maintenance.add_argument("--retention-days", type=int, default=DEFAULT_RETENTION_DAYS, help="Keep bucket and dedupe history for this many days") maintenance.add_argument("--value-retention-days", type=int, default=DEFAULT_VALUE_RETENTION_DAYS, help="Prune stale one-off categorical values after this many days") maintenance.add_argument("--max-values-per-field", type=int, default=DEFAULT_MAX_VALUES_PER_FIELD, help="Keep at most this many categorical values per stream/entity/field") maintenance.add_argument("--vacuum", action="store_true", help="Run SQLite VACUUM after pruning to return disk space") maintenance.set_defaults(func=baseline_maintenance) replay = subparsers.add_parser("replay", help="Replay a historical log export against temporary baselines") replay.add_argument("--logs", required=True, help="Historic JSONL or key/value log export") replay.add_argument("--config-file", default="state/fgai-config.json", help="Stream profile configuration") replay.add_argument("--stream-id", default="", help="Apply this configured Graylog stream profile to exported events") replay.add_argument("--bucket-seconds", type=int, default=300, help="Replay baseline bucket size") replay.add_argument("--compare-config-file", default="", help="Replay the same export against another saved config and report detector deltas") replay.set_defaults(func=replay_history) replay_mcp = subparsers.add_parser("replay-graylog", help="Replay a Graylog MCP time window against temporary baselines") replay_mcp.add_argument("--config-file", default="state/fgai-config.json", help="Runtime configuration with Graylog MCP URL, token, streams and profiles") replay_mcp.add_argument("--stream-id", action="append", default=[], help="Replay only this stream ID. Repeat for multiple streams. Defaults to enabled streams.") replay_mcp.add_argument("--range-seconds", type=int, default=24 * 3600, help="Graylog relative time window to fetch") replay_mcp.add_argument("--max-events-per-stream", type=int, default=5_000, help="Maximum events fetched from each selected stream") replay_mcp.add_argument("--bucket-seconds", type=int, default=300, help="Replay baseline bucket size") replay_mcp.add_argument("--compare-config-file", default="", help="Replay the same Graylog events against another saved config and report detector deltas") replay_mcp.set_defaults(func=replay_graylog) return parser def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) try: return args.func(args) except Exception as exc: print(f"fgai: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())