diff --git a/README.md b/README.md index 9bab077..1af3d05 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ Or use the helper script, which creates/uses `.venv` automatically and runs `pip ./start.sh stop ``` +`./start.sh` starts three local background processes: + +- UDP syslog listener writing `logs/fg_syslog.jsonl` +- Continuous monitor writing `state/fgai-status.json` +- Local dashboard at `http://127.0.0.1:8088` + The script activates `.venv` inside the script process. If you also want your current shell prompt to show the venv, run: ```bash @@ -39,6 +45,19 @@ Analyze local logs: fgai analyze-logs --logs logs/fg_syslog.jsonl ``` +Open the live UI after `./start.sh`: + +```bash +xdg-open http://127.0.0.1:8088 +``` + +Score likely traffic anomalies: + +```bash +fgai detect-anomalies --logs logs/fg_syslog.jsonl --min-score 35 +fgai detect-anomalies --logs logs/fg_syslog.jsonl --min-score 35 --llm --llm-timeout 300 +``` + Listen for FortiGate syslog locally: ```bash diff --git a/src/fgai/anomaly.py b/src/fgai/anomaly.py new file mode 100644 index 0000000..8eb20b4 --- /dev/null +++ b/src/fgai/anomaly.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import ipaddress +from collections import Counter, defaultdict +from statistics import mean, pstdev + +from .logs import THREAT_ACTIONS, event_score, is_utm_event +from .models import AnomalyFinding, LogEvent + + +def _as_int(value: str | None) -> int: + if not value: + return 0 + try: + return int(float(value)) + except ValueError: + return 0 + + +def _severity(score: int) -> str: + if score >= 80: + return "critical" + if score >= 60: + return "high" + if score >= 35: + return "medium" + return "low" + + +def _confidence(event_count: int, reason_count: int) -> str: + if event_count >= 20 and reason_count >= 3: + return "high" + if event_count >= 5 and reason_count >= 2: + return "medium" + return "low" + + +def _is_public_ip(value: str) -> bool: + try: + return ipaddress.ip_address(value).is_global + except ValueError: + return False + + +def detect_source_anomalies(events: list[LogEvent], *, limit: int = 20) -> list[AnomalyFinding]: + by_src: dict[str, list[LogEvent]] = defaultdict(list) + for event in events: + if event.src_ip: + by_src[event.src_ip].append(event) + + if not by_src: + return [] + + event_counts = [len(src_events) for src_events in by_src.values()] + distinct_dst_counts = [ + len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")}) + for src_events in by_src.values() + ] + byte_totals = [ + sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events) + for src_events in by_src.values() + ] + avg_events = mean(event_counts) + std_events = pstdev(event_counts) or 1.0 + avg_dst = mean(distinct_dst_counts) + std_dst = pstdev(distinct_dst_counts) or 1.0 + avg_bytes = mean(byte_totals) + std_bytes = pstdev(byte_totals) or 1.0 + + findings: list[AnomalyFinding] = [] + for src_ip, src_events in by_src.items(): + event_count = len(src_events) + distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")}) + distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")}) + total_bytes = sum(_as_int(event.fields.get("sentbyte")) + _as_int(event.fields.get("rcvdbyte")) for event in src_events) + deny_count = sum(1 for event in src_events if event.action in THREAT_ACTIONS) + utm_count = sum(1 for event in src_events if is_utm_event(event)) + high_severity_count = sum(1 for event in src_events if event.severity in {"critical", "high", "alert", "emergency"}) + policies = {event.fields.get("policyid") for event in src_events if event.fields.get("policyid")} + + reasons: list[str] = [] + score = 0 + + event_z = (event_count - avg_events) / std_events + if event_count >= 25 and event_z >= 2: + points = min(25, 10 + int(event_z * 5)) + score += points + reasons.append(f"unusually high event volume for source ({event_count} events, z={event_z:.1f})") + + dst_z = (distinct_dst - avg_dst) / std_dst + if distinct_dst >= 10 and dst_z >= 2: + points = min(25, 10 + int(dst_z * 5)) + score += points + reasons.append(f"source contacted unusually many destinations ({distinct_dst}, z={dst_z:.1f})") + + byte_z = (total_bytes - avg_bytes) / std_bytes + if total_bytes >= 50_000_000 and byte_z >= 2: + points = min(20, 8 + int(byte_z * 4)) + score += points + reasons.append(f"unusually high byte volume ({total_bytes} bytes, z={byte_z:.1f})") + + if event_count >= 5: + deny_rate = deny_count / event_count + if deny_count >= 10 and deny_rate >= 0.5: + score += min(20, 8 + int(deny_rate * 12)) + reasons.append(f"high deny/threat-action rate ({deny_count}/{event_count})") + + if utm_count: + utm_score = sum(event_score(event) for event in src_events if is_utm_event(event)) + points = min(35, 5 + utm_score) + score += points + reasons.append(f"UTM/security detections observed ({utm_count} events)") + + if high_severity_count: + score += min(20, high_severity_count * 8) + reasons.append(f"high or critical severity events observed ({high_severity_count})") + + if distinct_services >= 8 and event_count >= 10: + score += min(15, distinct_services) + reasons.append(f"many distinct services used ({distinct_services})") + + if _is_public_ip(src_ip) and (utm_count or deny_count >= 10): + score += 10 + reasons.append("public source with repeated security-relevant events") + + if not reasons: + continue + + score = min(score, 100) + findings.append( + AnomalyFinding( + subject=src_ip, + score=score, + severity=_severity(score), + confidence=_confidence(event_count, len(reasons)), + reasons=reasons, + evidence={ + "events": event_count, + "distinct_destinations": distinct_dst, + "distinct_services": distinct_services, + "deny_or_threat_actions": deny_count, + "utm_events": utm_count, + "high_severity_events": high_severity_count, + "total_bytes": total_bytes, + "policy_count": len(policies), + }, + ) + ) + + return sorted(findings, key=lambda finding: finding.score, reverse=True)[:limit] + + +def anomaly_summary(findings: list[AnomalyFinding]) -> dict[str, int]: + counts = Counter(finding.severity for finding in findings) + return { + "total": len(findings), + "critical": counts["critical"], + "high": counts["high"], + "medium": counts["medium"], + "low": counts["low"], + } diff --git a/src/fgai/cli.py b/src/fgai/cli.py index f692abe..957cba3 100644 --- a/src/fgai/cli.py +++ b/src/fgai/cli.py @@ -5,11 +5,14 @@ import json import sys from pathlib import Path +from .anomaly import anomaly_summary, detect_source_anomalies +from .dashboard import serve_dashboard from .llm import ollama_summary from .logs import local_in_failures, read_events, summarize_events, top_field_values from .mitigation import FortiGateClient, parse_allowlist, suggest_block_candidates from .policies import audit_policies, read_policies from .syslog_server import listen_udp_syslog +from .monitor import monitor_loop def _print_json(data: object) -> None: @@ -24,8 +27,10 @@ def analyze_logs(args: argparse.Namespace) -> int: min_score=args.min_score, allowlist=parse_allowlist(args.allowlist), ) + anomalies = detect_source_anomalies(events, limit=args.anomaly_limit) analysis = { "summary": summarize_events(events), + "anomaly_summary": anomaly_summary(anomalies), "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), "top_services": top_field_values(events, "service", limit=10), @@ -37,6 +42,17 @@ def analyze_logs(args: argparse.Namespace) -> int: {"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons} for candidate in candidates ], + "anomalies": [ + { + "subject": finding.subject, + "score": finding.score, + "severity": finding.severity, + "confidence": finding.confidence, + "reasons": finding.reasons, + "evidence": finding.evidence, + } + for finding in anomalies + ], } _print_json(analysis) if args.llm: @@ -78,6 +94,31 @@ def suggest_blocks(args: argparse.Namespace) -> int: return 0 +def detect_anomalies(args: argparse.Namespace) -> int: + events = read_events(args.logs) + anomalies = detect_source_anomalies(events, limit=args.limit) + output = { + "summary": anomaly_summary(anomalies), + "anomalies": [ + { + "subject": finding.subject, + "score": finding.score, + "severity": finding.severity, + "confidence": finding.confidence, + "reasons": finding.reasons, + "evidence": finding.evidence, + } + for finding in anomalies + if finding.score >= args.min_score + ], + } + _print_json(output) + if args.llm: + print("\nLLM summary:") + print(ollama_summary([], [], args.model, analysis=output, timeout=args.llm_timeout)) + return 0 + + def test_connection(args: argparse.Namespace) -> int: client = FortiGateClient.from_env() status = client.system_status() @@ -105,6 +146,22 @@ def listen_syslog(args: argparse.Namespace) -> int: return 0 +def run_monitor(args: argparse.Namespace) -> int: + monitor_loop( + args.logs, + args.output, + policy_path=args.policies, + interval=args.interval, + anomaly_limit=args.anomaly_limit, + ) + return 0 + + +def run_dashboard(args: argparse.Namespace) -> int: + serve_dashboard(args.host, args.port, args.status_file, image_dir=args.image_dir) + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Local FortiGate AI/ML inspection tool") subparsers = parser.add_subparsers(required=True) @@ -117,8 +174,18 @@ def build_parser() -> argparse.ArgumentParser: logs.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") logs.add_argument("--model", default=None, help="Ollama model name") logs.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") + logs.add_argument("--anomaly-limit", type=int, default=20, help="Maximum anomaly findings to include") logs.set_defaults(func=analyze_logs) + anomalies = subparsers.add_parser("detect-anomalies", help="Score likely traffic anomalies by source IP") + anomalies.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") + anomalies.add_argument("--limit", type=int, default=20, help="Maximum anomaly findings to include") + anomalies.add_argument("--min-score", type=int, default=1, help="Minimum anomaly score to output") + anomalies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") + anomalies.add_argument("--model", default=None, help="Ollama model name") + anomalies.add_argument("--llm-timeout", type=int, default=None, help="Ollama request timeout in seconds") + anomalies.set_defaults(func=detect_anomalies) + policies = subparsers.add_parser("audit-policies", help="Audit FortiOS firewall policy config") policies.add_argument("--config", required=True, help="Path to FortiOS config backup") policies.add_argument("--llm", action="store_true", help="Ask local Ollama to summarize results") @@ -149,6 +216,21 @@ def build_parser() -> argparse.ArgumentParser: listener.add_argument("--quiet", action="store_true", help="Do not print each received syslog message") listener.set_defaults(func=listen_syslog) + monitor = subparsers.add_parser("monitor", help="Continuously analyze logs and write dashboard status JSON") + monitor.add_argument("--logs", required=True, help="Path to syslog JSONL or key/value log file") + monitor.add_argument("--output", default="state/fgai-status.json", help="Status JSON written for dashboard") + monitor.add_argument("--policies", default=None, help="Optional FortiGate policy JSON/config file to audit continuously") + monitor.add_argument("--interval", type=int, default=10, help="Seconds between analysis runs") + monitor.add_argument("--anomaly-limit", type=int, default=20, help="Maximum anomaly findings to include") + monitor.set_defaults(func=run_monitor) + + dashboard = subparsers.add_parser("dashboard", help="Serve local fgAI dashboard") + dashboard.add_argument("--host", default="127.0.0.1", help="Dashboard bind address") + dashboard.add_argument("--port", type=int, default=8088, help="Dashboard TCP port") + dashboard.add_argument("--status-file", default="state/fgai-status.json", help="Status JSON produced by monitor") + dashboard.add_argument("--image-dir", default="images", help="Directory containing dashboard images") + dashboard.set_defaults(func=run_dashboard) + return parser diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py new file mode 100644 index 0000000..ae5d7bd --- /dev/null +++ b/src/fgai/dashboard.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +HTML = """ + + + + + fgAI Monitor + + + +

fgAI Monitor

+
+
+ FortiGate AI/ML Analyzer +
+
+

Live Status

Waiting for monitor data.
+
+
+

Anomalies

+

Block Candidates

+

Policy Findings

+

Diagnostics

+
+ + + +""" + + +def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | None = None) -> None: + status_path = Path(status_file) + image_root = Path(image_dir) if image_dir else None + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path == "/": + self._send(200, "text/html; charset=utf-8", HTML.encode("utf-8")) + return + if self.path == "/api/status": + if status_path.exists(): + body = status_path.read_bytes() + else: + body = json.dumps({"summary": {}, "anomalies": [], "block_candidates": []}).encode("utf-8") + self._send(200, "application/json", body) + return + if self.path.startswith("/images/") and image_root: + image_path = image_root / Path(self.path).name + if image_path.exists() and image_path.is_file(): + content_type = "image/png" if image_path.suffix.lower() == ".png" else "application/octet-stream" + self._send(200, content_type, image_path.read_bytes()) + return + self._send(404, "text/plain; charset=utf-8", b"not found") + + def log_message(self, format: str, *args: object) -> None: + return + + def _send(self, status: int, content_type: str, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + server = ThreadingHTTPServer((host, port), Handler) + print(f"Dashboard listening on http://{host}:{port}") + print(f"Reading status from {status_path}") + server.serve_forever() diff --git a/src/fgai/models.py b/src/fgai/models.py index 90fc248..994c450 100644 --- a/src/fgai/models.py +++ b/src/fgai/models.py @@ -52,3 +52,13 @@ class BlockCandidate: score: int reasons: list[str] sample_events: list[LogEvent] + + +@dataclass(frozen=True) +class AnomalyFinding: + subject: str + score: int + severity: str + confidence: str + reasons: list[str] + evidence: dict[str, int | str | float] diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py new file mode 100644 index 0000000..dd9bd97 --- /dev/null +++ b/src/fgai/monitor.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from .anomaly import anomaly_summary, detect_source_anomalies +from .logs import local_in_failures, read_events, summarize_events, top_field_values +from .mitigation import parse_allowlist, suggest_block_candidates +from .policies import audit_policies, read_policies + + +def build_status( + log_path: str, + *, + policy_path: str | None = None, + min_block_events: int = 3, + min_block_score: int = 7, + anomaly_limit: int = 20, +) -> dict[str, object]: + events = read_events(log_path) if Path(log_path).exists() else [] + anomalies = detect_source_anomalies(events, limit=anomaly_limit) + block_candidates = suggest_block_candidates( + events, + min_events=min_block_events, + min_score=min_block_score, + allowlist=parse_allowlist(), + ) + + policy_findings: list[dict[str, str | None]] = [] + policy_error: str | None = None + if policy_path and Path(policy_path).exists(): + try: + policy_findings = [finding.__dict__ for finding in audit_policies(read_policies(policy_path))] + except Exception as exc: + policy_error = str(exc) + + return { + "generated_at": int(time.time()), + "log_path": log_path, + "policy_path": policy_path, + "summary": summarize_events(events), + "anomaly_summary": anomaly_summary(anomalies), + "diagnostics": { + "top_source_ips": top_field_values(events, "srcip", limit=10), + "top_services": top_field_values(events, "service", limit=10), + "top_actions": top_field_values(events, "action", limit=10), + "top_subtypes": top_field_values(events, "subtype", limit=10), + "local_in_failures": local_in_failures(events, limit=10), + }, + "anomalies": [ + { + "subject": finding.subject, + "score": finding.score, + "severity": finding.severity, + "confidence": finding.confidence, + "reasons": finding.reasons, + "evidence": finding.evidence, + } + for finding in anomalies + ], + "block_candidates": [ + {"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons} + for candidate in block_candidates + ], + "policy_findings": policy_findings, + "policy_error": policy_error, + } + + +def write_status(status: dict[str, object], output: str) -> None: + output_path = Path(output) + output_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = output_path.with_suffix(f"{output_path.suffix}.tmp") + tmp_path.write_text(json.dumps(status, indent=2, sort_keys=True), encoding="utf-8") + tmp_path.replace(output_path) + + +def monitor_loop( + log_path: str, + output: str, + *, + policy_path: str | None = None, + interval: int = 10, + anomaly_limit: int = 20, +) -> None: + print(f"Monitoring {log_path}") + print(f"Writing status to {output}") + while True: + status = build_status(log_path, policy_path=policy_path, anomaly_limit=anomaly_limit) + write_status(status, output) + time.sleep(interval) diff --git a/start.sh b/start.sh index 93e76bd..7873e60 100755 --- a/start.sh +++ b/start.sh @@ -7,7 +7,16 @@ PORT="${FGAI_SYSLOG_PORT:-5514}" HOST="${FGAI_SYSLOG_HOST:-0.0.0.0}" LOG_FILE="${FGAI_SYSLOG_FILE:-$ROOT_DIR/logs/fg_syslog.jsonl}" LISTENER_LOG="${FGAI_LISTENER_LOG:-$ROOT_DIR/logs/fgai-listener.log}" -PID_FILE="${FGAI_PID_FILE:-$ROOT_DIR/run/fgai-listener.pid}" +POLICY_FILE="${FGAI_POLICY_FILE:-$ROOT_DIR/exports/policies.json}" +STATE_FILE="${FGAI_STATE_FILE:-$ROOT_DIR/state/fgai-status.json}" +MONITOR_INTERVAL="${FGAI_MONITOR_INTERVAL:-10}" +DASHBOARD_HOST="${FGAI_DASHBOARD_HOST:-127.0.0.1}" +DASHBOARD_PORT="${FGAI_DASHBOARD_PORT:-8088}" +LISTENER_PID_FILE="${FGAI_LISTENER_PID_FILE:-$ROOT_DIR/run/fgai-listener.pid}" +MONITOR_PID_FILE="${FGAI_MONITOR_PID_FILE:-$ROOT_DIR/run/fgai-monitor.pid}" +DASHBOARD_PID_FILE="${FGAI_DASHBOARD_PID_FILE:-$ROOT_DIR/run/fgai-dashboard.pid}" +MONITOR_LOG="${FGAI_MONITOR_LOG:-$ROOT_DIR/logs/fgai-monitor.log}" +DASHBOARD_LOG="${FGAI_DASHBOARD_LOG:-$ROOT_DIR/logs/fgai-dashboard.log}" usage() { printf 'Usage: %s [start|stop|restart|status|tail|analyze|install]\n' "$0" @@ -17,6 +26,7 @@ usage() { printf ' FGAI_SYSLOG_HOST=%s\n' "$HOST" printf ' FGAI_SYSLOG_FILE=%s\n' "$LOG_FILE" printf ' FGAI_LISTENER_LOG=%s\n' "$LISTENER_LOG" + printf ' FGAI_DASHBOARD_PORT=%s\n' "$DASHBOARD_PORT" } activate_venv_for_script() { @@ -38,8 +48,8 @@ install_deps() { python -m pip install -e "$ROOT_DIR" } -is_running() { - [ -f "$PID_FILE" ] && ps -p "$(cat "$PID_FILE")" >/dev/null 2>&1 +is_pid_running() { + [ -f "$1" ] && ps -p "$(cat "$1")" >/dev/null 2>&1 } needs_privileged_port() { @@ -56,11 +66,11 @@ listener_command() { start_listener() { install_deps - mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$LISTENER_LOG")" "$(dirname "$PID_FILE")" + mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$LISTENER_LOG")" "$(dirname "$LISTENER_PID_FILE")" "$(dirname "$STATE_FILE")" touch "$LOG_FILE" - if is_running; then - printf 'fgAI syslog listener already running, pid %s\n' "$(cat "$PID_FILE")" + if is_pid_running "$LISTENER_PID_FILE"; then + printf 'fgAI syslog listener already running, pid %s\n' "$(cat "$LISTENER_PID_FILE")" return 0 fi @@ -78,57 +88,121 @@ start_listener() { --output "$LOG_FILE" \ --quiet > "$LISTENER_LOG" 2>&1 & fi - printf '%s\n' "$!" > "$PID_FILE" + printf '%s\n' "$!" > "$LISTENER_PID_FILE" - printf 'Started fgAI syslog listener, pid %s\n' "$(cat "$PID_FILE")" + printf 'Started fgAI syslog listener, pid %s\n' "$(cat "$LISTENER_PID_FILE")" printf 'Input: udp://%s:%s\n' "$HOST" "$PORT" printf 'Syslog file: %s\n' "$LOG_FILE" printf 'Process log: %s\n' "$LISTENER_LOG" } -stop_listener() { - if ! is_running; then - printf 'fgAI syslog listener is not running\n' - rm -f "$PID_FILE" +start_monitor() { + install_deps + mkdir -p "$(dirname "$STATE_FILE")" "$(dirname "$MONITOR_LOG")" "$(dirname "$MONITOR_PID_FILE")" + + if is_pid_running "$MONITOR_PID_FILE"; then + printf 'fgAI monitor already running, pid %s\n' "$(cat "$MONITOR_PID_FILE")" return 0 fi - if ! kill "$(cat "$PID_FILE")" 2>/dev/null; then - sudo kill "$(cat "$PID_FILE")" + monitor_args=(monitor --logs "$LOG_FILE" --output "$STATE_FILE" --interval "$MONITOR_INTERVAL") + if [ -f "$POLICY_FILE" ]; then + monitor_args+=(--policies "$POLICY_FILE") fi - rm -f "$PID_FILE" - printf 'Stopped fgAI syslog listener\n' + + nohup "$VENV_DIR/bin/fgai" "${monitor_args[@]}" > "$MONITOR_LOG" 2>&1 & + printf '%s\n' "$!" > "$MONITOR_PID_FILE" + printf 'Started fgAI monitor, pid %s\n' "$(cat "$MONITOR_PID_FILE")" + printf 'Status file: %s\n' "$STATE_FILE" } -status_listener() { - if is_running; then - printf 'fgAI syslog listener running, pid %s\n' "$(cat "$PID_FILE")" +start_dashboard() { + install_deps + mkdir -p "$(dirname "$DASHBOARD_LOG")" "$(dirname "$DASHBOARD_PID_FILE")" + + if is_pid_running "$DASHBOARD_PID_FILE"; then + printf 'fgAI dashboard already running, pid %s\n' "$(cat "$DASHBOARD_PID_FILE")" + return 0 + fi + + nohup "$VENV_DIR/bin/fgai" dashboard \ + --host "$DASHBOARD_HOST" \ + --port "$DASHBOARD_PORT" \ + --status-file "$STATE_FILE" \ + --image-dir "$ROOT_DIR/images" > "$DASHBOARD_LOG" 2>&1 & + printf '%s\n' "$!" > "$DASHBOARD_PID_FILE" + printf 'Started fgAI dashboard, pid %s\n' "$(cat "$DASHBOARD_PID_FILE")" + printf 'Dashboard: http://%s:%s\n' "$DASHBOARD_HOST" "$DASHBOARD_PORT" +} + +start_all() { + start_listener + start_monitor + start_dashboard +} + +stop_pid() { + label="$1" + pid_file="$2" + if ! is_pid_running "$pid_file"; then + printf '%s is not running\n' "$label" + rm -f "$pid_file" + return 0 + fi + + if ! kill "$(cat "$pid_file")" 2>/dev/null; then + sudo kill "$(cat "$pid_file")" + fi + rm -f "$pid_file" + printf 'Stopped %s\n' "$label" +} + +stop_all() { + stop_pid "fgAI dashboard" "$DASHBOARD_PID_FILE" + stop_pid "fgAI monitor" "$MONITOR_PID_FILE" + stop_pid "fgAI syslog listener" "$LISTENER_PID_FILE" +} + +status_one() { + label="$1" + pid_file="$2" + if is_pid_running "$pid_file"; then + printf '%s running, pid %s\n' "$label" "$(cat "$pid_file")" + return 0 + fi + printf '%s is not running\n' "$label" + return 1 +} + +status_all() { + status_one "fgAI syslog listener" "$LISTENER_PID_FILE" || true + if is_pid_running "$LISTENER_PID_FILE"; then printf 'Input: udp://%s:%s\n' "$HOST" "$PORT" printf 'Syslog file: %s\n' "$LOG_FILE" if command -v ss >/dev/null 2>&1; then ss -lunp 2>/dev/null | awk -v port=":$PORT" '$0 ~ port {print}' fi - return 0 fi - - printf 'fgAI syslog listener is not running\n' - return 1 + status_one "fgAI monitor" "$MONITOR_PID_FILE" || true + status_one "fgAI dashboard" "$DASHBOARD_PID_FILE" || true + printf 'Dashboard: http://%s:%s\n' "$DASHBOARD_HOST" "$DASHBOARD_PORT" + printf 'Status file: %s\n' "$STATE_FILE" } command="${1:-start}" case "$command" in start) - start_listener + start_all ;; stop) - stop_listener + stop_all ;; restart) - stop_listener - start_listener + stop_all + start_all ;; status) - status_listener + status_all ;; tail) mkdir -p "$(dirname "$LOG_FILE")" diff --git a/tests/test_anomaly.py b/tests/test_anomaly.py new file mode 100644 index 0000000..52a2dec --- /dev/null +++ b/tests/test_anomaly.py @@ -0,0 +1,42 @@ +import unittest + +from fgai.anomaly import anomaly_summary, detect_source_anomalies +from fgai.logs import parse_log_line + + +class AnomalyTests(unittest.TestCase): + def test_scores_repeated_utm_public_source_as_anomaly(self): + events = [ + parse_log_line(f'type=traffic srcip=10.0.0.{i} dstip=1.1.1.1 service=https action=accept sentbyte=100 rcvdbyte=100') + for i in range(1, 8) + ] + events.extend( + parse_log_line( + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https ' + 'action=blocked severity=critical sentbyte=0 rcvdbyte=0' + ) + for _ in range(5) + ) + + findings = detect_source_anomalies(events) + + self.assertEqual(findings[0].subject, "8.8.8.8") + self.assertGreaterEqual(findings[0].score, 60) + self.assertIn(findings[0].severity, {"high", "critical"}) + + def test_summary_counts_severities(self): + events = [ + parse_log_line( + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https ' + 'action=blocked severity=critical' + ) + for _ in range(5) + ] + + summary = anomaly_summary(detect_source_anomalies(events)) + + self.assertEqual(summary["total"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_monitor.py b/tests/test_monitor.py new file mode 100644 index 0000000..8d3f8e7 --- /dev/null +++ b/tests/test_monitor.py @@ -0,0 +1,38 @@ +import tempfile +import unittest +from pathlib import Path + +from fgai.monitor import build_status, write_status + + +class MonitorTests(unittest.TestCase): + def test_build_status_from_log_file(self): + with tempfile.TemporaryDirectory() as tmp: + log_path = Path(tmp) / "fg.log" + log_path.write_text( + "\n".join( + [ + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', + ] + ), + encoding="utf-8", + ) + + status = build_status(str(log_path)) + + self.assertEqual(status["summary"]["total"], 3) + self.assertGreaterEqual(len(status["anomalies"]), 1) + + def test_write_status_creates_parent_directory(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "state" / "status.json" + + write_status({"ok": True}, str(output)) + + self.assertTrue(output.exists()) + + +if __name__ == "__main__": + unittest.main()