From 5bb381a0636589474191f34abe978e26bff5acca8592cffc3cce8b886ee36e91 Mon Sep 17 00:00:00 2001 From: larssand Date: Sun, 21 Jun 2026 15:19:24 +0200 Subject: [PATCH] fix ui --- src/fgai/anomaly.py | 16 +++++++++++- src/fgai/baseline.py | 20 +++++++++++++-- src/fgai/dashboard.py | 53 ++++++++++++++++++++++++++++------------ src/fgai/monitor.py | 6 ++++- src/fgai/threat_intel.py | 5 ++++ tests/test_baseline.py | 1 + 6 files changed, 81 insertions(+), 20 deletions(-) diff --git a/src/fgai/anomaly.py b/src/fgai/anomaly.py index ce4b072..9024a19 100644 --- a/src/fgai/anomaly.py +++ b/src/fgai/anomaly.py @@ -75,7 +75,7 @@ def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]: def detect_source_anomalies( - events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, float | int]] | None = None + events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None ) -> list[AnomalyFinding]: baselines = baselines or {} by_src: dict[str, list[LogEvent]] = defaultdict(list) @@ -200,6 +200,19 @@ def detect_source_anomalies( score += min(15, distinct_dst_ports) reasons.append(f"many destination ports contacted ({distinct_dst_ports})") + baseline = baselines.get(src_ip) + if baseline: + known_destinations = set(baseline.get("known_destinations", [])) + known_ports = set(baseline.get("known_destination_ports", [])) + new_destinations = {event.dst_ip for event in src_events if event.dst_ip and event.dst_ip not in known_destinations} + new_ports = {event.fields.get("dstport") for event in src_events if event.fields.get("dstport") and event.fields.get("dstport") not in known_ports} + if len(known_destinations) >= 5 and len(new_destinations) >= 3: + score += min(15, 5 + len(new_destinations)) + reasons.append(f"new destinations relative to historical baseline ({len(new_destinations)})") + if len(known_ports) >= 3 and len(new_ports) >= 2: + score += min(12, 4 + len(new_ports)) + reasons.append(f"new destination ports relative to historical baseline ({len(new_ports)})") + if _is_public_ip(src_ip) and (utm_count or deny_count >= 10): score += 10 reasons.append("public source with repeated security-relevant events") @@ -232,6 +245,7 @@ def detect_source_anomalies( "distinct_dst_ports": distinct_dst_ports, "policy_count": len(policies), "implicit_deny_events": implicit_deny_count, + "baseline_ready": int(src_ip in baselines), }, ) ) diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 8278768..282237e 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -35,6 +35,10 @@ class BaselineStore: denies integer not null, utm integer not null, primary key (source_ip, bucket_start) ); + create table if not exists source_values ( + source_ip text not null, kind text not null, value text not null, + seen_count integer not null, primary key (source_ip, kind, value) + ); """ ) @@ -59,6 +63,13 @@ class BaselineStore: values[2] += _number(event.fields.get("hitcount")) values[3] += int(event.action in THREAT_ACTIONS) values[4] += int(is_utm_event(event)) + for kind, value in (("destination", event.dst_ip), ("destination_port", event.fields.get("dstport"))): + if value: + connection.execute( + """insert into source_values values (?, ?, ?, 1) + on conflict(source_ip, kind, value) do update set seen_count=seen_count+1""", + (event.src_ip, kind, value), + ) inserted += 1 for (source_ip, bucket), values in pending.items(): connection.execute( @@ -70,8 +81,8 @@ class BaselineStore: ) return inserted - def profiles(self, source_ips: set[str]) -> dict[str, dict[str, float | int]]: - profiles: dict[str, dict[str, float | int]] = {} + def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]: + profiles: dict[str, dict[str, object]] = {} with self._connect() as connection: for source_ip in source_ips: rows = connection.execute( @@ -83,9 +94,14 @@ class BaselineStore: continue rates = [row[0] * 60 / self.bucket_seconds for row in rows] hit_rates = [row[2] * 60 / self.bucket_seconds for row in rows] + known = connection.execute( + "select kind, value from source_values where source_ip=?", (source_ip,) + ).fetchall() profiles[source_ip] = { "samples": len(rows), "event_rate_mean": mean(rates), "event_rate_stddev": pstdev(rates) or 1.0, "hitcount_rate_mean": mean(hit_rates), "hitcount_rate_stddev": pstdev(hit_rates) or 1.0, + "known_destinations": [value for kind, value in known if kind == "destination"], + "known_destination_ports": [value for kind, value in known if kind == "destination_port"], } return profiles diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 3edf67b..6cabbda 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -14,14 +14,14 @@ HTML = """ -

fgAI Monitor

+

fgAI Monitor

FortiGate AI/ML Analyzer @@ -46,13 +56,10 @@ HTML = """

Live Status

Waiting for monitor data.
-

AI Assessment

LLM assessment disabled.
-

Anomalies

-

Recommendations

-

Block Candidates

-

Threat Intelligence

-

Policy Findings

-

Diagnostics

+ +

AI Assessment

LLM assessment disabled.

Anomalies

Recommendations

+

Block Candidates

Threat Intelligence

Policy Findings

+

Diagnostics

diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index d25d73e..c103960 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -11,7 +11,7 @@ from .logs import local_in_failures, read_events, summarize_events, top_field_va from .mitigation import parse_allowlist, suggest_block_candidates from .policies import audit_policies, read_policies from .recommendations import build_recommendations -from .threat_intel import enrich_ips, is_public_ip +from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip def build_status( @@ -37,6 +37,7 @@ def build_status( } ) reputation = enrich_ips(intel_ips, limit=25) + threat_intel_status = ThreatIntelClient().status() recommendations = build_recommendations(events, anomalies, reputation) block_candidates = suggest_block_candidates( events, @@ -60,8 +61,11 @@ def build_status( "summary": summarize_events(events), "anomaly_summary": anomaly_summary(anomalies), "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events}, + "capabilities": {"threat_intel": threat_intel_status}, "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), + "top_destination_ips": top_field_values(events, "dstip", limit=10), + "top_policy_ids": top_field_values(events, "policyid", limit=10), "top_destination_ports": top_field_values(events, "dstport", limit=10), "top_source_ports": top_field_values(events, "srcport", limit=10), "top_services": top_field_values(events, "service", limit=10), diff --git a/src/fgai/threat_intel.py b/src/fgai/threat_intel.py index e2a0eae..df45e90 100644 --- a/src/fgai/threat_intel.py +++ b/src/fgai/threat_intel.py @@ -29,6 +29,11 @@ class ThreatIntelClient: self.ttl_seconds = ttl_seconds self.cache = self._read_cache() + def status(self) -> dict[str, object]: + provider = self._select_provider() + has_key = bool(self.abuseipdb_key if provider == "abuseipdb" else self.virustotal_key) + return {"enabled": self.enabled, "provider": provider, "configured": has_key} + def _read_cache(self) -> dict[str, dict[str, object]]: if not self.cache_path.exists(): return {} diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 08f42b1..8d925e6 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -14,3 +14,4 @@ class BaselineTests(unittest.TestCase): store.ingest([parse_log_line(f"srcip=10.0.0.1 dstport={1000 + index} hitcount=2 sentbyte=5")], observed_at=1_700_000_000 + index * 300) profiles = store.profiles({"10.0.0.1"}) self.assertEqual(profiles["10.0.0.1"]["samples"], 12) + self.assertIn("1000", profiles["10.0.0.1"]["known_destination_ports"])