This commit is contained in:
larssand
2026-09-23 13:32:09 +02:00
parent bf1109be81
commit a3f545f1b0
6 changed files with 159 additions and 5 deletions

View File

@@ -5,7 +5,7 @@ from datetime import datetime
from statistics import mean, pstdev from statistics import mean, pstdev
from .entities import is_public_ip as _is_public_ip from .entities import is_public_ip as _is_public_ip
from .feedback import SUPPRESSING_STATUSES from .feedback import SUPPRESSING_STATUSES, feedback_matches
from .logs import THREAT_ACTIONS, event_score, is_utm_event from .logs import THREAT_ACTIONS, event_score, is_utm_event
from .models import AnomalyFinding, LogEvent from .models import AnomalyFinding, LogEvent
@@ -69,7 +69,7 @@ def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]:
def detect_source_anomalies( def detect_source_anomalies(
events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None, feedback: list[dict[str, object]] | None = None
) -> list[AnomalyFinding]: ) -> list[AnomalyFinding]:
baselines = baselines or {} baselines = baselines or {}
field_deviations = field_deviations or {} field_deviations = field_deviations or {}
@@ -105,6 +105,13 @@ def detect_source_anomalies(
findings: list[AnomalyFinding] = [] findings: list[AnomalyFinding] = []
for src_ip, src_events in by_src.items(): for src_ip, src_events in by_src.items():
if feedback:
suppression = next(
(item for item in feedback if feedback_matches(item, entity=src_ip, stream_id="", field="", value="")),
None,
)
if suppression and suppression.get("status") in SUPPRESSING_STATUSES:
continue
event_count = len(src_events) event_count = len(src_events)
distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")}) distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")}) distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")})

File diff suppressed because one or more lines are too long

View File

@@ -526,7 +526,7 @@ def build_status(
deviation["score"] = 0 deviation["score"] = 0
generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3) generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3)
feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values) feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values)
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations) anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations, feedback=feedback)
baseline_events = baseline.ingest(events) if baseline else 0 baseline_events = baseline.ingest(events) if baseline else 0
profile_baseline_fields = ( profile_baseline_fields = (
baseline.ingest_profile_fields( baseline.ingest_profile_fields(

View File

@@ -24,6 +24,62 @@ class AnomalyTests(unittest.TestCase):
self.assertGreaterEqual(findings[0].score, 60) self.assertGreaterEqual(findings[0].score, 60)
self.assertIn(findings[0].severity, {"high", "critical"}) self.assertIn(findings[0].severity, {"high", "critical"})
def test_entity_scope_feedback_suppresses_anomaly_even_without_field_deviations(self):
events = [
parse_log_line(f'type=traffic srcip=10.0.0.{i} dstip=1.1.1.1 service=https action=accept sentbyte=100 rcvdbyte=100')
for i in range(1, 8)
]
events.extend(
parse_log_line(
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https '
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
)
for _ in range(5)
)
# Confirm it would normally fire (no field_deviations involved at all).
self.assertTrue(detect_source_anomalies(events))
feedback = [{"scope": "entity", "entity": "8.8.8.8", "status": "false_positive"}]
findings = detect_source_anomalies(events, feedback=feedback)
self.assertEqual(findings, [])
def test_entity_scope_feedback_does_not_affect_other_sources(self):
events = [
parse_log_line(
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https '
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
)
for _ in range(5)
]
events.extend(
parse_log_line(
'type=utm subtype=ips srcip=9.9.9.9 dstip=10.0.0.11 service=https '
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
)
for _ in range(5)
)
feedback = [{"scope": "entity", "entity": "8.8.8.8", "status": "false_positive"}]
findings = detect_source_anomalies(events, feedback=feedback)
self.assertEqual([finding.subject for finding in findings], ["9.9.9.9"])
def test_missing_feedback_preserves_default_behavior(self):
events = [
parse_log_line(
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https '
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
)
for _ in range(5)
]
with_none = detect_source_anomalies(events, feedback=None)
without_param = detect_source_anomalies(events)
self.assertEqual([f.subject for f in with_none], [f.subject for f in without_param])
self.assertTrue(with_none)
def test_summary_counts_severities(self): def test_summary_counts_severities(self):
events = [ events = [
parse_log_line( parse_log_line(

View File

@@ -94,6 +94,21 @@ class DashboardTests(unittest.TestCase):
self.assertIn("openDetailIds", HTML) self.assertIn("openDetailIds", HTML)
self.assertIn("addEventListener('toggle'", HTML) self.assertIn("addEventListener('toggle'", HTML)
def test_dashboard_has_anomaly_bulk_bar(self):
self.assertIn('id="anomalyBulkBar"', HTML)
self.assertIn('anomaly-select', HTML)
self.assertIn('id="selectAllAnomalies"', HTML)
self.assertIn('id="anomalyBulkSubmit"', HTML)
def test_dashboard_anomaly_bulk_requires_expiry_and_only_ever_posts_entity_scope(self):
self.assertIn('at least 1 day', HTML)
start = HTML.index("getElementById('anomalyBulkSubmit')")
end = HTML.index('};', start)
submit_handler = HTML[start:end]
self.assertIn("scope: 'entity'", submit_handler)
self.assertIn('days <= 0', submit_handler)
self.assertIn('confirm(', submit_handler)
def test_dashboard_field_deviations_are_collapsible_and_open_by_default(self): def test_dashboard_field_deviations_are_collapsible_and_open_by_default(self):
start = HTML.index("Field Baseline Deviations") start = HTML.index("Field Baseline Deviations")
preceding = HTML[max(0, start - 80):start] preceding = HTML[max(0, start - 80):start]

View File

@@ -5,6 +5,7 @@ from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
from fgai.config import ConfigStore from fgai.config import ConfigStore
from fgai.feedback import FeedbackStore
from fgai.history import StatusSnapshotStore from fgai.history import StatusSnapshotStore
from fgai.history import FieldDiscoveryStore from fgai.history import FieldDiscoveryStore
from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
@@ -38,6 +39,27 @@ class MonitorTests(unittest.TestCase):
self.assertFalse(status["status_cache"]["served_from_cache"]) self.assertFalse(status["status_cache"]["served_from_cache"])
self.assertGreaterEqual(len(status["anomalies"]), 1) self.assertGreaterEqual(len(status["anomalies"]), 1)
def test_entity_scope_feedback_suppresses_anomaly_in_build_status(self):
with tempfile.TemporaryDirectory() as tmp:
log_path = Path(tmp) / "fg.log"
log_path.write_text(
"\n".join(
[
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
]
),
encoding="utf-8",
)
feedback_store = FeedbackStore(str(Path(tmp) / "feedback.json"))
feedback_store.add({"scope": "entity", "entity": "8.8.8.8", "status": "false_positive", "expires_at": 9999999999})
with patch("fgai.monitor.FeedbackStore", return_value=feedback_store):
status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json"))
self.assertNotIn("8.8.8.8", [item["subject"] for item in status["anomalies"]])
def test_write_status_creates_parent_directory(self): def test_write_status_creates_parent_directory(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
output = Path(tmp) / "state" / "status.json" output = Path(tmp) / "state" / "status.json"