fix anon
This commit is contained in:
@@ -5,7 +5,7 @@ from datetime import datetime
|
|||||||
from statistics import mean, pstdev
|
from statistics import mean, pstdev
|
||||||
|
|
||||||
from .entities import is_public_ip as _is_public_ip
|
from .entities import is_public_ip as _is_public_ip
|
||||||
from .feedback import SUPPRESSING_STATUSES
|
from .feedback import SUPPRESSING_STATUSES, feedback_matches
|
||||||
from .logs import THREAT_ACTIONS, event_score, is_utm_event
|
from .logs import THREAT_ACTIONS, event_score, is_utm_event
|
||||||
from .models import AnomalyFinding, LogEvent
|
from .models import AnomalyFinding, LogEvent
|
||||||
|
|
||||||
@@ -69,7 +69,7 @@ def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]:
|
|||||||
|
|
||||||
|
|
||||||
def detect_source_anomalies(
|
def detect_source_anomalies(
|
||||||
events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None
|
events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None, feedback: list[dict[str, object]] | None = None
|
||||||
) -> list[AnomalyFinding]:
|
) -> list[AnomalyFinding]:
|
||||||
baselines = baselines or {}
|
baselines = baselines or {}
|
||||||
field_deviations = field_deviations or {}
|
field_deviations = field_deviations or {}
|
||||||
@@ -105,6 +105,13 @@ def detect_source_anomalies(
|
|||||||
|
|
||||||
findings: list[AnomalyFinding] = []
|
findings: list[AnomalyFinding] = []
|
||||||
for src_ip, src_events in by_src.items():
|
for src_ip, src_events in by_src.items():
|
||||||
|
if feedback:
|
||||||
|
suppression = next(
|
||||||
|
(item for item in feedback if feedback_matches(item, entity=src_ip, stream_id="", field="", value="")),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if suppression and suppression.get("status") in SUPPRESSING_STATUSES:
|
||||||
|
continue
|
||||||
event_count = len(src_events)
|
event_count = len(src_events)
|
||||||
distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
|
distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")})
|
||||||
distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")})
|
distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")})
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -526,7 +526,7 @@ def build_status(
|
|||||||
deviation["score"] = 0
|
deviation["score"] = 0
|
||||||
generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3)
|
generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3)
|
||||||
feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values)
|
feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values)
|
||||||
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
|
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations, feedback=feedback)
|
||||||
baseline_events = baseline.ingest(events) if baseline else 0
|
baseline_events = baseline.ingest(events) if baseline else 0
|
||||||
profile_baseline_fields = (
|
profile_baseline_fields = (
|
||||||
baseline.ingest_profile_fields(
|
baseline.ingest_profile_fields(
|
||||||
|
|||||||
@@ -24,6 +24,62 @@ class AnomalyTests(unittest.TestCase):
|
|||||||
self.assertGreaterEqual(findings[0].score, 60)
|
self.assertGreaterEqual(findings[0].score, 60)
|
||||||
self.assertIn(findings[0].severity, {"high", "critical"})
|
self.assertIn(findings[0].severity, {"high", "critical"})
|
||||||
|
|
||||||
|
def test_entity_scope_feedback_suppresses_anomaly_even_without_field_deviations(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line(f'type=traffic srcip=10.0.0.{i} dstip=1.1.1.1 service=https action=accept sentbyte=100 rcvdbyte=100')
|
||||||
|
for i in range(1, 8)
|
||||||
|
]
|
||||||
|
events.extend(
|
||||||
|
parse_log_line(
|
||||||
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https '
|
||||||
|
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
|
||||||
|
)
|
||||||
|
for _ in range(5)
|
||||||
|
)
|
||||||
|
# Confirm it would normally fire (no field_deviations involved at all).
|
||||||
|
self.assertTrue(detect_source_anomalies(events))
|
||||||
|
|
||||||
|
feedback = [{"scope": "entity", "entity": "8.8.8.8", "status": "false_positive"}]
|
||||||
|
findings = detect_source_anomalies(events, feedback=feedback)
|
||||||
|
|
||||||
|
self.assertEqual(findings, [])
|
||||||
|
|
||||||
|
def test_entity_scope_feedback_does_not_affect_other_sources(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line(
|
||||||
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https '
|
||||||
|
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
|
||||||
|
)
|
||||||
|
for _ in range(5)
|
||||||
|
]
|
||||||
|
events.extend(
|
||||||
|
parse_log_line(
|
||||||
|
'type=utm subtype=ips srcip=9.9.9.9 dstip=10.0.0.11 service=https '
|
||||||
|
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
|
||||||
|
)
|
||||||
|
for _ in range(5)
|
||||||
|
)
|
||||||
|
|
||||||
|
feedback = [{"scope": "entity", "entity": "8.8.8.8", "status": "false_positive"}]
|
||||||
|
findings = detect_source_anomalies(events, feedback=feedback)
|
||||||
|
|
||||||
|
self.assertEqual([finding.subject for finding in findings], ["9.9.9.9"])
|
||||||
|
|
||||||
|
def test_missing_feedback_preserves_default_behavior(self):
|
||||||
|
events = [
|
||||||
|
parse_log_line(
|
||||||
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https '
|
||||||
|
'action=blocked severity=critical sentbyte=0 rcvdbyte=0'
|
||||||
|
)
|
||||||
|
for _ in range(5)
|
||||||
|
]
|
||||||
|
|
||||||
|
with_none = detect_source_anomalies(events, feedback=None)
|
||||||
|
without_param = detect_source_anomalies(events)
|
||||||
|
|
||||||
|
self.assertEqual([f.subject for f in with_none], [f.subject for f in without_param])
|
||||||
|
self.assertTrue(with_none)
|
||||||
|
|
||||||
def test_summary_counts_severities(self):
|
def test_summary_counts_severities(self):
|
||||||
events = [
|
events = [
|
||||||
parse_log_line(
|
parse_log_line(
|
||||||
|
|||||||
@@ -94,6 +94,21 @@ class DashboardTests(unittest.TestCase):
|
|||||||
self.assertIn("openDetailIds", HTML)
|
self.assertIn("openDetailIds", HTML)
|
||||||
self.assertIn("addEventListener('toggle'", HTML)
|
self.assertIn("addEventListener('toggle'", HTML)
|
||||||
|
|
||||||
|
def test_dashboard_has_anomaly_bulk_bar(self):
|
||||||
|
self.assertIn('id="anomalyBulkBar"', HTML)
|
||||||
|
self.assertIn('anomaly-select', HTML)
|
||||||
|
self.assertIn('id="selectAllAnomalies"', HTML)
|
||||||
|
self.assertIn('id="anomalyBulkSubmit"', HTML)
|
||||||
|
|
||||||
|
def test_dashboard_anomaly_bulk_requires_expiry_and_only_ever_posts_entity_scope(self):
|
||||||
|
self.assertIn('at least 1 day', HTML)
|
||||||
|
start = HTML.index("getElementById('anomalyBulkSubmit')")
|
||||||
|
end = HTML.index('};', start)
|
||||||
|
submit_handler = HTML[start:end]
|
||||||
|
self.assertIn("scope: 'entity'", submit_handler)
|
||||||
|
self.assertIn('days <= 0', submit_handler)
|
||||||
|
self.assertIn('confirm(', submit_handler)
|
||||||
|
|
||||||
def test_dashboard_field_deviations_are_collapsible_and_open_by_default(self):
|
def test_dashboard_field_deviations_are_collapsible_and_open_by_default(self):
|
||||||
start = HTML.index("Field Baseline Deviations")
|
start = HTML.index("Field Baseline Deviations")
|
||||||
preceding = HTML[max(0, start - 80):start]
|
preceding = HTML[max(0, start - 80):start]
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from pathlib import Path
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from fgai.config import ConfigStore
|
from fgai.config import ConfigStore
|
||||||
|
from fgai.feedback import FeedbackStore
|
||||||
from fgai.history import StatusSnapshotStore
|
from fgai.history import StatusSnapshotStore
|
||||||
from fgai.history import FieldDiscoveryStore
|
from fgai.history import FieldDiscoveryStore
|
||||||
from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
|
from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
|
||||||
@@ -38,6 +39,27 @@ class MonitorTests(unittest.TestCase):
|
|||||||
self.assertFalse(status["status_cache"]["served_from_cache"])
|
self.assertFalse(status["status_cache"]["served_from_cache"])
|
||||||
self.assertGreaterEqual(len(status["anomalies"]), 1)
|
self.assertGreaterEqual(len(status["anomalies"]), 1)
|
||||||
|
|
||||||
|
def test_entity_scope_feedback_suppresses_anomaly_in_build_status(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
log_path = Path(tmp) / "fg.log"
|
||||||
|
log_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
||||||
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
||||||
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
feedback_store = FeedbackStore(str(Path(tmp) / "feedback.json"))
|
||||||
|
feedback_store.add({"scope": "entity", "entity": "8.8.8.8", "status": "false_positive", "expires_at": 9999999999})
|
||||||
|
|
||||||
|
with patch("fgai.monitor.FeedbackStore", return_value=feedback_store):
|
||||||
|
status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json"))
|
||||||
|
|
||||||
|
self.assertNotIn("8.8.8.8", [item["subject"] for item in status["anomalies"]])
|
||||||
|
|
||||||
def test_write_status_creates_parent_directory(self):
|
def test_write_status_creates_parent_directory(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
output = Path(tmp) / "state" / "status.json"
|
output = Path(tmp) / "state" / "status.json"
|
||||||
|
|||||||
Reference in New Issue
Block a user