diff --git a/src/fgai/anomaly.py b/src/fgai/anomaly.py index 9d53931..b1ce3e2 100644 --- a/src/fgai/anomaly.py +++ b/src/fgai/anomaly.py @@ -5,7 +5,7 @@ from datetime import datetime from statistics import mean, pstdev from .entities import is_public_ip as _is_public_ip -from .feedback import SUPPRESSING_STATUSES +from .feedback import SUPPRESSING_STATUSES, feedback_matches from .logs import THREAT_ACTIONS, event_score, is_utm_event from .models import AnomalyFinding, LogEvent @@ -69,7 +69,7 @@ def _rate_per_minute(events: list[LogEvent]) -> tuple[float | None, float]: def detect_source_anomalies( - events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None + events: list[LogEvent], *, limit: int = 20, baselines: dict[str, dict[str, object]] | None = None, field_deviations: dict[str, list[dict[str, object]]] | None = None, feedback: list[dict[str, object]] | None = None ) -> list[AnomalyFinding]: baselines = baselines or {} field_deviations = field_deviations or {} @@ -105,6 +105,13 @@ def detect_source_anomalies( findings: list[AnomalyFinding] = [] for src_ip, src_events in by_src.items(): + if feedback: + suppression = next( + (item for item in feedback if feedback_matches(item, entity=src_ip, stream_id="", field="", value="")), + None, + ) + if suppression and suppression.get("status") in SUPPRESSING_STATUSES: + continue event_count = len(src_events) distinct_dst = len({event.fields.get("dstip") for event in src_events if event.fields.get("dstip")}) distinct_services = len({event.fields.get("service") for event in src_events if event.fields.get("service")}) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 0a25802..82397b7 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -190,7 +190,7 @@ HTML = """ -

Events and Anomalies

Baseline and Stream Health

Operator Guidance

Waiting for monitor data.

Correlation Map

Waiting for correlated entities.

Investigation Incidents

Anomalies

Recommendations

AI Assessment

LLM assessment disabled.
+

Events and Anomalies

Baseline and Stream Health

Operator Guidance

Waiting for monitor data.

Correlation Map

Waiting for correlated entities.

Investigation Incidents

Anomalies

Recommendations

AI Assessment

LLM assessment disabled.

Triage Queue

Field Baseline Deviations

Active Feedback Rules

Related Activity Across Sources
Block Candidates
Threat Intelligence
Policy Findings

Diagnostics

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

@@ -225,6 +225,7 @@ const uiCache = {correlations: [], fieldRows: [], correlationHitboxes: [], selec let pendingReview = null; let lastReviewAuthor = ''; let selectedDeviationKeys = new Set(); +let selectedAnomalyKeys = new Set(); function deviationKey(r) { return `${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; } let openDetailIds = new Set(['diag-stream-coverage', 'diag-profile-readiness']); function collapsibleSection(id, title, innerHtml) { @@ -695,7 +696,10 @@ async function refresh() { ], 'profile-suggestions') : '

No missing stream profiles. Enable "show existing profiles" to inspect already configured profiles.

'; const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '
') : esc(llm.error || 'LLM assessment disabled or waiting for first run.'); document.getElementById('llmAssessment').innerHTML = `
Status: ${esc(llm.status || (llmEnabled ? 'starting' : 'disabled'))}

${llmText}

`; - document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [ + const anomalyRows = data.anomalies || []; + document.getElementById('anomalySummary').textContent = `${anomalyRows.length} shown.`; + document.getElementById('anomalies').innerHTML = table(anomalyRows, [ + {label:'', render:r => ``}, {label:'Source', key:'subject', render:r => entityCell(r, 'subject')}, {label:'Score', key:'score'}, {label:'Severity', render:r => `${esc(r.severity)}`}, @@ -707,6 +711,56 @@ async function refresh() { }}, {label:'Reasons', render:r => esc((r.reasons || []).join('; '))} ], 'anomalies'); + const anomalyBulkBar = document.getElementById('anomalyBulkBar'); + const anomalyBulkWarning = document.getElementById('anomalyBulkWarning'); + const updateAnomalyBulkBar = () => { + const count = selectedAnomalyKeys.size; + anomalyBulkBar.hidden = count === 0; + document.getElementById('anomalyBulkCount').textContent = `${count} selected`; + }; + document.querySelectorAll('.anomaly-select').forEach(box => box.addEventListener('change', () => { + if (box.checked) selectedAnomalyKeys.add(box.dataset.key); else selectedAnomalyKeys.delete(box.dataset.key); + updateAnomalyBulkBar(); + })); + const selectAllAnomaliesBox = document.getElementById('selectAllAnomalies'); + selectAllAnomaliesBox.checked = anomalyRows.length > 0 && anomalyRows.every(row => selectedAnomalyKeys.has(row.subject)); + selectAllAnomaliesBox.onchange = () => { + anomalyRows.forEach(row => { + if (selectAllAnomaliesBox.checked) selectedAnomalyKeys.add(row.subject); else selectedAnomalyKeys.delete(row.subject); + }); + refresh(); + }; + updateAnomalyBulkBar(); + document.getElementById('anomalyBulkClear').onclick = () => { + selectedAnomalyKeys.clear(); + refresh(); + }; + document.getElementById('anomalyBulkSubmit').onclick = async () => { + const selectedSubjects = [...selectedAnomalyKeys]; + if (!selectedSubjects.length) return; + const days = Number(document.getElementById('anomalyBulkExpiryDays').value || '0'); + if (days <= 0) { + anomalyBulkWarning.hidden = false; + return; + } + anomalyBulkWarning.hidden = true; + const status = document.getElementById('anomalyBulkStatus').value; + const note = document.getElementById('anomalyBulkNote').value || ''; + const author = document.getElementById('anomalyBulkAuthor').value || ''; + const expiresAt = Math.floor(Date.now() / 1000) + days * 86400; + if (!confirm(`This will suppress ALL findings for ${selectedSubjects.length} source(s) — ${selectedSubjects.join(', ')} — across every stream and field until they expire in ${days} day(s). Continue?`)) return; + const items = selectedSubjects.map(subject => ({status, scope: 'entity', entity: subject, note, author, expires_at: expiresAt})); + const response = await fetch('/api/feedback/bulk', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({items})}); + const result = await response.json().catch(() => ({})); + if (response.ok) { + const errorCount = (result.errors || []).length; + document.getElementById('anomalyNotice').textContent = errorCount ? `Applied to ${(result.added||[]).length} of ${items.length} selected sources; ${errorCount} skipped.` : `Applied to ${items.length} selected source(s).`; + selectedAnomalyKeys.clear(); + refresh(); + } else { + document.getElementById('anomalyNotice').textContent = `Could not apply bulk review: ${result.error || 'invalid request'}.`; + } + }; document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [ {label:'Subject', key:'subject', render:r => entityCell(r, 'subject')}, {label:'Score', key:'score'}, diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index 9eb3bb4..3ff1178 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -526,7 +526,7 @@ def build_status( deviation["score"] = 0 generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3) feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values) - anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations) + anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations, feedback=feedback) baseline_events = baseline.ingest(events) if baseline else 0 profile_baseline_fields = ( baseline.ingest_profile_fields( diff --git a/tests/test_anomaly.py b/tests/test_anomaly.py index 6e860d4..bd76e8c 100644 --- a/tests/test_anomaly.py +++ b/tests/test_anomaly.py @@ -24,6 +24,62 @@ class AnomalyTests(unittest.TestCase): self.assertGreaterEqual(findings[0].score, 60) self.assertIn(findings[0].severity, {"high", "critical"}) + def test_entity_scope_feedback_suppresses_anomaly_even_without_field_deviations(self): + events = [ + parse_log_line(f'type=traffic srcip=10.0.0.{i} dstip=1.1.1.1 service=https action=accept sentbyte=100 rcvdbyte=100') + for i in range(1, 8) + ] + events.extend( + parse_log_line( + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https ' + 'action=blocked severity=critical sentbyte=0 rcvdbyte=0' + ) + for _ in range(5) + ) + # Confirm it would normally fire (no field_deviations involved at all). + self.assertTrue(detect_source_anomalies(events)) + + feedback = [{"scope": "entity", "entity": "8.8.8.8", "status": "false_positive"}] + findings = detect_source_anomalies(events, feedback=feedback) + + self.assertEqual(findings, []) + + def test_entity_scope_feedback_does_not_affect_other_sources(self): + events = [ + parse_log_line( + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https ' + 'action=blocked severity=critical sentbyte=0 rcvdbyte=0' + ) + for _ in range(5) + ] + events.extend( + parse_log_line( + 'type=utm subtype=ips srcip=9.9.9.9 dstip=10.0.0.11 service=https ' + 'action=blocked severity=critical sentbyte=0 rcvdbyte=0' + ) + for _ in range(5) + ) + + feedback = [{"scope": "entity", "entity": "8.8.8.8", "status": "false_positive"}] + findings = detect_source_anomalies(events, feedback=feedback) + + self.assertEqual([finding.subject for finding in findings], ["9.9.9.9"]) + + def test_missing_feedback_preserves_default_behavior(self): + events = [ + parse_log_line( + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https ' + 'action=blocked severity=critical sentbyte=0 rcvdbyte=0' + ) + for _ in range(5) + ] + + with_none = detect_source_anomalies(events, feedback=None) + without_param = detect_source_anomalies(events) + + self.assertEqual([f.subject for f in with_none], [f.subject for f in without_param]) + self.assertTrue(with_none) + def test_summary_counts_severities(self): events = [ parse_log_line( diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 5db0cd5..5d5fe98 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -94,6 +94,21 @@ class DashboardTests(unittest.TestCase): self.assertIn("openDetailIds", HTML) self.assertIn("addEventListener('toggle'", HTML) + def test_dashboard_has_anomaly_bulk_bar(self): + self.assertIn('id="anomalyBulkBar"', HTML) + self.assertIn('anomaly-select', HTML) + self.assertIn('id="selectAllAnomalies"', HTML) + self.assertIn('id="anomalyBulkSubmit"', HTML) + + def test_dashboard_anomaly_bulk_requires_expiry_and_only_ever_posts_entity_scope(self): + self.assertIn('at least 1 day', HTML) + start = HTML.index("getElementById('anomalyBulkSubmit')") + end = HTML.index('};', start) + submit_handler = HTML[start:end] + self.assertIn("scope: 'entity'", submit_handler) + self.assertIn('days <= 0', submit_handler) + self.assertIn('confirm(', submit_handler) + def test_dashboard_field_deviations_are_collapsible_and_open_by_default(self): start = HTML.index("Field Baseline Deviations") preceding = HTML[max(0, start - 80):start] diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 380bcdc..4b8be39 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -5,6 +5,7 @@ from pathlib import Path from unittest.mock import patch from fgai.config import ConfigStore +from fgai.feedback import FeedbackStore from fgai.history import StatusSnapshotStore from fgai.history import FieldDiscoveryStore from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status @@ -38,6 +39,27 @@ class MonitorTests(unittest.TestCase): self.assertFalse(status["status_cache"]["served_from_cache"]) self.assertGreaterEqual(len(status["anomalies"]), 1) + def test_entity_scope_feedback_suppresses_anomaly_in_build_status(self): + with tempfile.TemporaryDirectory() as tmp: + log_path = Path(tmp) / "fg.log" + log_path.write_text( + "\n".join( + [ + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', + 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', + ] + ), + encoding="utf-8", + ) + feedback_store = FeedbackStore(str(Path(tmp) / "feedback.json")) + feedback_store.add({"scope": "entity", "entity": "8.8.8.8", "status": "false_positive", "expires_at": 9999999999}) + + with patch("fgai.monitor.FeedbackStore", return_value=feedback_store): + status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json")) + + self.assertNotIn("8.8.8.8", [item["subject"] for item in status["anomalies"]]) + def test_write_status_creates_parent_directory(self): with tempfile.TemporaryDirectory() as tmp: output = Path(tmp) / "state" / "status.json"