import tempfile import json import unittest from pathlib import Path from unittest.mock import patch from fgai.config import ConfigStore from fgai.feedback import FeedbackStore from fgai.history import StatusSnapshotStore from fgai.history import FieldDiscoveryStore from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status, _cluster_field_deviations class MonitorTests(unittest.TestCase): def test_monitor_cycle_timeout_keeps_large_buffer(self): self.assertEqual(monitor_cycle_timeout_seconds(120, 8), 600) self.assertEqual(monitor_cycle_timeout_seconds(240, 8), 600) self.assertEqual(monitor_cycle_timeout_seconds(900, 8), 1080) def test_build_status_from_log_file(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "fg.log" log_path.write_text( "\n".join( [ 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', ] ), encoding="utf-8", ) status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertEqual(status["summary"]["total"], 3) self.assertEqual(status["status_schema"], 2) self.assertFalse(status["stale"]) self.assertFalse(status["status_cache"]["served_from_cache"]) self.assertGreaterEqual(len(status["anomalies"]), 1) def test_cluster_field_deviations_dampens_large_identical_clusters(self): field_deviations = { f"10.0.0.{index}": [{"detector": "relationship", "field": "client_ip->context", "score": 40}] for index in range(1, 6) } _cluster_field_deviations(field_deviations) for entity, deviations in field_deviations.items(): deviation = deviations[0] self.assertEqual(deviation["cluster_size"], 5) self.assertEqual(deviation["score"], 25) self.assertTrue(deviation["cluster_dampened"]) def test_cluster_field_deviations_leaves_small_clusters_untouched(self): field_deviations = { f"10.0.0.{index}": [{"detector": "relationship", "field": "client_ip->context", "score": 40}] for index in range(1, 4) } _cluster_field_deviations(field_deviations) for entity, deviations in field_deviations.items(): deviation = deviations[0] self.assertEqual(deviation["cluster_size"], 3) self.assertEqual(deviation["score"], 40) self.assertNotIn("cluster_dampened", deviation) def test_cluster_field_deviations_excludes_suppressed_entities_from_cluster_size(self): field_deviations = { f"10.0.0.{index}": [{"detector": "relationship", "field": "client_ip->context", "score": 40}] for index in range(1, 5) } field_deviations["10.0.0.99"] = [{"detector": "relationship", "field": "client_ip->context", "score": 40, "feedback": "false_positive"}] _cluster_field_deviations(field_deviations) self.assertEqual(field_deviations["10.0.0.1"][0]["cluster_size"], 4) self.assertNotIn("cluster_dampened", field_deviations["10.0.0.1"][0]) self.assertEqual(field_deviations["10.0.0.99"][0]["score"], 40) def test_cluster_field_deviations_dampen_large_clusters_in_build_status(self): with tempfile.TemporaryDirectory() as tmp: srcips = [f"203.0.113.{index}" for index in range(1, 6)] log_path = Path(tmp) / "fg.log" log_path.write_text( "\n".join( f"type=utm subtype=ips srcip={ip} dstip=10.0.0.10 service=https action=blocked severity=critical" for ip in srcips ), encoding="utf-8", ) canned_deviations = { ip: [{"detector": "relationship", "field": "client_ip->context", "stream_id": "utm", "score": 40, "reason": "new relationship: client_ip->context"}] for ip in srcips } with patch("fgai.monitor.BaselineStore.profile_deviations", return_value=canned_deviations): status = build_status(str(log_path), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json")) for ip in srcips: deviation = status["field_deviations"][ip][0] self.assertEqual(deviation["cluster_size"], 5) self.assertEqual(deviation["score"], 25) self.assertTrue(deviation["cluster_dampened"]) anomalies_by_subject = {item["subject"]: item for item in status["anomalies"]} self.assertIn("new relationship: client_ip->context", anomalies_by_subject[srcips[0]]["reasons"]) def test_entity_scope_feedback_suppresses_anomaly_in_build_status(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "fg.log" log_path.write_text( "\n".join( [ 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', 'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical', ] ), encoding="utf-8", ) feedback_store = FeedbackStore(str(Path(tmp) / "feedback.json")) feedback_store.add({"scope": "entity", "entity": "8.8.8.8", "status": "false_positive", "expires_at": 9999999999}) with patch("fgai.monitor.FeedbackStore", return_value=feedback_store): status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertNotIn("8.8.8.8", [item["subject"] for item in status["anomalies"]]) def test_write_status_creates_parent_directory(self): with tempfile.TemporaryDirectory() as tmp: output = Path(tmp) / "state" / "status.json" write_status({"ok": True}, str(output)) self.assertTrue(output.exists()) def test_write_refreshing_status_clears_stale_mcp_error(self): with tempfile.TemporaryDirectory() as tmp: output = Path(tmp) / "status.json" output.write_text( json.dumps( { "status_schema": 2, "stale": True, "stale_reason": "live_mcp_error", "capabilities": {"graylog_mcp": {"status": "error", "error": "old dns error", "events_fetched": 99, "aggregate_events": 1234, "fetch_mode": "aggregate"}}, } ), encoding="utf-8", ) write_refreshing_status( str(output), call_timeout_seconds=8, poll_timeout_seconds=120, runtime_values={ "log_source": "graylog_mcp", "graylog_streams": [{"id": "fw", "title": "Firewall", "enabled": True}], "graylog_stream_profiles": [{"stream_id": "fw", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}], }, ) status = json.loads(output.read_text(encoding="utf-8")) self.assertFalse(status["stale"]) self.assertEqual(status["stale_reason"], "") self.assertEqual(status["capabilities"]["graylog_mcp"]["status"], "refreshing") self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_status"], "error") self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_error"], "old dns error") self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_events_fetched"], 99) self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_aggregate_events"], 1234) self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_fetch_mode"], "aggregate") self.assertEqual(status["capabilities"]["graylog_mcp"]["call_timeout_seconds"], 8) self.assertEqual(status["capabilities"]["graylog_mcp"]["poll_timeout_seconds"], 120) self.assertEqual(status["configuration"]["enabled_streams"], 1) self.assertEqual(status["stream_coverage"][0]["stream_name"], "Firewall") self.assertEqual(status["stream_coverage"][0]["profile"], "Firewall profile") self.assertEqual(status["stream_coverage"][0]["entity_field"], "srcip") self.assertFalse(status["status_cache"]["served_from_cache"]) def test_write_refreshing_status_uses_last_good_cache(self): with tempfile.TemporaryDirectory() as tmp: output = Path(tmp) / "status.json" cache_path = str(Path(tmp) / "status-cache.sqlite3") output.write_text( json.dumps({"capabilities": {"graylog_mcp": {"status": "refreshing", "events_fetched": 0, "aggregate_events": 0}}}), encoding="utf-8", ) StatusSnapshotStore(cache_path).save( "last_good", { "summary": {"total": 1234}, "capabilities": {"graylog_mcp": {"status": "connected", "events_fetched": 99, "raw_events_fetched": 88, "aggregate_events": 1234, "poll_completed_at": 123456, "fetch_mode": "aggregate", "coverage_status": "complete_window"}}, "cross_source_correlations": [{"entity": "10.0.0.1", "entity_label": "host01 (10.0.0.1)"}], "llm_assessment": {"enabled": True, "status": "cached", "text": "previous assessment"}, }, ) write_refreshing_status(str(output), cache_path=cache_path) status = json.loads(output.read_text(encoding="utf-8")) mcp = status["capabilities"]["graylog_mcp"] self.assertEqual(status["summary"]["total"], 1234) self.assertEqual(mcp["status"], "refreshing") self.assertEqual(mcp["events_fetched"], 99) self.assertEqual(mcp["raw_events_fetched"], 88) self.assertEqual(mcp["aggregate_events"], 1234) self.assertEqual(mcp["previous_status"], "connected") self.assertGreater(mcp["poll_started_at"], 123456) self.assertEqual(mcp["previous_poll_completed_at"], 123456) self.assertTrue(status["llm_assessment"]["enabled"]) self.assertTrue(status["status_cache"]["served_from_cache"]) def test_profile_readiness_includes_stream_and_profile_names(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "graylog_streams": [{"id": "66fe", "title": "Fortigate", "enabled": True}], "graylog_stream_profiles": [{"stream_id": "66fe", "name": "66fe", "entity_field": "srcip", "categorical_fields": ["action"]}], } ), encoding="utf-8", ) with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[{"stream_id": "66fe", "field": "action", "buckets": 12, "ready": True}]): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertEqual(status["profile_readiness"][0]["stream_name"], "Fortigate") self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile") self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile") def test_shared_entity_baselines_lists_streams_pooled_by_entity_concept(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "graylog_streams": [ {"id": "firewall", "title": "Firewall", "enabled": True}, {"id": "iis", "title": "IIS", "enabled": True}, ], "graylog_stream_profiles": [ {"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}, {"stream_id": "iis", "name": "IIS profile", "entity_field": "client_ip", "categorical_fields": ["action"]}, ], } ), encoding="utf-8", ) with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertEqual(len(status["shared_entity_baselines"]), 1) pool = status["shared_entity_baselines"][0] self.assertEqual(pool["group"], "srcip") self.assertEqual(set(pool["streams"]), {"firewall", "iis"}) def test_detector_health_lists_known_detectors_with_no_data_by_default(self): with tempfile.TemporaryDirectory() as tmp: with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), incident_path=str(Path(tmp) / "incidents.json")) health_by_detector = {item["detector"]: item for item in status["detector_health"]} self.assertIn("auth_failure_burst", health_by_detector) self.assertIn("new_relationship", health_by_detector) self.assertEqual(health_by_detector["auth_failure_burst"]["learning_state"], "no_data") self.assertIsNone(health_by_detector["auth_failure_burst"]["precision"]) def test_detector_health_reflects_readiness_and_precision(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "graylog_streams": [{"id": "windows", "title": "Windows", "enabled": True}], "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "categorical_fields": ["action"], "detectors": {"auth_failure": {"enabled": True}}}], } ), encoding="utf-8", ) readiness_rows = [{"stream_id": "windows", "detector": "auth_failure", "buckets": 12, "age_days": 10, "training_days": 0, "ready": True}] precision_rows = {"auth_failure_burst": {"confirmed": 3, "false_positive": 1, "expected": 0, "precision": 0.75, "reviewed_total": 4}} with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]), \ patch("fgai.monitor.BaselineStore.detector_readiness", return_value=readiness_rows), \ patch("fgai.monitor.DetectorPrecisionStore.precision", return_value=precision_rows): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) row = next(item for item in status["detector_health"] if item["detector"] == "auth_failure_burst") self.assertEqual(row["learning_state"], "ready") self.assertEqual(row["streams_ready"], 1) self.assertEqual(row["streams_configured"], 1) self.assertEqual(row["confirmed"], 3) self.assertEqual(row["precision"], 0.75) def test_shared_entity_baselines_empty_for_single_stream(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "graylog_streams": [{"id": "firewall", "title": "Firewall", "enabled": True}], "graylog_stream_profiles": [{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}], } ), encoding="utf-8", ) with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[]): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertEqual(status["shared_entity_baselines"], []) def test_stream_coverage_marks_missing_profiles(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "graylog_streams": [ {"id": "firewall", "title": "Firewall", "enabled": True}, {"id": "dns", "title": "DNS", "enabled": True}, ], "graylog_stream_profiles": [{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}], } ), encoding="utf-8", ) with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[{"stream_id": "firewall", "field": "action", "buckets": 12, "ready": True}]): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) coverage = {item["stream_id"]: item for item in status["stream_coverage"]} self.assertEqual(coverage["firewall"]["health"], "no_events") self.assertEqual(coverage["firewall"]["readiness"], "1/1") self.assertEqual(coverage["dns"]["health"], "missing_profile") def test_graylog_mcp_without_enabled_streams_reports_clear_status(self): with tempfile.TemporaryDirectory() as tmp: config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "log_source": "graylog_mcp", "graylog_mcp_url": "https://graylog.example/api/mcp", "graylog_mcp_token": "token", "graylog_streams": [{"id": "disabled", "title": "Disabled", "enabled": False}], } ), encoding="utf-8", ) status = build_status(str(Path(tmp) / "missing.log"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertEqual(status["capabilities"]["graylog_mcp"]["status"], "no_streams_enabled") self.assertEqual(status["summary"]["total"], 0) def test_add_llm_assessment_uses_cached_error_status_with_previous_text(self): status = {"summary": {}, "anomalies": []} with patch("fgai.monitor.ollama_dashboard_assessment", side_effect=TimeoutError("timeout")): add_llm_assessment(status, previous="old text") self.assertEqual(status["llm_assessment"]["status"], "cached_assessment_error") self.assertEqual(status["llm_assessment"]["text"], "old text") def test_add_llm_assessment_records_unavailable_without_previous_text(self): status = {"summary": {}, "anomalies": []} with patch("fgai.monitor.ollama_dashboard_assessment", side_effect=TimeoutError("timeout")): add_llm_assessment(status) self.assertEqual(status["llm_assessment"]["status"], "assessment_unavailable") self.assertEqual(status["llm_assessment"]["text"], "") def test_add_llm_assessment_records_text(self): status = {"summary": {}, "anomalies": []} with patch("fgai.monitor.ollama_dashboard_assessment", return_value="looks noisy"): add_llm_assessment(status) self.assertEqual(status["llm_assessment"]["status"], "ok") self.assertEqual(status["llm_assessment"]["text"], "looks noisy") def test_profile_advisor_status_records_heuristic_fallback(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice eventid=4625 action=failure\n", encoding="utf-8") config_path = Path(tmp) / "config.json" config_path.write_text(json.dumps({"profile_advisor_enabled": True, "profile_advisor_model": "qwen3:8b"}), encoding="utf-8") with patch("fgai.monitor.ollama_profile_advice", side_effect=TimeoutError("timeout")): status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) advisor = status["capabilities"]["profile_advisor"] self.assertEqual(advisor["status"], "heuristic_fallback") self.assertIn("timeout", advisor["error"]) self.assertEqual(advisor["candidates"], 1) self.assertEqual(status["profile_suggestions"][0]["profile_advisor"]["status"], "heuristic") self.assertIn("timeout", status["profile_suggestions"][0]["profile_advisor"]["error"]) def test_profile_advisor_skips_when_all_streams_have_profiles(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" names = ["alice", "bob", "carol"] log_path.write_text( "\n".join( f"fgai_stream_id=windows fgai_stream=Windows username={names[index % 3]}" for index in range(25) ) + "\n", encoding="utf-8", ) config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "profile_advisor_enabled": True, "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}], } ), encoding="utf-8", ) with patch("fgai.monitor.ollama_profile_advice") as advisor: status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) advisor.assert_not_called() self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "skipped_no_profile_changes") def test_profile_advisor_runs_when_existing_profile_has_new_fields(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice eventid=4625 action=failure\n", encoding="utf-8") config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "profile_advisor_enabled": True, "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}], } ), encoding="utf-8", ) with patch("fgai.monitor.ollama_profile_advice", return_value=[]) as advisor: status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) advisor.assert_called_once() self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty") def test_profile_advisor_runs_for_low_confidence_field_even_with_full_profile(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" # Mixed-shape, low-sample-count values keep "username"'s classified # confidence below the default threshold even though its name already # makes it the entity field and the profile's field set is unchanged. log_path.write_text( "fgai_stream_id=windows fgai_stream=Windows username=alice\n" "fgai_stream_id=windows fgai_stream=Windows username=99\n", encoding="utf-8", ) config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps( { "profile_advisor_enabled": True, "graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}], } ), encoding="utf-8", ) with patch("fgai.monitor.ollama_profile_advice", return_value=[]) as advisor: status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) advisor.assert_called_once() _, kwargs = advisor.call_args self.assertIn("username", kwargs["ambiguous_fields_by_stream"].get("windows", [])) self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty") def test_auto_apply_writes_high_confidence_new_stream_profile(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" log_path.write_text( "\n".join( f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}" for index in range(1, 25) ) + "\n", encoding="utf-8", ) config_path = Path(tmp) / "config.json" config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8") build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) saved = ConfigStore(str(config_path)).read() profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]} self.assertIn("custom_vendor", profiles) self.assertIn("x7_field_a", profiles["custom_vendor"]["entity_fields"]) self.assertEqual(profiles["custom_vendor"]["field_origin"]["x7_field_a"], "auto") def test_auto_apply_never_overwrites_a_manual_origin_field(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" log_path.write_text( "\n".join( f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}" for index in range(1, 25) ) + "\n", encoding="utf-8", ) config_path = Path(tmp) / "config.json" config_path.write_text( json.dumps({ "auto_apply_profile_suggestions": True, "graylog_stream_profiles": [{ "stream_id": "custom_vendor", "name": "Custom Vendor profile", "entity_field": "x7_field_a", "entity_fields": ["x7_field_a"], "timestamp_field": "timestamp", "field_origin": {"x7_field_a": "manual"}, }], }), encoding="utf-8", ) build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json")) saved = ConfigStore(str(config_path)).read() profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]} self.assertEqual(profiles["custom_vendor"]["field_origin"], {"x7_field_a": "manual"}) def test_profile_drift_flags_confidence_drop_without_changing_live_profile(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" config_path = Path(tmp) / "config.json" history_path = Path(tmp) / "history.sqlite3" config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8") log_path.write_text( "\n".join( f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}" for index in range(1, 25) ) + "\n", encoding="utf-8", ) first = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json")) applied_profile = {item["stream_id"]: item for item in first["stream_profiles"]}["custom_vendor"] self.assertEqual(applied_profile["field_origin"]["x7_field_a"], "auto") self.assertEqual(first["profile_drift"], []) log_path.write_text( "\n".join( f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=some odd free text value number {index}" for index in range(1, 30) ) + "\n", encoding="utf-8", ) second = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json")) drift_entries = {(item["stream_id"], item["field"]) for item in second["profile_drift"]} self.assertIn(("custom_vendor", "x7_field_a"), drift_entries) still_applied_profile = {item["stream_id"]: item for item in second["stream_profiles"]}["custom_vendor"] self.assertIn("x7_field_a", still_applied_profile["entity_fields"]) def test_confidence_history_maintenance_runs_each_cycle(self): with tempfile.TemporaryDirectory() as tmp: log_path = Path(tmp) / "events.log" log_path.write_text("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login\n", encoding="utf-8") config_path = Path(tmp) / "config.json" config_path.write_text(json.dumps({}), encoding="utf-8") history_path = Path(tmp) / "history.sqlite3" status = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json")) self.assertIn("confidence_history_maintenance", status["profile_auto_apply"]) self.assertIn("remaining", status["profile_auto_apply"]["confidence_history_maintenance"]) def test_profile_suggestions_use_cached_discovered_fields(self): with tempfile.TemporaryDirectory() as tmp: history_path = str(Path(tmp) / "history.sqlite3") incidents = str(Path(tmp) / "incidents.json") first = Path(tmp) / "first.log" first.write_text( "\n".join( f"fgai_stream_id=app fgai_stream=App lcs_actor=user{index % 4} lcs_result=r{index % 3} action=ok" for index in range(1, 20) ), encoding="utf-8", ) build_status(str(first), history_path=history_path, incident_path=incidents) second = Path(tmp) / "second.log" second.write_text( "\n".join( f"fgai_stream_id=app fgai_stream=App action=ok" for _index in range(1, 5) ), encoding="utf-8", ) status = build_status(str(second), history_path=history_path, incident_path=incidents) suggestion = status["profile_suggestions"][0] self.assertIn("lcs_actor", suggestion["entity_fields"]) self.assertIn("lcs_result", suggestion["categorical_fields"]) self.assertGreater(status["baseline"]["discovery_cache_events"], 0) def test_field_catalog_creates_discovery_events_without_raw_values(self): with tempfile.TemporaryDirectory() as tmp: store = FieldDiscoveryStore(str(Path(tmp) / "history.sqlite3")) store.ingest_catalog( "app", "App", [ {"name": "lcs_customer_id", "type": {"type": "string", "properties": ["enumerable"]}}, {"name": "lcs_duration_ms", "type": {"type": "long", "properties": ["numeric"]}}, ], ) events = store.synthetic_events() fields = {field for event in events for field in event.fields} self.assertIn("lcs_customer_id", fields) self.assertIn("lcs_duration_ms", fields) def test_cached_status_with_error_keeps_last_good_dashboard_data(self): with tempfile.TemporaryDirectory() as tmp: cache_path = str(Path(tmp) / "status-cache.sqlite3") StatusSnapshotStore(cache_path).save( "last_good", { "summary": {"total": 42}, "capabilities": {"graylog_mcp": {"status": "connected"}}, "stream_coverage": [{"stream_name": "Firewall", "aggregate_status": "error", "error": "old aggregate error"}], "cross_source_correlations": [{"entity": "10.0.0.1"}], }, ) status = cached_status_with_error(cache_path, {"status": "error", "error": "mcp down"}) self.assertIsNotNone(status) self.assertEqual(status["summary"]["total"], 42) self.assertEqual(status["status_schema"], 2) self.assertTrue(status["stale"]) self.assertEqual(status["capabilities"]["graylog_mcp"]["status"], "error") self.assertTrue(status["status_cache"]["served_from_cache"]) self.assertEqual(status["stream_coverage"][0]["aggregate_error"], "old aggregate error") self.assertEqual(status["stream_coverage"][0]["raw_error"], "") if __name__ == "__main__": unittest.main()