510 lines
26 KiB
Python
510 lines
26 KiB
Python
import tempfile
|
|
import json
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from fgai.config import ConfigStore
|
|
from fgai.feedback import FeedbackStore
|
|
from fgai.history import StatusSnapshotStore
|
|
from fgai.history import FieldDiscoveryStore
|
|
from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, monitor_cycle_timeout_seconds, write_refreshing_status, write_status
|
|
|
|
|
|
class MonitorTests(unittest.TestCase):
|
|
def test_monitor_cycle_timeout_keeps_large_buffer(self):
|
|
self.assertEqual(monitor_cycle_timeout_seconds(120, 8), 600)
|
|
self.assertEqual(monitor_cycle_timeout_seconds(240, 8), 600)
|
|
self.assertEqual(monitor_cycle_timeout_seconds(900, 8), 1080)
|
|
|
|
def test_build_status_from_log_file(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "fg.log"
|
|
log_path.write_text(
|
|
"\n".join(
|
|
[
|
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
self.assertEqual(status["summary"]["total"], 3)
|
|
self.assertEqual(status["status_schema"], 2)
|
|
self.assertFalse(status["stale"])
|
|
self.assertFalse(status["status_cache"]["served_from_cache"])
|
|
self.assertGreaterEqual(len(status["anomalies"]), 1)
|
|
|
|
def test_entity_scope_feedback_suppresses_anomaly_in_build_status(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "fg.log"
|
|
log_path.write_text(
|
|
"\n".join(
|
|
[
|
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
|
'type=utm subtype=ips srcip=8.8.8.8 dstip=10.0.0.10 service=https action=blocked severity=critical',
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
feedback_store = FeedbackStore(str(Path(tmp) / "feedback.json"))
|
|
feedback_store.add({"scope": "entity", "entity": "8.8.8.8", "status": "false_positive", "expires_at": 9999999999})
|
|
|
|
with patch("fgai.monitor.FeedbackStore", return_value=feedback_store):
|
|
status = build_status(str(log_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
self.assertNotIn("8.8.8.8", [item["subject"] for item in status["anomalies"]])
|
|
|
|
def test_write_status_creates_parent_directory(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
output = Path(tmp) / "state" / "status.json"
|
|
|
|
write_status({"ok": True}, str(output))
|
|
|
|
self.assertTrue(output.exists())
|
|
|
|
def test_write_refreshing_status_clears_stale_mcp_error(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
output = Path(tmp) / "status.json"
|
|
output.write_text(
|
|
json.dumps(
|
|
{
|
|
"status_schema": 2,
|
|
"stale": True,
|
|
"stale_reason": "live_mcp_error",
|
|
"capabilities": {"graylog_mcp": {"status": "error", "error": "old dns error", "events_fetched": 99, "aggregate_events": 1234, "fetch_mode": "aggregate"}},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
write_refreshing_status(
|
|
str(output),
|
|
call_timeout_seconds=8,
|
|
poll_timeout_seconds=120,
|
|
runtime_values={
|
|
"log_source": "graylog_mcp",
|
|
"graylog_streams": [{"id": "fw", "title": "Firewall", "enabled": True}],
|
|
"graylog_stream_profiles": [{"stream_id": "fw", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}],
|
|
},
|
|
)
|
|
|
|
status = json.loads(output.read_text(encoding="utf-8"))
|
|
self.assertFalse(status["stale"])
|
|
self.assertEqual(status["stale_reason"], "")
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["status"], "refreshing")
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_status"], "error")
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_error"], "old dns error")
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_events_fetched"], 99)
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_aggregate_events"], 1234)
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["previous_fetch_mode"], "aggregate")
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["call_timeout_seconds"], 8)
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["poll_timeout_seconds"], 120)
|
|
self.assertEqual(status["configuration"]["enabled_streams"], 1)
|
|
self.assertEqual(status["stream_coverage"][0]["stream_name"], "Firewall")
|
|
self.assertEqual(status["stream_coverage"][0]["profile"], "Firewall profile")
|
|
self.assertEqual(status["stream_coverage"][0]["entity_field"], "srcip")
|
|
self.assertFalse(status["status_cache"]["served_from_cache"])
|
|
|
|
def test_write_refreshing_status_uses_last_good_cache(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
output = Path(tmp) / "status.json"
|
|
cache_path = str(Path(tmp) / "status-cache.sqlite3")
|
|
output.write_text(
|
|
json.dumps({"capabilities": {"graylog_mcp": {"status": "refreshing", "events_fetched": 0, "aggregate_events": 0}}}),
|
|
encoding="utf-8",
|
|
)
|
|
StatusSnapshotStore(cache_path).save(
|
|
"last_good",
|
|
{
|
|
"summary": {"total": 1234},
|
|
"capabilities": {"graylog_mcp": {"status": "connected", "events_fetched": 99, "raw_events_fetched": 88, "aggregate_events": 1234, "poll_completed_at": 123456, "fetch_mode": "aggregate", "coverage_status": "complete_window"}},
|
|
"cross_source_correlations": [{"entity": "10.0.0.1", "entity_label": "host01 (10.0.0.1)"}],
|
|
"llm_assessment": {"enabled": True, "status": "cached", "text": "previous assessment"},
|
|
},
|
|
)
|
|
|
|
write_refreshing_status(str(output), cache_path=cache_path)
|
|
|
|
status = json.loads(output.read_text(encoding="utf-8"))
|
|
mcp = status["capabilities"]["graylog_mcp"]
|
|
self.assertEqual(status["summary"]["total"], 1234)
|
|
self.assertEqual(mcp["status"], "refreshing")
|
|
self.assertEqual(mcp["events_fetched"], 99)
|
|
self.assertEqual(mcp["raw_events_fetched"], 88)
|
|
self.assertEqual(mcp["aggregate_events"], 1234)
|
|
self.assertEqual(mcp["previous_status"], "connected")
|
|
self.assertGreater(mcp["poll_started_at"], 123456)
|
|
self.assertEqual(mcp["previous_poll_completed_at"], 123456)
|
|
self.assertTrue(status["llm_assessment"]["enabled"])
|
|
self.assertTrue(status["status_cache"]["served_from_cache"])
|
|
|
|
def test_profile_readiness_includes_stream_and_profile_names(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"graylog_streams": [{"id": "66fe", "title": "Fortigate", "enabled": True}],
|
|
"graylog_stream_profiles": [{"stream_id": "66fe", "name": "66fe", "entity_field": "srcip", "categorical_fields": ["action"]}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[{"stream_id": "66fe", "field": "action", "buckets": 12, "ready": True}]):
|
|
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
self.assertEqual(status["profile_readiness"][0]["stream_name"], "Fortigate")
|
|
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile")
|
|
self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile")
|
|
|
|
def test_stream_coverage_marks_missing_profiles(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"graylog_streams": [
|
|
{"id": "firewall", "title": "Firewall", "enabled": True},
|
|
{"id": "dns", "title": "DNS", "enabled": True},
|
|
],
|
|
"graylog_stream_profiles": [{"stream_id": "firewall", "name": "Firewall profile", "entity_field": "srcip", "categorical_fields": ["action"]}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with patch("fgai.monitor.BaselineStore.profile_readiness", return_value=[{"stream_id": "firewall", "field": "action", "buckets": 12, "ready": True}]):
|
|
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
coverage = {item["stream_id"]: item for item in status["stream_coverage"]}
|
|
self.assertEqual(coverage["firewall"]["health"], "no_events")
|
|
self.assertEqual(coverage["firewall"]["readiness"], "1/1")
|
|
self.assertEqual(coverage["dns"]["health"], "missing_profile")
|
|
|
|
def test_graylog_mcp_without_enabled_streams_reports_clear_status(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"log_source": "graylog_mcp",
|
|
"graylog_mcp_url": "https://graylog.example/api/mcp",
|
|
"graylog_mcp_token": "token",
|
|
"graylog_streams": [{"id": "disabled", "title": "Disabled", "enabled": False}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
status = build_status(str(Path(tmp) / "missing.log"), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["status"], "no_streams_enabled")
|
|
self.assertEqual(status["summary"]["total"], 0)
|
|
|
|
def test_add_llm_assessment_uses_cached_error_status_with_previous_text(self):
|
|
status = {"summary": {}, "anomalies": []}
|
|
|
|
with patch("fgai.monitor.ollama_dashboard_assessment", side_effect=TimeoutError("timeout")):
|
|
add_llm_assessment(status, previous="old text")
|
|
|
|
self.assertEqual(status["llm_assessment"]["status"], "cached_assessment_error")
|
|
self.assertEqual(status["llm_assessment"]["text"], "old text")
|
|
|
|
def test_add_llm_assessment_records_unavailable_without_previous_text(self):
|
|
status = {"summary": {}, "anomalies": []}
|
|
|
|
with patch("fgai.monitor.ollama_dashboard_assessment", side_effect=TimeoutError("timeout")):
|
|
add_llm_assessment(status)
|
|
|
|
self.assertEqual(status["llm_assessment"]["status"], "assessment_unavailable")
|
|
self.assertEqual(status["llm_assessment"]["text"], "")
|
|
|
|
def test_add_llm_assessment_records_text(self):
|
|
status = {"summary": {}, "anomalies": []}
|
|
|
|
with patch("fgai.monitor.ollama_dashboard_assessment", return_value="looks noisy"):
|
|
add_llm_assessment(status)
|
|
|
|
self.assertEqual(status["llm_assessment"]["status"], "ok")
|
|
self.assertEqual(status["llm_assessment"]["text"], "looks noisy")
|
|
|
|
def test_profile_advisor_status_records_heuristic_fallback(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice eventid=4625 action=failure\n", encoding="utf-8")
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(json.dumps({"profile_advisor_enabled": True, "profile_advisor_model": "qwen3:8b"}), encoding="utf-8")
|
|
|
|
with patch("fgai.monitor.ollama_profile_advice", side_effect=TimeoutError("timeout")):
|
|
status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
advisor = status["capabilities"]["profile_advisor"]
|
|
self.assertEqual(advisor["status"], "heuristic_fallback")
|
|
self.assertIn("timeout", advisor["error"])
|
|
self.assertEqual(advisor["candidates"], 1)
|
|
self.assertEqual(status["profile_suggestions"][0]["profile_advisor"]["status"], "heuristic")
|
|
self.assertIn("timeout", status["profile_suggestions"][0]["profile_advisor"]["error"])
|
|
|
|
def test_profile_advisor_skips_when_all_streams_have_profiles(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
names = ["alice", "bob", "carol"]
|
|
log_path.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=windows fgai_stream=Windows username={names[index % 3]}"
|
|
for index in range(25)
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"profile_advisor_enabled": True,
|
|
"graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with patch("fgai.monitor.ollama_profile_advice") as advisor:
|
|
status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
advisor.assert_not_called()
|
|
self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "skipped_no_profile_changes")
|
|
|
|
def test_profile_advisor_runs_when_existing_profile_has_new_fields(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice eventid=4625 action=failure\n", encoding="utf-8")
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"profile_advisor_enabled": True,
|
|
"graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with patch("fgai.monitor.ollama_profile_advice", return_value=[]) as advisor:
|
|
status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
advisor.assert_called_once()
|
|
self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty")
|
|
|
|
def test_profile_advisor_runs_for_low_confidence_field_even_with_full_profile(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
# Mixed-shape, low-sample-count values keep "username"'s classified
|
|
# confidence below the default threshold even though its name already
|
|
# makes it the entity field and the profile's field set is unchanged.
|
|
log_path.write_text(
|
|
"fgai_stream_id=windows fgai_stream=Windows username=alice\n"
|
|
"fgai_stream_id=windows fgai_stream=Windows username=99\n",
|
|
encoding="utf-8",
|
|
)
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"profile_advisor_enabled": True,
|
|
"graylog_stream_profiles": [{"stream_id": "windows", "name": "Windows profile", "entity_field": "username", "entity_fields": ["username"], "timestamp_field": "timestamp"}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with patch("fgai.monitor.ollama_profile_advice", return_value=[]) as advisor:
|
|
status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
advisor.assert_called_once()
|
|
_, kwargs = advisor.call_args
|
|
self.assertIn("username", kwargs["ambiguous_fields_by_stream"].get("windows", []))
|
|
self.assertEqual(status["capabilities"]["profile_advisor"]["status"], "empty")
|
|
|
|
def test_auto_apply_writes_high_confidence_new_stream_profile(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
log_path.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
|
|
for index in range(1, 25)
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8")
|
|
|
|
build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
saved = ConfigStore(str(config_path)).read()
|
|
profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]}
|
|
self.assertIn("custom_vendor", profiles)
|
|
self.assertIn("x7_field_a", profiles["custom_vendor"]["entity_fields"])
|
|
self.assertEqual(profiles["custom_vendor"]["field_origin"]["x7_field_a"], "auto")
|
|
|
|
def test_auto_apply_never_overwrites_a_manual_origin_field(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
log_path.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
|
|
for index in range(1, 25)
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(
|
|
json.dumps({
|
|
"auto_apply_profile_suggestions": True,
|
|
"graylog_stream_profiles": [{
|
|
"stream_id": "custom_vendor",
|
|
"name": "Custom Vendor profile",
|
|
"entity_field": "x7_field_a",
|
|
"entity_fields": ["x7_field_a"],
|
|
"timestamp_field": "timestamp",
|
|
"field_origin": {"x7_field_a": "manual"},
|
|
}],
|
|
}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
saved = ConfigStore(str(config_path)).read()
|
|
profiles = {item["stream_id"]: item for item in saved["graylog_stream_profiles"]}
|
|
self.assertEqual(profiles["custom_vendor"]["field_origin"], {"x7_field_a": "manual"})
|
|
|
|
def test_profile_drift_flags_confidence_drop_without_changing_live_profile(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
config_path = Path(tmp) / "config.json"
|
|
history_path = Path(tmp) / "history.sqlite3"
|
|
config_path.write_text(json.dumps({"auto_apply_profile_suggestions": True}), encoding="utf-8")
|
|
|
|
log_path.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}"
|
|
for index in range(1, 25)
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
first = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
applied_profile = {item["stream_id"]: item for item in first["stream_profiles"]}["custom_vendor"]
|
|
self.assertEqual(applied_profile["field_origin"]["x7_field_a"], "auto")
|
|
self.assertEqual(first["profile_drift"], [])
|
|
|
|
log_path.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=some odd free text value number {index}"
|
|
for index in range(1, 30)
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
second = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
drift_entries = {(item["stream_id"], item["field"]) for item in second["profile_drift"]}
|
|
self.assertIn(("custom_vendor", "x7_field_a"), drift_entries)
|
|
still_applied_profile = {item["stream_id"]: item for item in second["stream_profiles"]}["custom_vendor"]
|
|
self.assertIn("x7_field_a", still_applied_profile["entity_fields"])
|
|
|
|
def test_confidence_history_maintenance_runs_each_cycle(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
log_path = Path(tmp) / "events.log"
|
|
log_path.write_text("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login\n", encoding="utf-8")
|
|
config_path = Path(tmp) / "config.json"
|
|
config_path.write_text(json.dumps({}), encoding="utf-8")
|
|
history_path = Path(tmp) / "history.sqlite3"
|
|
|
|
status = build_status(str(log_path), config_path=str(config_path), history_path=str(history_path), incident_path=str(Path(tmp) / "incidents.json"))
|
|
|
|
self.assertIn("confidence_history_maintenance", status["profile_auto_apply"])
|
|
self.assertIn("remaining", status["profile_auto_apply"]["confidence_history_maintenance"])
|
|
|
|
def test_profile_suggestions_use_cached_discovered_fields(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
history_path = str(Path(tmp) / "history.sqlite3")
|
|
incidents = str(Path(tmp) / "incidents.json")
|
|
first = Path(tmp) / "first.log"
|
|
first.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=app fgai_stream=App lcs_actor=user{index % 4} lcs_result=r{index % 3} action=ok"
|
|
for index in range(1, 20)
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
build_status(str(first), history_path=history_path, incident_path=incidents)
|
|
second = Path(tmp) / "second.log"
|
|
second.write_text(
|
|
"\n".join(
|
|
f"fgai_stream_id=app fgai_stream=App action=ok"
|
|
for _index in range(1, 5)
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
status = build_status(str(second), history_path=history_path, incident_path=incidents)
|
|
|
|
suggestion = status["profile_suggestions"][0]
|
|
self.assertIn("lcs_actor", suggestion["entity_fields"])
|
|
self.assertIn("lcs_result", suggestion["categorical_fields"])
|
|
self.assertGreater(status["baseline"]["discovery_cache_events"], 0)
|
|
|
|
def test_field_catalog_creates_discovery_events_without_raw_values(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
store = FieldDiscoveryStore(str(Path(tmp) / "history.sqlite3"))
|
|
store.ingest_catalog(
|
|
"app",
|
|
"App",
|
|
[
|
|
{"name": "lcs_customer_id", "type": {"type": "string", "properties": ["enumerable"]}},
|
|
{"name": "lcs_duration_ms", "type": {"type": "long", "properties": ["numeric"]}},
|
|
],
|
|
)
|
|
|
|
events = store.synthetic_events()
|
|
|
|
fields = {field for event in events for field in event.fields}
|
|
self.assertIn("lcs_customer_id", fields)
|
|
self.assertIn("lcs_duration_ms", fields)
|
|
|
|
def test_cached_status_with_error_keeps_last_good_dashboard_data(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
cache_path = str(Path(tmp) / "status-cache.sqlite3")
|
|
StatusSnapshotStore(cache_path).save(
|
|
"last_good",
|
|
{
|
|
"summary": {"total": 42},
|
|
"capabilities": {"graylog_mcp": {"status": "connected"}},
|
|
"stream_coverage": [{"stream_name": "Firewall", "aggregate_status": "error", "error": "old aggregate error"}],
|
|
"cross_source_correlations": [{"entity": "10.0.0.1"}],
|
|
},
|
|
)
|
|
|
|
status = cached_status_with_error(cache_path, {"status": "error", "error": "mcp down"})
|
|
|
|
self.assertIsNotNone(status)
|
|
self.assertEqual(status["summary"]["total"], 42)
|
|
self.assertEqual(status["status_schema"], 2)
|
|
self.assertTrue(status["stale"])
|
|
self.assertEqual(status["capabilities"]["graylog_mcp"]["status"], "error")
|
|
self.assertTrue(status["status_cache"]["served_from_cache"])
|
|
self.assertEqual(status["stream_coverage"][0]["aggregate_error"], "old aggregate error")
|
|
self.assertEqual(status["stream_coverage"][0]["raw_error"], "")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|