From 7e6b510b9ebf298ae032a316ea72dda023d16fdacf921e2595969db2756fd806 Mon Sep 17 00:00:00 2001 From: larssand Date: Thu, 25 Jun 2026 18:47:27 +0200 Subject: [PATCH] fix --- src/fgai/data_quality.py | 2 +- src/fgai/incidents.py | 2 +- src/fgai/monitor.py | 48 +++++++++++++++++++++++++++++++--------- tests/test_incidents.py | 4 ++++ tests/test_monitor.py | 5 +++-- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/fgai/data_quality.py b/src/fgai/data_quality.py index ad242d3..38eb6a8 100644 --- a/src/fgai/data_quality.py +++ b/src/fgai/data_quality.py @@ -8,4 +8,4 @@ def assess_data_quality(events: list[LogEvent], stream_status: dict[str, object] missing_timestamp = sum(not (event.fields.get("eventtime") or event.fields.get("timestamp") or (event.fields.get("date") and event.fields.get("time"))) for event in events) missing_source = sum(not event.src_ip or event.src_ip == "-" for event in events) streams = stream_status.get("streams", []) if isinstance(stream_status.get("streams"), list) else [] - return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]} + return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_name") or item.get("stream_title") or item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]} diff --git a/src/fgai/incidents.py b/src/fgai/incidents.py index e2d1b4a..caee8c3 100644 --- a/src/fgai/incidents.py +++ b/src/fgai/incidents.py @@ -37,7 +37,7 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str, continue score = int(group["score"]) severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low" - streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")}) + streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id") or item.get("stream_name") or item.get("stream_title")}) timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20] incidents.append({ "entity": entity, diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index f8105a5..5c62eb7 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -24,6 +24,25 @@ from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip from .stream_profiles import parse_profiles +def _stream_titles(config: dict[str, object]) -> dict[str, str]: + return { + str(item.get("id", "")): str(item.get("title", "") or item.get("id", "")) + for item in config.get("graylog_streams", []) + if isinstance(item, dict) and item.get("id") + } + + +def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str: + return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id + + +def _profile_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str: + name = str(getattr(profile, "name", "") or "").strip() + if not name or name == stream_id: + return f"{_stream_name(stream_id, stream_titles, profile)} profile" + return name + + def build_status( log_path: str, *, @@ -40,6 +59,7 @@ def build_status( runtime_values = config_store.read() if config_exists and config_store else {} runtime_config = config_store.public() if config_store else {} stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", [])) + stream_titles = _stream_titles(runtime_values) events = read_events(log_path) if Path(log_path).exists() else [] mcp_status: dict[str, object] = {"status": "not_configured"} if runtime_values.get("log_source") == "graylog_mcp": @@ -65,9 +85,10 @@ def build_status( *tuple(str(field) for field in getattr(profile, "categorical_fields", ())), *tuple(str(field) for field in getattr(profile, "numeric_fields", ())), ) if profile else () - stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id)), profile_fields).fetch() + stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id) + stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch() events.extend(stream_events) - stream_statuses.append({"stream_id": stream_id, **stream_status}) + stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status}) mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)} except RuntimeError as exc: mcp_status = {"status": "error", "error": str(exc)} @@ -75,6 +96,18 @@ def build_status( baseline = BaselineStore(baseline_path) if baseline_path else None profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {} field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {} + for deviations in field_deviations.values(): + for deviation in deviations: + stream_id = str(deviation.get("stream_id", "")) + profile = stream_profiles.get(stream_id) + name = _stream_name(stream_id, stream_titles, profile) + deviation["stream_name"] = name + deviation["stream_title"] = name + deviation["profile_name"] = _profile_name(stream_id, stream_titles, profile) + deviation["sample_events"] = [ + {"stream": name, **sample} if isinstance(sample, dict) and not sample.get("stream") else sample + for sample in deviation.get("sample_events", []) + ] feedback = FeedbackStore().entries() for entity, deviations in field_deviations.items(): for deviation in deviations: @@ -93,16 +126,11 @@ def build_status( baseline_events = baseline.ingest(events) if baseline else 0 profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0 profile_readiness = baseline.profile_readiness(stream_profiles) if baseline else [] - stream_titles = { - str(item.get("id", "")): str(item.get("title", "")) - for item in runtime_values.get("graylog_streams", []) - if isinstance(item, dict) and item.get("id") - } profile_readiness = [ { **item, - "profile_name": getattr(stream_profiles.get(str(item.get("stream_id", ""))), "name", str(item.get("stream_id", ""))), - "stream_name": stream_titles.get(str(item.get("stream_id", ""))) or getattr(stream_profiles.get(str(item.get("stream_id", ""))), "name", str(item.get("stream_id", ""))), + "profile_name": _profile_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))), + "stream_name": _stream_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))), } for item in profile_readiness ] @@ -143,7 +171,7 @@ def build_status( "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields}, "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status}, "configuration": runtime_config, - "stream_profiles": [{"stream_id": item.stream_id, "name": item.name, "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()], + "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()], "profile_readiness": profile_readiness, "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), diff --git a/tests/test_incidents.py b/tests/test_incidents.py index c383380..a3fffb0 100644 --- a/tests/test_incidents.py +++ b/tests/test_incidents.py @@ -12,3 +12,7 @@ class IncidentTests(unittest.TestCase): result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, []) self.assertEqual(result[0]["entity"], "alice") self.assertEqual(result[0]["field_deviations"], 1) + + def test_incident_uses_stream_name_for_field_deviation(self): + result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "6a3993", "stream_name": "Windows"}]}, []) + self.assertEqual(result[0]["correlated_streams"], ["Windows"]) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index a9e6707..7a29d25 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -42,7 +42,7 @@ class MonitorTests(unittest.TestCase): json.dumps( { "graylog_streams": [{"id": "66fe", "title": "Fortigate", "enabled": True}], - "graylog_stream_profiles": [{"stream_id": "66fe", "name": "Firewall baseline", "entity_field": "srcip", "categorical_fields": ["action"]}], + "graylog_stream_profiles": [{"stream_id": "66fe", "name": "66fe", "entity_field": "srcip", "categorical_fields": ["action"]}], } ), encoding="utf-8", @@ -51,7 +51,8 @@ class MonitorTests(unittest.TestCase): status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path)) self.assertEqual(status["profile_readiness"][0]["stream_name"], "Fortigate") - self.assertEqual(status["profile_readiness"][0]["profile_name"], "Firewall baseline") + self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile") + self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile") def test_add_llm_assessment_records_error_without_ollama(self): status = {"summary": {}, "anomalies": []}