fix
This commit is contained in:
@@ -8,4 +8,4 @@ def assess_data_quality(events: list[LogEvent], stream_status: dict[str, object]
|
|||||||
missing_timestamp = sum(not (event.fields.get("eventtime") or event.fields.get("timestamp") or (event.fields.get("date") and event.fields.get("time"))) for event in events)
|
missing_timestamp = sum(not (event.fields.get("eventtime") or event.fields.get("timestamp") or (event.fields.get("date") and event.fields.get("time"))) for event in events)
|
||||||
missing_source = sum(not event.src_ip or event.src_ip == "-" for event in events)
|
missing_source = sum(not event.src_ip or event.src_ip == "-" for event in events)
|
||||||
streams = stream_status.get("streams", []) if isinstance(stream_status.get("streams"), list) else []
|
streams = stream_status.get("streams", []) if isinstance(stream_status.get("streams"), list) else []
|
||||||
return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]}
|
return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_name") or item.get("stream_title") or item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]}
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str,
|
|||||||
continue
|
continue
|
||||||
score = int(group["score"])
|
score = int(group["score"])
|
||||||
severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
|
severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
|
||||||
streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")})
|
streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id") or item.get("stream_name") or item.get("stream_title")})
|
||||||
timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
|
timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
|
||||||
incidents.append({
|
incidents.append({
|
||||||
"entity": entity,
|
"entity": entity,
|
||||||
|
|||||||
@@ -24,6 +24,25 @@ from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
|
|||||||
from .stream_profiles import parse_profiles
|
from .stream_profiles import parse_profiles
|
||||||
|
|
||||||
|
|
||||||
|
def _stream_titles(config: dict[str, object]) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
str(item.get("id", "")): str(item.get("title", "") or item.get("id", ""))
|
||||||
|
for item in config.get("graylog_streams", [])
|
||||||
|
if isinstance(item, dict) and item.get("id")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
|
||||||
|
return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
|
||||||
|
|
||||||
|
|
||||||
|
def _profile_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
|
||||||
|
name = str(getattr(profile, "name", "") or "").strip()
|
||||||
|
if not name or name == stream_id:
|
||||||
|
return f"{_stream_name(stream_id, stream_titles, profile)} profile"
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
def build_status(
|
def build_status(
|
||||||
log_path: str,
|
log_path: str,
|
||||||
*,
|
*,
|
||||||
@@ -40,6 +59,7 @@ def build_status(
|
|||||||
runtime_values = config_store.read() if config_exists and config_store else {}
|
runtime_values = config_store.read() if config_exists and config_store else {}
|
||||||
runtime_config = config_store.public() if config_store else {}
|
runtime_config = config_store.public() if config_store else {}
|
||||||
stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
|
stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
|
||||||
|
stream_titles = _stream_titles(runtime_values)
|
||||||
events = read_events(log_path) if Path(log_path).exists() else []
|
events = read_events(log_path) if Path(log_path).exists() else []
|
||||||
mcp_status: dict[str, object] = {"status": "not_configured"}
|
mcp_status: dict[str, object] = {"status": "not_configured"}
|
||||||
if runtime_values.get("log_source") == "graylog_mcp":
|
if runtime_values.get("log_source") == "graylog_mcp":
|
||||||
@@ -65,9 +85,10 @@ def build_status(
|
|||||||
*tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
|
*tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
|
||||||
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
|
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
|
||||||
) if profile else ()
|
) if profile else ()
|
||||||
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id)), profile_fields).fetch()
|
stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
|
||||||
|
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch()
|
||||||
events.extend(stream_events)
|
events.extend(stream_events)
|
||||||
stream_statuses.append({"stream_id": stream_id, **stream_status})
|
stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status})
|
||||||
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
|
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
|
||||||
except RuntimeError as exc:
|
except RuntimeError as exc:
|
||||||
mcp_status = {"status": "error", "error": str(exc)}
|
mcp_status = {"status": "error", "error": str(exc)}
|
||||||
@@ -75,6 +96,18 @@ def build_status(
|
|||||||
baseline = BaselineStore(baseline_path) if baseline_path else None
|
baseline = BaselineStore(baseline_path) if baseline_path else None
|
||||||
profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
|
profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
|
||||||
field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {}
|
field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {}
|
||||||
|
for deviations in field_deviations.values():
|
||||||
|
for deviation in deviations:
|
||||||
|
stream_id = str(deviation.get("stream_id", ""))
|
||||||
|
profile = stream_profiles.get(stream_id)
|
||||||
|
name = _stream_name(stream_id, stream_titles, profile)
|
||||||
|
deviation["stream_name"] = name
|
||||||
|
deviation["stream_title"] = name
|
||||||
|
deviation["profile_name"] = _profile_name(stream_id, stream_titles, profile)
|
||||||
|
deviation["sample_events"] = [
|
||||||
|
{"stream": name, **sample} if isinstance(sample, dict) and not sample.get("stream") else sample
|
||||||
|
for sample in deviation.get("sample_events", [])
|
||||||
|
]
|
||||||
feedback = FeedbackStore().entries()
|
feedback = FeedbackStore().entries()
|
||||||
for entity, deviations in field_deviations.items():
|
for entity, deviations in field_deviations.items():
|
||||||
for deviation in deviations:
|
for deviation in deviations:
|
||||||
@@ -93,16 +126,11 @@ def build_status(
|
|||||||
baseline_events = baseline.ingest(events) if baseline else 0
|
baseline_events = baseline.ingest(events) if baseline else 0
|
||||||
profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
|
profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
|
||||||
profile_readiness = baseline.profile_readiness(stream_profiles) if baseline else []
|
profile_readiness = baseline.profile_readiness(stream_profiles) if baseline else []
|
||||||
stream_titles = {
|
|
||||||
str(item.get("id", "")): str(item.get("title", ""))
|
|
||||||
for item in runtime_values.get("graylog_streams", [])
|
|
||||||
if isinstance(item, dict) and item.get("id")
|
|
||||||
}
|
|
||||||
profile_readiness = [
|
profile_readiness = [
|
||||||
{
|
{
|
||||||
**item,
|
**item,
|
||||||
"profile_name": getattr(stream_profiles.get(str(item.get("stream_id", ""))), "name", str(item.get("stream_id", ""))),
|
"profile_name": _profile_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))),
|
||||||
"stream_name": stream_titles.get(str(item.get("stream_id", ""))) or getattr(stream_profiles.get(str(item.get("stream_id", ""))), "name", str(item.get("stream_id", ""))),
|
"stream_name": _stream_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))),
|
||||||
}
|
}
|
||||||
for item in profile_readiness
|
for item in profile_readiness
|
||||||
]
|
]
|
||||||
@@ -143,7 +171,7 @@ def build_status(
|
|||||||
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields},
|
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields},
|
||||||
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
|
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
|
||||||
"configuration": runtime_config,
|
"configuration": runtime_config,
|
||||||
"stream_profiles": [{"stream_id": item.stream_id, "name": item.name, "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()],
|
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()],
|
||||||
"profile_readiness": profile_readiness,
|
"profile_readiness": profile_readiness,
|
||||||
"diagnostics": {
|
"diagnostics": {
|
||||||
"top_source_ips": top_field_values(events, "srcip", limit=10),
|
"top_source_ips": top_field_values(events, "srcip", limit=10),
|
||||||
|
|||||||
@@ -12,3 +12,7 @@ class IncidentTests(unittest.TestCase):
|
|||||||
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, [])
|
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, [])
|
||||||
self.assertEqual(result[0]["entity"], "alice")
|
self.assertEqual(result[0]["entity"], "alice")
|
||||||
self.assertEqual(result[0]["field_deviations"], 1)
|
self.assertEqual(result[0]["field_deviations"], 1)
|
||||||
|
|
||||||
|
def test_incident_uses_stream_name_for_field_deviation(self):
|
||||||
|
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "6a3993", "stream_name": "Windows"}]}, [])
|
||||||
|
self.assertEqual(result[0]["correlated_streams"], ["Windows"])
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ class MonitorTests(unittest.TestCase):
|
|||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
"graylog_streams": [{"id": "66fe", "title": "Fortigate", "enabled": True}],
|
"graylog_streams": [{"id": "66fe", "title": "Fortigate", "enabled": True}],
|
||||||
"graylog_stream_profiles": [{"stream_id": "66fe", "name": "Firewall baseline", "entity_field": "srcip", "categorical_fields": ["action"]}],
|
"graylog_stream_profiles": [{"stream_id": "66fe", "name": "66fe", "entity_field": "srcip", "categorical_fields": ["action"]}],
|
||||||
}
|
}
|
||||||
),
|
),
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
@@ -51,7 +51,8 @@ class MonitorTests(unittest.TestCase):
|
|||||||
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path))
|
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path))
|
||||||
|
|
||||||
self.assertEqual(status["profile_readiness"][0]["stream_name"], "Fortigate")
|
self.assertEqual(status["profile_readiness"][0]["stream_name"], "Fortigate")
|
||||||
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Firewall baseline")
|
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile")
|
||||||
|
self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile")
|
||||||
|
|
||||||
def test_add_llm_assessment_records_error_without_ollama(self):
|
def test_add_llm_assessment_records_error_without_ollama(self):
|
||||||
status = {"summary": {}, "anomalies": []}
|
status = {"summary": {}, "anomalies": []}
|
||||||
|
|||||||
Reference in New Issue
Block a user