Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
larssand
2026-06-25 18:47:27 +02:00
parent c6274f840a
commit 7e6b510b9e
5 changed files with 47 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/fgai/data_quality.py
View File

@@ -8,4 +8,4 @@ def assess_data_quality(events: list[LogEvent], stream_status: dict[str, object]
missing_timestamp = sum(not (event.fields.get("eventtime") or event.fields.get("timestamp") or (event.fields.get("date") and event.fields.get("time"))) for event in events)
missing_source = sum(not event.src_ip or event.src_ip == "-" for event in events)
streams = stream_status.get("streams", []) if isinstance(stream_status.get("streams"), list) else []
return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]}
return {"events": total, "missing_timestamp": missing_timestamp, "missing_source": missing_source, "timestamp_coverage": round(100 * (total - missing_timestamp) / total, 1) if total else 0, "source_coverage": round(100 * (total - missing_source) / total, 1) if total else 0, "truncated_streams": [item.get("stream_name") or item.get("stream_title") or item.get("stream_id") for item in streams if isinstance(item, dict) and item.get("truncated")]}

2
src/fgai/incidents.py
View File

@@ -37,7 +37,7 @@ def build_incidents(anomalies: list[AnomalyFinding], field_deviations: dict[str,
continue
score = int(group["score"])
severity = "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id")})
streams = sorted({stream for item in group["correlations"] for stream in item.get("streams", [])} | {str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")) for item in group["fields"] if item.get("stream_id") or item.get("stream_name") or item.get("stream_title")})
timeline = sorted(group["timeline"], key=lambda item: str(item.get("timestamp", "")))[:20]
incidents.append({
"entity": entity,

48
src/fgai/monitor.py
View File

@@ -24,6 +24,25 @@ from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
from .stream_profiles import parse_profiles
def _stream_titles(config: dict[str, object]) -> dict[str, str]:
return {
str(item.get("id", "")): str(item.get("title", "") or item.get("id", ""))
for item in config.get("graylog_streams", [])
if isinstance(item, dict) and item.get("id")
}
def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
def _profile_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
name = str(getattr(profile, "name", "") or "").strip()
if not name or name == stream_id:
return f"{_stream_name(stream_id, stream_titles, profile)} profile"
return name
def build_status(
log_path: str,
*,
@@ -40,6 +59,7 @@ def build_status(
runtime_values = config_store.read() if config_exists and config_store else {}
runtime_config = config_store.public() if config_store else {}
stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
stream_titles = _stream_titles(runtime_values)
events = read_events(log_path) if Path(log_path).exists() else []
mcp_status: dict[str, object] = {"status": "not_configured"}
if runtime_values.get("log_source") == "graylog_mcp":
@@ -65,9 +85,10 @@ def build_status(
*tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
) if profile else ()
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id)), profile_fields).fetch()
stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch()
events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, **stream_status})
stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status})
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
except RuntimeError as exc:
mcp_status = {"status": "error", "error": str(exc)}
@@ -75,6 +96,18 @@ def build_status(
baseline = BaselineStore(baseline_path) if baseline_path else None
profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {}
for deviations in field_deviations.values():
for deviation in deviations:
stream_id = str(deviation.get("stream_id", ""))
profile = stream_profiles.get(stream_id)
name = _stream_name(stream_id, stream_titles, profile)
deviation["stream_name"] = name
deviation["stream_title"] = name
deviation["profile_name"] = _profile_name(stream_id, stream_titles, profile)
deviation["sample_events"] = [
{"stream": name, **sample} if isinstance(sample, dict) and not sample.get("stream") else sample
for sample in deviation.get("sample_events", [])
]
feedback = FeedbackStore().entries()
for entity, deviations in field_deviations.items():
for deviation in deviations:
@@ -93,16 +126,11 @@ def build_status(
baseline_events = baseline.ingest(events) if baseline else 0
profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
profile_readiness = baseline.profile_readiness(stream_profiles) if baseline else []
stream_titles = {
str(item.get("id", "")): str(item.get("title", ""))
for item in runtime_values.get("graylog_streams", [])
if isinstance(item, dict) and item.get("id")
}
profile_readiness = [
{
**item,
"profile_name": getattr(stream_profiles.get(str(item.get("stream_id", ""))), "name", str(item.get("stream_id", ""))),
"stream_name": stream_titles.get(str(item.get("stream_id", ""))) or getattr(stream_profiles.get(str(item.get("stream_id", ""))), "name", str(item.get("stream_id", ""))),
"profile_name": _profile_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))),
"stream_name": _stream_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))),
}
for item in profile_readiness
]
@@ -143,7 +171,7 @@ def build_status(
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
"configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": item.name, "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()],
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()],
"profile_readiness": profile_readiness,
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),

4
tests/test_incidents.py
View File

@@ -12,3 +12,7 @@ class IncidentTests(unittest.TestCase):
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "windows"}]}, [])
self.assertEqual(result[0]["entity"], "alice")
self.assertEqual(result[0]["field_deviations"], 1)
def test_incident_uses_stream_name_for_field_deviation(self):
result = build_incidents([], {"alice": [{"score": 15, "reason": "new login country", "stream_id": "6a3993", "stream_name": "Windows"}]}, [])
self.assertEqual(result[0]["correlated_streams"], ["Windows"])

5
tests/test_monitor.py
View File

@@ -42,7 +42,7 @@ class MonitorTests(unittest.TestCase):
json.dumps(
{
"graylog_streams": [{"id": "66fe", "title": "Fortigate", "enabled": True}],
"graylog_stream_profiles": [{"stream_id": "66fe", "name": "Firewall baseline", "entity_field": "srcip", "categorical_fields": ["action"]}],
"graylog_stream_profiles": [{"stream_id": "66fe", "name": "66fe", "entity_field": "srcip", "categorical_fields": ["action"]}],
}
),
encoding="utf-8",
@@ -51,7 +51,8 @@ class MonitorTests(unittest.TestCase):
status = build_status(str(Path(tmp) / "missing.log"), baseline_path=str(Path(tmp) / "baseline.sqlite3"), config_path=str(config_path))
self.assertEqual(status["profile_readiness"][0]["stream_name"], "Fortigate")
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Firewall baseline")
self.assertEqual(status["profile_readiness"][0]["profile_name"], "Fortigate profile")
self.assertEqual(status["stream_profiles"][0]["name"], "Fortigate profile")
def test_add_llm_assessment_records_error_without_ollama(self):
status = {"summary": {}, "anomalies": []}