Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add more matching

Browse Source
This commit is contained in:
larssand
2026-06-22 19:19:31 +02:00
parent a5489d3c87
commit f2257c74de
8 changed files with 85 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
correlation.py Normal file
View File

@@ -0,0 +1,21 @@
from __future__ import annotations
from collections import defaultdict
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
def correlate_source_ips(events: list[LogEvent], *, limit: int = 20) -> list[dict[str, object]]:
grouped: dict[str, list[LogEvent]] = defaultdict(list)
for event in events:
if event.src_ip:
grouped[event.src_ip].append(event)
correlations = []
for source_ip, source_events in grouped.items():
streams = sorted({event.fields.get("fgai_stream", "local_syslog") for event in source_events})
if len(streams) < 2:
continue
threat_events = sum(event.action in THREAT_ACTIONS or is_utm_event(event) for event in source_events)
correlations.append({"source_ip": source_ip, "streams": streams, "events": len(source_events), "security_events": threat_events})
return sorted(correlations, key=lambda item: (int(item["security_events"]), int(item["events"])), reverse=True)[:limit]

8
src/fgai/dashboard.py
View File

@@ -6,6 +6,7 @@ from pathlib import Path
from .config import ConfigStore from .config import ConfigStore
from .graylog_mcp import GraylogMcpClient from .graylog_mcp import GraylogMcpClient
from .metrics import prometheus_metrics
HTML = """<!doctype html> HTML = """<!doctype html>
@@ -241,6 +242,13 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
body = json.dumps({"summary": {}, "anomalies": [], "block_candidates": []}).encode("utf-8") body = json.dumps({"summary": {}, "anomalies": [], "block_candidates": []}).encode("utf-8")
self._send(200, "application/json", body) self._send(200, "application/json", body)
return return
if self.path == "/metrics":
try:
status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {}
except json.JSONDecodeError:
status = {}
self._send(200, "text/plain; version=0.0.4; charset=utf-8", prometheus_metrics(status).encode("utf-8"))
return
if self.path.startswith("/images/") and image_root: if self.path.startswith("/images/") and image_root:
image_path = image_root / Path(self.path).name image_path = image_root / Path(self.path).name
if image_path.exists() and image_path.is_file(): if image_path.exists() and image_path.is_file():

4
src/fgai/graylog_source.py
View File

@@ -41,10 +41,11 @@ def _records(value: object) -> Iterable[dict[str, object]]:
class GraylogStreamSource: class GraylogStreamSource:
def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "") -> None: def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "", stream_label: str = "") -> None:
self.client = client self.client = client
self.stream = stream self.stream = stream
self.query = query or "*" self.query = query or "*"
self.stream_label = stream_label or stream
try: try:
self.mapping = json.loads(field_mapping) if field_mapping else {} self.mapping = json.loads(field_mapping) if field_mapping else {}
except json.JSONDecodeError as exc: except json.JSONDecodeError as exc:
@@ -88,6 +89,7 @@ class GraylogStreamSource:
def _event(self, record: dict[str, object]) -> LogEvent: def _event(self, record: dict[str, object]) -> LogEvent:
fields = {str(key).lower(): str(value) for key, value in record.items() if value is not None} fields = {str(key).lower(): str(value) for key, value in record.items() if value is not None}
fields["fgai_stream"] = self.stream_label
for canonical, candidates in DEFAULT_FIELD_MAP.items(): for canonical, candidates in DEFAULT_FIELD_MAP.items():
mapped = self.mapping.get(canonical) mapped = self.mapping.get(canonical)
candidates = (str(mapped),) if mapped else candidates candidates = (str(mapped),) if mapped else candidates

1
src/fgai/llm.py
View File

@@ -59,6 +59,7 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
"event_context": analysis.get("event_context", {}), "event_context": analysis.get("event_context", {}),
"diagnostics": analysis.get("diagnostics", {}), "diagnostics": analysis.get("diagnostics", {}),
"capabilities": analysis.get("capabilities", {}), "capabilities": analysis.get("capabilities", {}),
"cross_source_correlations": analysis.get("cross_source_correlations", [])[:20],
} }
return ollama_summary( return ollama_summary(
[], [],

19
src/fgai/metrics.py Normal file
View File

@@ -0,0 +1,19 @@
from __future__ import annotations
def prometheus_metrics(status: dict[str, object]) -> str:
summary = status.get("summary", {}) if isinstance(status.get("summary"), dict) else {}
anomalies = status.get("anomaly_summary", {}) if isinstance(status.get("anomaly_summary"), dict) else {}
baseline = status.get("baseline", {}) if isinstance(status.get("baseline"), dict) else {}
capabilities = status.get("capabilities", {}) if isinstance(status.get("capabilities"), dict) else {}
mcp = capabilities.get("graylog_mcp", {}) if isinstance(capabilities.get("graylog_mcp"), dict) else {}
lines = ["# HELP fgai_events_total Events in the latest analysis window.", "# TYPE fgai_events_total gauge"]
for key in ("total", "utm", "threat_actions", "critical_or_high"):
lines.append(f'fgai_events_total{{kind="{key}"}} {int(summary.get(key, 0) or 0)}')
lines += ["# HELP fgai_anomalies Anomalies by severity.", "# TYPE fgai_anomalies gauge"]
for severity in ("critical", "high", "medium", "low"):
lines.append(f'fgai_anomalies{{severity="{severity}"}} {int(anomalies.get(severity, 0) or 0)}')
lines += ["# HELP fgai_baseline_sources_ready Sources with sufficient baseline history.", "# TYPE fgai_baseline_sources_ready gauge", f'fgai_baseline_sources_ready {int(baseline.get("sources_ready", 0) or 0)}']
lines += ["# HELP fgai_graylog_mcp_connected Graylog MCP connectivity state.", "# TYPE fgai_graylog_mcp_connected gauge", f'fgai_graylog_mcp_connected {1 if mcp.get("status") == "connected" else 0}']
lines += ["# HELP fgai_graylog_events_fetched Events fetched from Graylog in the latest poll.", "# TYPE fgai_graylog_events_fetched gauge", f'fgai_graylog_events_fetched {int(mcp.get("events_fetched", 0) or 0)}']
return "\n".join(lines) + "\n"

12
src/fgai/monitor.py
View File

@@ -7,6 +7,7 @@ from pathlib import Path
from .anomaly import anomaly_summary, detect_source_anomalies from .anomaly import anomaly_summary, detect_source_anomalies
from .baseline import BaselineStore from .baseline import BaselineStore
from .config import ConfigStore from .config import ConfigStore
from .correlation import correlate_source_ips
from .event_context import build_event_context from .event_context import build_event_context
from .graylog_mcp import GraylogMcpClient from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource from .graylog_source import GraylogStreamSource
@@ -42,13 +43,15 @@ def build_status(
else: else:
try: try:
configured_streams = runtime_values.get("graylog_streams", []) configured_streams = runtime_values.get("graylog_streams", [])
stream_ids = [str(item.get("id")) for item in configured_streams if isinstance(item, dict) and item.get("enabled") and item.get("id")] stream_configs = [item for item in configured_streams if isinstance(item, dict) and item.get("enabled") and item.get("id")]
stream_ids = [str(item.get("id")) for item in stream_configs]
if not stream_ids: if not stream_ids:
stream_ids = [str(runtime_values.get("graylog_stream", ""))] stream_configs = [{"id": str(runtime_values.get("graylog_stream", "")), "title": "Graylog"}]
stream_statuses = [] stream_statuses = []
events = [] events = []
for stream_id in stream_ids: for stream_config in stream_configs:
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", ""))).fetch() stream_id = str(stream_config["id"])
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), str(stream_config.get("title", stream_id))).fetch()
events.extend(stream_events) events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, **stream_status}) stream_statuses.append({"stream_id": stream_id, **stream_status})
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)} mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
@@ -107,6 +110,7 @@ def build_status(
"local_in_failures": local_in_failures(events, limit=10), "local_in_failures": local_in_failures(events, limit=10),
}, },
"event_context": build_event_context(events), "event_context": build_event_context(events),
"cross_source_correlations": correlate_source_ips(events),
"anomalies": [ "anomalies": [
{ {
"subject": finding.subject, "subject": finding.subject,

15
tests/test_correlation.py Normal file
View File

@@ -0,0 +1,15 @@
import unittest
from fgai.correlation import correlate_source_ips
from fgai.logs import parse_log_line
class CorrelationTests(unittest.TestCase):
def test_correlates_same_source_across_streams(self):
events = [
parse_log_line("srcip=10.0.0.5 fgai_stream=Fortigate action=blocked"),
parse_log_line("srcip=10.0.0.5 fgai_stream=DNS action=deny"),
]
result = correlate_source_ips(events)
self.assertEqual(result[0]["source_ip"], "10.0.0.5")
self.assertEqual(result[0]["streams"], ["DNS", "Fortigate"])

10
tests/test_metrics.py Normal file
View File

@@ -0,0 +1,10 @@
import unittest
from fgai.metrics import prometheus_metrics
class MetricsTests(unittest.TestCase):
def test_renders_low_cardinality_metrics(self):
output = prometheus_metrics({"summary": {"total": 10}, "anomaly_summary": {"high": 2}, "baseline": {"sources_ready": 3}, "capabilities": {"graylog_mcp": {"status": "connected", "events_fetched": 9}}})
self.assertIn('fgai_events_total{kind="total"} 10', output)
self.assertIn("fgai_graylog_mcp_connected 1", output)