From 5010e53e0a57732211b7cf965fac1474cf8582b6bab49da2b2452760dac54c91 Mon Sep 17 00:00:00 2001
From: larssand <larssand@LPW0JW6L1>
Date: Sun, 21 Jun 2026 22:15:02 +0200
Subject: [PATCH] add mcp earch

---
 src/fgai/config.py           |  2 ++
 src/fgai/dashboard.py        |  2 +-
 src/fgai/graylog_mcp.py      |  3 ++
 src/fgai/graylog_source.py   | 69 ++++++++++++++++++++++++++++++++++++
 src/fgai/monitor.py          | 18 +++++-----
 tests/test_graylog_source.py | 26 ++++++++++++++
 6 files changed, 111 insertions(+), 9 deletions(-)
 create mode 100644 src/fgai/graylog_source.py
 create mode 100644 tests/test_graylog_source.py
diff --git a/src/fgai/config.py b/src/fgai/config.py
index 9ad798a..2f61e9d 100644
--- a/src/fgai/config.py
+++ b/src/fgai/config.py
@@ -9,6 +9,8 @@ DEFAULT_CONFIG: dict[str, object] = {
     "log_source": "local_syslog",
     "graylog_mcp_url": "",
     "graylog_stream": "",
+    "graylog_query": "*",
+    "graylog_field_mapping": "",
     "llm_enabled": False,
     "llm_model": "",
     "threat_intel_enabled": False,
diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py
index 01b21f1..2670945 100644
--- a/src/fgai/dashboard.py
+++ b/src/fgai/dashboard.py
@@ -62,7 +62,7 @@ HTML = """<!doctype html>
     <div data-view="overview" class="active"><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
     <div data-view="findings"><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
     <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
-    <div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog stream<br><input name="graylog_stream" placeholder="FortiGate stream ID or name"></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
+    <div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog stream<br><input name="graylog_stream" placeholder="FortiGate stream ID or name"></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
   </main>
 <script>
 function esc(value) {
diff --git a/src/fgai/graylog_mcp.py b/src/fgai/graylog_mcp.py
index 8898854..8d1a402 100644
--- a/src/fgai/graylog_mcp.py
+++ b/src/fgai/graylog_mcp.py
@@ -74,3 +74,6 @@ class GraylogMcpClient:
         names = [str(item.get("name")) for item in tool_list if isinstance(item, dict) and item.get("name")]
         version = initialized.get("result", {}).get("serverInfo", {}).get("version", "") if isinstance(initialized.get("result"), dict) else ""
         return {"status": "connected", "tools": names, "server_version": version}
+
+    def call_tool(self, name: str, arguments: dict[str, object]) -> dict[str, object]:
+        return self._call("tools/call", {"name": name, "arguments": arguments})
diff --git a/src/fgai/graylog_source.py b/src/fgai/graylog_source.py
new file mode 100644
index 0000000..c59b32e
--- /dev/null
+++ b/src/fgai/graylog_source.py
@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+
+from .graylog_mcp import GraylogMcpClient
+from .models import LogEvent
+
+
+DEFAULT_FIELD_MAP = {
+    "srcip": ("srcip", "src_ip", "source_ip", "client_ip"),
+    "dstip": ("dstip", "dst_ip", "destination_ip", "server_ip"),
+    "dstport": ("dstport", "dst_port", "destination_port"),
+    "srcport": ("srcport", "src_port", "source_port"),
+    "eventtime": ("eventtime", "timestamp", "time"),
+    "severity": ("severity", "level"),
+    "action": ("action", "event_action", "disposition"),
+}
+
+
+def _records(value: object) -> Iterable[dict[str, object]]:
+    if isinstance(value, dict):
+        for key in ("messages", "results", "events", "data"):
+            if isinstance(value.get(key), list):
+                yield from (item for item in value[key] if isinstance(item, dict))
+                return
+        if value:
+            yield value
+    elif isinstance(value, list):
+        yield from (item for item in value if isinstance(item, dict))
+
+
+class GraylogStreamSource:
+    def __init__(self, client: GraylogMcpClient, stream: str, query: str = "*", field_mapping: str = "") -> None:
+        self.client = client
+        self.stream = stream
+        self.query = query or "*"
+        try:
+            self.mapping = json.loads(field_mapping) if field_mapping else {}
+        except json.JSONDecodeError as exc:
+            raise RuntimeError("invalid_graylog_field_mapping") from exc
+        if not isinstance(self.mapping, dict):
+            raise RuntimeError("invalid_graylog_field_mapping")
+
+    def fetch(self) -> tuple[list[LogEvent], dict[str, object]]:
+        status = self.client.probe()
+        result = self.client.call_tool("search_messages", {"query": self.query, "stream": self.stream, "limit": 1000})
+        content = result.get("result", {}).get("content", []) if isinstance(result.get("result"), dict) else []
+        records: list[dict[str, object]] = []
+        for item in content if isinstance(content, list) else []:
+            if isinstance(item, dict) and item.get("type") == "text":
+                try:
+                    records.extend(_records(json.loads(str(item.get("text", "")))))
+                except json.JSONDecodeError:
+                    continue
+        events = [self._event(record) for record in records]
+        status.update({"source": "graylog_mcp", "events_fetched": len(events)})
+        return events, status
+
+    def _event(self, record: dict[str, object]) -> LogEvent:
+        fields = {str(key).lower(): str(value) for key, value in record.items() if value is not None}
+        for canonical, candidates in DEFAULT_FIELD_MAP.items():
+            mapped = self.mapping.get(canonical)
+            candidates = (str(mapped),) if mapped else candidates
+            for candidate in candidates:
+                if candidate.lower() in fields:
+                    fields[canonical] = fields[candidate.lower()]
+                    break
+        return LogEvent(raw=json.dumps(record, sort_keys=True), fields=fields)
diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py
index 66a7be5..159c8dd 100644
--- a/src/fgai/monitor.py
+++ b/src/fgai/monitor.py
@@ -8,6 +8,7 @@ from .anomaly import anomaly_summary, detect_source_anomalies
 from .baseline import BaselineStore
 from .config import ConfigStore
 from .graylog_mcp import GraylogMcpClient
+from .graylog_source import GraylogStreamSource
 from .llm import ollama_dashboard_assessment
 from .logs import local_in_failures, read_events, summarize_events, top_field_values
 from .mitigation import parse_allowlist, suggest_block_candidates
@@ -26,26 +27,27 @@ def build_status(
     baseline_path: str | None = None,
     config_path: str | None = None,
 ) -> dict[str, object]:
-    events = read_events(log_path) if Path(log_path).exists() else []
-    baseline = BaselineStore(baseline_path) if baseline_path else None
-    profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
-    anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles)
-    baseline_events = baseline.ingest(events) if baseline else 0
     config_store = ConfigStore(config_path) if config_path else None
     config_exists = bool(config_store and config_store.path.exists())
     runtime_values = config_store.read() if config_exists and config_store else {}
     runtime_config = config_store.public() if config_store else {}
+    events = read_events(log_path) if Path(log_path).exists() else []
     mcp_status: dict[str, object] = {"status": "not_configured"}
     if runtime_values.get("log_source") == "graylog_mcp":
-        url = str(runtime_values.get("graylog_mcp_url", ""))
-        token = str(runtime_values.get("graylog_mcp_token", ""))
+        url, token = str(runtime_values.get("graylog_mcp_url", "")), str(runtime_values.get("graylog_mcp_token", ""))
         if not url or not token:
             mcp_status = {"status": "missing_configuration"}
+            events = []
         else:
             try:
-                mcp_status = GraylogMcpClient(url, token).probe()
+                events, mcp_status = GraylogStreamSource(GraylogMcpClient(url, token), str(runtime_values.get("graylog_stream", "")), str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", ""))).fetch()
             except RuntimeError as exc:
                 mcp_status = {"status": "error", "error": str(exc)}
+                events = []
+    baseline = BaselineStore(baseline_path) if baseline_path else None
+    profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
+    anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles)
+    baseline_events = baseline.ingest(events) if baseline else 0
     intel_ips = sorted(
         {
             ip
diff --git a/tests/test_graylog_source.py b/tests/test_graylog_source.py
new file mode 100644
index 0000000..3138fe0
--- /dev/null
+++ b/tests/test_graylog_source.py
@@ -0,0 +1,26 @@
+import unittest
+
+from fgai.graylog_source import GraylogStreamSource
+
+
+class _Client:
+    def probe(self):
+        return {"status": "connected"}
+
+    def call_tool(self, _name, _arguments):
+        return {"result": {"content": [{"type": "text", "text": '{"messages":[{"client":"10.0.0.1","server":"8.8.8.8","result":"deny"}]}' }]}}
+
+
+class GraylogSourceTests(unittest.TestCase):
+    def test_applies_custom_mapping_to_generic_stream_message(self):
+        events, status = GraylogStreamSource(
+            _Client(), "vpn", field_mapping='{"srcip":"client","dstip":"server","action":"result"}'
+        ).fetch()
+        self.assertEqual(status["events_fetched"], 1)
+        self.assertEqual(events[0].src_ip, "10.0.0.1")
+        self.assertEqual(events[0].dst_ip, "8.8.8.8")
+        self.assertEqual(events[0].action, "deny")
+
+
+if __name__ == "__main__":
+    unittest.main()
