Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix mcp

Browse Source
This commit is contained in:
larssand
2026-06-30 12:46:09 +02:00
parent e781f66003
commit 86065eca59
8 changed files with 58 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/fgai/config.py
View File

@@ -13,6 +13,7 @@ DEFAULT_CONFIG: dict[str, object] = {
"graylog_stream_profiles": [],
"graylog_query": "*",
"graylog_fetch_mode": "auto",
"graylog_tls_verify": True,
"graylog_range_seconds": 300,
"graylog_max_events_per_stream": 5000,
"graylog_raw_sample_events": 5000,
@@ -64,7 +65,7 @@ class ConfigStore:
continue
if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "":
continue
if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled"}:
if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled", "graylog_tls_verify"}:
current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value

2
src/fgai/dashboard.py
View File

@@ -116,7 +116,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Installed Ollama Models</h2><div id="ollamaModels" class="muted">Loading local Ollama models.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog fetch mode<br><select name="graylog_fetch_mode"><option value="auto">Auto aggregate</option><option value="aggregate">Aggregate + sample</option><option value="raw">Raw only</option></select></label><label>Graylog MCP poll window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="300"></label><label>Graylog max events per stream<br><input name="graylog_max_events_per_stream" type="number" min="1" step="1000" placeholder="5000"></label><label>Raw sample events per stream<br><input name="graylog_raw_sample_events" type="number" min="1" step="1000" placeholder="5000"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Installed Ollama Models</h2><div id="ollamaModels" class="muted">Loading local Ollama models.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog fetch mode<br><select name="graylog_fetch_mode"><option value="auto">Auto aggregate</option><option value="aggregate">Aggregate + sample</option><option value="raw">Raw only</option></select></label><label>Graylog MCP poll window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="300"></label><label>Graylog max events per stream<br><input name="graylog_max_events_per_stream" type="number" min="1" step="1000" placeholder="5000"></label><label>Raw sample events per stream<br><input name="graylog_raw_sample_events" type="number" min="1" step="1000" placeholder="5000"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label><input name="graylog_tls_verify" type="checkbox"> Verify Graylog TLS certificate</label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main>
<script>
function esc(value) {

14
src/fgai/graylog_aggregate.py
View File

@@ -70,7 +70,19 @@ class GraylogAggregateSource:
self.query = query or "*"
def fetch_count(self, *, range_seconds: int = 300) -> dict[str, object]:
status = self.client.probe()
try:
status = self.client.probe()
except RuntimeError as exc:
return {
"status": "error",
"source": "graylog_mcp_aggregate",
"aggregate_status": "error",
"aggregate_events": 0,
"aggregate_records": 0,
"aggregate_error": f"probe_error: {exc}",
"aggregate_errors": [f"probe_error: {exc}"],
"aggregate_schema_properties": [],
}
tool_schemas = status.get("tool_schemas", {}) if isinstance(status.get("tool_schemas"), dict) else {}
aggregate_schema = tool_schemas.get("aggregate_messages", {}) if isinstance(tool_schemas, dict) else {}
properties = _schema_properties(aggregate_schema)

7
src/fgai/graylog_mcp.py
View File

@@ -2,16 +2,18 @@ from __future__ import annotations
import base64
import json
import ssl
from urllib import error, request
class GraylogMcpClient:
"""Small Streamable HTTP MCP client used for Graylog connection checks."""
def __init__(self, url: str, token: str, *, timeout: int = 15) -> None:
def __init__(self, url: str, token: str, *, timeout: int = 15, verify_tls: bool = True) -> None:
self.url = url.rstrip("/")
self.token = token
self.timeout = timeout
self.verify_tls = verify_tls
self.session_id: str | None = None
def _call(self, method: str, params: dict[str, object] | None = None, *, notification: bool = False) -> dict[str, object]:
@@ -30,7 +32,8 @@ class GraylogMcpClient:
headers["Mcp-Session-Id"] = self.session_id
req = request.Request(self.url, data=json.dumps(payload).encode("utf-8"), method="POST", headers=headers)
try:
with request.urlopen(req, timeout=self.timeout) as response:
context = None if self.verify_tls else ssl._create_unverified_context()
with request.urlopen(req, timeout=self.timeout, context=context) as response:
self.session_id = response.headers.get("Mcp-Session-Id", self.session_id)
body = response.read().decode("utf-8")
except error.HTTPError as exc:

14
src/fgai/graylog_source.py
View File

@@ -46,7 +46,19 @@ class GraylogStreamSource:
raise RuntimeError("invalid_graylog_field_mapping")
def fetch(self, *, max_events: int = 5_000, range_seconds: int = 300) -> tuple[list[LogEvent], dict[str, object]]:
status = self.client.probe()
try:
status = self.client.probe()
except RuntimeError as exc:
return [], {
"status": "error",
"source": "graylog_mcp",
"events_fetched": 0,
"pages": 0,
"partial": True,
"error": f"probe_error: {exc}",
"truncated": False,
"latest_event_time": "",
}
mapping_fields = [str(value) for value in self.mapping.values() if isinstance(value, str)]
arguments: dict[str, object] = {
"query": self.query,

5
src/fgai/monitor.py
View File

@@ -125,6 +125,7 @@ def build_status(
mcp_status: dict[str, object] = {"status": "not_configured"}
if runtime_values.get("log_source") == "graylog_mcp":
url, token = str(runtime_values.get("graylog_mcp_url", "")), str(runtime_values.get("graylog_mcp_token", ""))
verify_tls = bool(runtime_values.get("graylog_tls_verify", True))
if not url or not token:
mcp_status = {"status": "missing_configuration"}
events = []
@@ -156,10 +157,10 @@ def build_status(
stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
aggregate_status: dict[str, object] = {}
if use_aggregate:
aggregate_status = GraylogAggregateSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*"))).fetch_count(range_seconds=range_seconds)
aggregate_status = GraylogAggregateSource(GraylogMcpClient(url, token, verify_tls=verify_tls), stream_id, str(runtime_values.get("graylog_query", "*"))).fetch_count(range_seconds=range_seconds)
aggregate_events_total += int(aggregate_status.get("aggregate_events", 0) or 0)
raw_limit = min(raw_sample_events, 10_000) if use_aggregate else max_events_per_stream
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(max_events=raw_limit, range_seconds=range_seconds)
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token, verify_tls=verify_tls), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(max_events=raw_limit, range_seconds=range_seconds)
events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **aggregate_status, **stream_status, "raw_sample_limit": raw_limit})
sample_limited_streams = [item for item in stream_statuses if item.get("truncated") and use_aggregate]

10
tests/test_graylog_aggregate.py
View File

@@ -20,6 +20,11 @@ class _AggregateClient:
return self.responses.pop(0)
class _ProbeErrorClient:
def probe(self):
raise RuntimeError("connection refused")
class GraylogAggregateTests(unittest.TestCase):
def test_reads_count_from_graylog_schema_rows(self):
client = _AggregateClient([
@@ -33,6 +38,11 @@ class GraylogAggregateTests(unittest.TestCase):
self.assertEqual(client.arguments[0]["streams"], ["firewall"])
self.assertEqual(client.arguments[0]["metrics"], ["count()"])
def test_returns_error_status_instead_of_raising_on_probe_error(self):
status = GraylogAggregateSource(_ProbeErrorClient(), "firewall").fetch_count()
self.assertEqual(status["aggregate_status"], "error")
self.assertIn("probe_error", status["aggregate_error"])
def test_tries_fallback_argument_shape_after_tool_error(self):
client = _AggregateClient([
{"result": {"isError": True, "content": [{"type": "text", "text": "bad metrics"}]}},

11
tests/test_graylog_source.py
View File

@@ -23,6 +23,11 @@ class _ErrorClient:
return {"result": {"isError": True, "content": [{"type": "text", "text": "Tool call failed: timeout"}]}}
class _ProbeErrorClient:
def probe(self):
raise RuntimeError("connection refused")
class GraylogSourceTests(unittest.TestCase):
def test_applies_custom_mapping_to_generic_stream_message(self):
client = _Client()
@@ -56,6 +61,12 @@ class GraylogSourceTests(unittest.TestCase):
self.assertTrue(status["partial"])
self.assertIn("graylog_search_error", status["error"])
def test_returns_partial_status_instead_of_raising_on_probe_error(self):
events, status = GraylogStreamSource(_ProbeErrorClient(), "vpn").fetch()
self.assertEqual(events, [])
self.assertTrue(status["partial"])
self.assertIn("probe_error", status["error"])
def test_requests_selected_profile_fields(self):
client = _Client()
GraylogStreamSource(client, "windows", profile_fields=("TargetUserName", "EventID")).fetch()