Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add more streams

Browse Source
This commit is contained in:
larssand
2026-06-22 19:09:25 +02:00
parent f99e699da6
commit a5489d3c87
6 changed files with 115 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/fgai/config.py
View File

@@ -9,6 +9,7 @@ DEFAULT_CONFIG: dict[str, object] = {
"log_source": "local_syslog", "log_source": "local_syslog",
"graylog_mcp_url": "", "graylog_mcp_url": "",
"graylog_stream": "", "graylog_stream": "",
"graylog_streams": [],
"graylog_query": "*", "graylog_query": "*",
"graylog_field_mapping": "", "graylog_field_mapping": "",
"llm_enabled": False, "llm_enabled": False,
@@ -46,6 +47,11 @@ class ConfigStore:
current[key] = bool(value) current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value current[key] = value
elif key == "graylog_streams" and isinstance(value, list):
current[key] = [
{"id": str(item.get("id", "")), "title": str(item.get("title", "")), "enabled": bool(item.get("enabled"))}
for item in value if isinstance(item, dict) and item.get("id")
]
elif isinstance(value, str): elif isinstance(value, str):
current[key] = value.strip() current[key] = value.strip()
self.path.parent.mkdir(parents=True, exist_ok=True) self.path.parent.mkdir(parents=True, exist_ok=True)

28
src/fgai/dashboard.py
View File

@@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
from .config import ConfigStore from .config import ConfigStore
from .graylog_mcp import GraylogMcpClient
HTML = """<!doctype html> HTML = """<!doctype html>
@@ -62,7 +63,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div> <div data-view="overview" class="active"><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div> <div data-view="findings"><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div> <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog stream<br><input name="graylog_stream" placeholder="FortiGate stream ID or name"></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div> <div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main> </main>
<script> <script>
function esc(value) { function esc(value) {
@@ -152,7 +153,10 @@ async function refresh() {
{label:'Detail', key:'detail'} {label:'Detail', key:'detail'}
]); ]);
const d = data.diagnostics || {}; const d = data.diagnostics || {};
const context = data.event_context || {};
document.getElementById('diagnostics').innerHTML = document.getElementById('diagnostics').innerHTML =
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}]) +
'<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
'<h3>Top Policy IDs</h3>' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + '<h3>Top Policy IDs</h3>' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
@@ -172,12 +176,20 @@ async function loadSettings() {
const token = form.elements.namedItem('graylog_mcp_token'); const token = form.elements.namedItem('graylog_mcp_token');
token.placeholder = config.graylog_mcp_token_configured ? 'Token configured; leave blank to keep it' : 'Paste a read-only token'; token.placeholder = config.graylog_mcp_token_configured ? 'Token configured; leave blank to keep it' : 'Paste a read-only token';
} }
async function loadStreams() {
const response = await fetch('/api/graylog/streams');
const payload = await response.json();
const selected = new Set((payload.selected || []).map(item => item.id));
document.getElementById('streamPicker').innerHTML = (payload.streams || []).map(stream => `<label><input type="checkbox" class="graylog-stream" data-id="${esc(stream.id)}" data-title="${esc(stream.title)}" ${selected.has(stream.id) ? 'checked' : ''}> ${esc(stream.title)}</label><br>`).join('') || esc(payload.error || 'No streams found.');
}
document.getElementById('loadStreams').addEventListener('click', loadStreams);
document.getElementById('settingsForm').addEventListener('submit', async event => { document.getElementById('settingsForm').addEventListener('submit', async event => {
event.preventDefault(); event.preventDefault();
const form = event.currentTarget; const form = event.currentTarget;
const values = Object.fromEntries(new FormData(form)); const values = Object.fromEntries(new FormData(form));
values.llm_enabled = form.elements.llm_enabled.checked; values.llm_enabled = form.elements.llm_enabled.checked;
values.threat_intel_enabled = form.elements.threat_intel_enabled.checked; values.threat_intel_enabled = form.elements.threat_intel_enabled.checked;
values.graylog_streams = [...document.querySelectorAll('.graylog-stream')].map(item => ({id:item.dataset.id, title:item.dataset.title, enabled:item.checked}));
const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(values)}); const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(values)});
document.getElementById('settingsResult').textContent = response.ok ? 'Saved. Monitor applies supported settings on its next cycle.' : 'Could not save configuration.'; document.getElementById('settingsResult').textContent = response.ok ? 'Saved. Monitor applies supported settings on its next cycle.' : 'Could not save configuration.';
if (response.ok) loadSettings(); if (response.ok) loadSettings();
@@ -208,6 +220,20 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
if self.path == "/api/config": if self.path == "/api/config":
self._send(200, "application/json", json.dumps(config_store.public()).encode("utf-8")) self._send(200, "application/json", json.dumps(config_store.public()).encode("utf-8"))
return return
if self.path == "/api/graylog/streams":
config = config_store.read()
try:
client = GraylogMcpClient(str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")))
client.probe()
result = client.call_tool("list_streams", {})
content = result.get("result", {}).get("content", [])
text = next((item.get("text", "") for item in content if isinstance(item, dict)), "")
streams = json.loads(text)
body = {"streams": streams, "selected": config.get("graylog_streams", [])}
self._send(200, "application/json", json.dumps(body).encode("utf-8"))
except Exception as exc:
self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
return
if self.path == "/api/status": if self.path == "/api/status":
if status_path.exists(): if status_path.exists():
body = status_path.read_bytes() body = status_path.read_bytes()

41
src/fgai/event_context.py Normal file
View File

@@ -0,0 +1,41 @@
from __future__ import annotations
from collections import Counter, defaultdict
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
def _entity(event: LogEvent) -> str:
return event.src_ip or event.fields.get("user") or event.fields.get("username") or event.fields.get("hostname") or event.fields.get("source") or "unknown"
def build_event_context(events: list[LogEvent], *, source_limit: int = 30, sample_limit: int = 15) -> dict[str, object]:
grouped: dict[str, list[LogEvent]] = defaultdict(list)
field_presence: Counter[str] = Counter()
for event in events:
grouped[_entity(event)].append(event)
field_presence.update(event.fields.keys())
source_profiles = []
for entity, source_events in grouped.items():
actions = Counter(event.action or "unknown" for event in source_events)
destinations = {event.dst_ip for event in source_events if event.dst_ip}
source_profiles.append({
"entity": entity, "events": len(source_events), "utm_events": sum(is_utm_event(event) for event in source_events),
"deny_or_threat_actions": sum(event.action in THREAT_ACTIONS for event in source_events),
"distinct_destinations": len(destinations), "top_actions": [action for action, _ in actions.most_common(3)],
})
source_profiles.sort(key=lambda item: (int(item["utm_events"]) + int(item["deny_or_threat_actions"]), int(item["events"])), reverse=True)
suspicious = [event for event in events if is_utm_event(event) or event.action in THREAT_ACTIONS or event.severity in {"critical", "high", "alert", "emergency"}]
samples = [{
"entity": _entity(event), "type": event.fields.get("type", ""), "subtype": event.subtype,
"action": event.action, "severity": event.severity, "dst": event.dst_ip or "", "service": event.fields.get("service", ""),
"policyid": event.fields.get("policyid", ""), "timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")),
} for event in suspicious[:sample_limit]]
return {
"entities_total": len(grouped), "source_profiles": source_profiles[:source_limit],
"field_coverage": [{"field": field, "events": count} for field, count in field_presence.most_common(30)],
"security_event_samples": samples,
}

13
src/fgai/llm.py
View File

@@ -16,7 +16,7 @@ def ollama_summary(
timeout: int | None = None, timeout: int | None = None,
) -> str: ) -> str:
host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/") host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
selected_model = model or os.getenv("OLLAMA_MODEL", "llama3.3") selected_model = model or os.getenv("OLLAMA_MODEL", "llama3.1")
selected_timeout = timeout or int(os.getenv("OLLAMA_TIMEOUT", "180")) selected_timeout = timeout or int(os.getenv("OLLAMA_TIMEOUT", "180"))
prompt = { prompt = {
"analysis": analysis or {}, "analysis": analysis or {},
@@ -35,8 +35,8 @@ def ollama_summary(
"temperature": 0.2, "temperature": 0.2,
}, },
"prompt": ( "prompt": (
"You are a local FortiGate security analyst. Summarize these policy findings " "You are a local security operations analyst. Analyze normalized events from one or more log sources. "
"and log diagnostics. Be concise. Include risk, likely cause, and next action. " "Be concise. Include risk, likely cause, affected entities, evidence, and next action. "
"Do not recommend blocking private/internal client IPs unless the data explicitly proves compromise. " "Do not recommend blocking private/internal client IPs unless the data explicitly proves compromise. "
f"Data: {json.dumps(prompt)}" f"Data: {json.dumps(prompt)}"
), ),
@@ -56,6 +56,9 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
"top_recommendations": analysis.get("recommendations", [])[:5], "top_recommendations": analysis.get("recommendations", [])[:5],
"block_candidates": analysis.get("block_candidates", [])[:5], "block_candidates": analysis.get("block_candidates", [])[:5],
"policy_findings": analysis.get("policy_findings", [])[:5], "policy_findings": analysis.get("policy_findings", [])[:5],
"event_context": analysis.get("event_context", {}),
"diagnostics": analysis.get("diagnostics", {}),
"capabilities": analysis.get("capabilities", {}),
} }
return ollama_summary( return ollama_summary(
[], [],
@@ -63,8 +66,8 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
model, model,
analysis={ analysis={
"task": ( "task": (
"Write a concise dashboard analyst note for a FortiGate admin. " "Write a concise dashboard analyst note. Compare activity across every listed entity, "
"Explain likely cause, whether this looks malicious or noisy, and the next action. " "identify the most unusual entity or behavior, and state the next investigation step. "
"Mention policyid=0 as implicit deny/drop, not an editable policy." "Mention policyid=0 as implicit deny/drop, not an editable policy."
), ),
"data": compact, "data": compact,

14
src/fgai/monitor.py
View File

@@ -7,6 +7,7 @@ from pathlib import Path
from .anomaly import anomaly_summary, detect_source_anomalies from .anomaly import anomaly_summary, detect_source_anomalies
from .baseline import BaselineStore from .baseline import BaselineStore
from .config import ConfigStore from .config import ConfigStore
from .event_context import build_event_context
from .graylog_mcp import GraylogMcpClient from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource from .graylog_source import GraylogStreamSource
from .llm import ollama_dashboard_assessment from .llm import ollama_dashboard_assessment
@@ -40,7 +41,17 @@ def build_status(
events = [] events = []
else: else:
try: try:
events, mcp_status = GraylogStreamSource(GraylogMcpClient(url, token), str(runtime_values.get("graylog_stream", "")), str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", ""))).fetch() configured_streams = runtime_values.get("graylog_streams", [])
stream_ids = [str(item.get("id")) for item in configured_streams if isinstance(item, dict) and item.get("enabled") and item.get("id")]
if not stream_ids:
stream_ids = [str(runtime_values.get("graylog_stream", ""))]
stream_statuses = []
events = []
for stream_id in stream_ids:
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", ""))).fetch()
events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, **stream_status})
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events)}
except RuntimeError as exc: except RuntimeError as exc:
mcp_status = {"status": "error", "error": str(exc)} mcp_status = {"status": "error", "error": str(exc)}
events = [] events = []
@@ -95,6 +106,7 @@ def build_status(
"top_subtypes": top_field_values(events, "subtype", limit=10), "top_subtypes": top_field_values(events, "subtype", limit=10),
"local_in_failures": local_in_failures(events, limit=10), "local_in_failures": local_in_failures(events, limit=10),
}, },
"event_context": build_event_context(events),
"anomalies": [ "anomalies": [
{ {
"subject": finding.subject, "subject": finding.subject,

20
tests/test_event_context.py Normal file
View File

@@ -0,0 +1,20 @@
import unittest
from fgai.event_context import build_event_context
from fgai.logs import parse_log_line
class EventContextTests(unittest.TestCase):
def test_includes_each_source_and_security_sample(self):
events = [
parse_log_line("srcip=10.0.0.1 dstip=1.1.1.1 action=accept"),
parse_log_line("srcip=10.0.0.2 dstip=8.8.8.8 type=utm subtype=ips action=blocked severity=high"),
]
context = build_event_context(events)
self.assertEqual(context["entities_total"], 2)
self.assertEqual(context["source_profiles"][0]["entity"], "10.0.0.2")
self.assertEqual(len(context["security_event_samples"]), 1)
if __name__ == "__main__":
unittest.main()