Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added local Ollama model listing in the U

Browse Source
This commit is contained in:
larssand
2026-06-30 09:15:03 +02:00
parent d7b8c494e9
commit 9c6c08b5ec
3 changed files with 82 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -72,6 +72,10 @@ validated against fields actually seen in the stream before it can be applied.
Unknown fields, raw message fields, internal `fgai_*` fields, and unknown Unknown fields, raw message fields, internal `fgai_*` fields, and unknown
detectors are rejected. detectors are rejected.
Settings also lists locally installed Ollama models from `http://127.0.0.1:11434/api/tags`.
Click a model name to fill both the dashboard analyst model and profile advisor
model fields.
The settings page treats stream enablement and profile editing separately. The The settings page treats stream enablement and profile editing separately. The
checkboxes decide which streams are monitored. Click `Edit profile` on one stream checkboxes decide which streams are monitored. Click `Edit profile` on one stream
to load its fields and edit only that stream's profile; saving with no active to load its fields and edit only that stream's profile; saving with no active

51
src/fgai/dashboard.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import json import json
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path from pathlib import Path
from urllib import error, request
from urllib.parse import parse_qs, urlparse from urllib.parse import parse_qs, urlparse
from .config import ConfigStore from .config import ConfigStore
@@ -13,6 +14,30 @@ from .feedback import FeedbackStore
from .incidents import IncidentStore from .incidents import IncidentStore
def _ollama_models(host: str = "http://127.0.0.1:11434") -> dict[str, object]:
try:
req = request.Request(f"{host.rstrip('/')}/api/tags", method="GET")
with request.urlopen(req, timeout=5) as response:
payload = json.loads(response.read().decode("utf-8"))
except error.URLError as exc:
return {"status": "error", "error": str(exc), "models": []}
except (json.JSONDecodeError, OSError) as exc:
return {"status": "error", "error": str(exc), "models": []}
models = payload.get("models", [])
if not isinstance(models, list):
models = []
output = []
for item in models:
if not isinstance(item, dict):
continue
output.append({
"name": str(item.get("name", "")),
"modified_at": str(item.get("modified_at", "")),
"size": int(item.get("size", 0) or 0),
})
return {"status": "ok", "models": sorted(output, key=lambda model: model["name"])}
HTML = """<!doctype html> HTML = """<!doctype html>
<html lang="en"> <html lang="en">
<head> <head>
@@ -72,6 +97,8 @@ HTML = """<!doctype html>
.graph { width: 100%; height: 300px; background: #04182d; border: 1px solid #163b59; } .graph { width: 100%; height: 300px; background: #04182d; border: 1px solid #163b59; }
.sort-button { border: 0; background: transparent; color: #83bce9; cursor: pointer; font: inherit; font-weight: 600; padding: 0; } .sort-button { border: 0; background: transparent; color: #83bce9; cursor: pointer; font: inherit; font-weight: 600; padding: 0; }
.sort-button:hover { color: #d9e8f7; } .sort-button:hover { color: #d9e8f7; }
.model-list { display: flex; flex-wrap: wrap; gap: 8px; }
.model-pill { border: 1px solid #39709a; background: #08243e; color: #d9e8f7; padding: 5px 8px; cursor: pointer; }
@media (max-width: 860px) { .hero, .split { grid-template-columns: 1fr; } .hero img { display: none; } } @media (max-width: 860px) { .hero, .split { grid-template-columns: 1fr; } .hero img { display: none; } }
</style> </style>
</head> </head>
@@ -89,7 +116,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div> <div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div> <div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div> <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div> <div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Installed Ollama Models</h2><div id="ollamaModels" class="muted">Loading local Ollama models.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main> </main>
<script> <script>
function esc(value) { function esc(value) {
@@ -335,6 +362,24 @@ async function loadSettings() {
vtKey.placeholder = config.virustotal_api_key_configured ? 'Key configured; leave blank to keep it' : 'Paste VirusTotal API key'; vtKey.placeholder = config.virustotal_api_key_configured ? 'Key configured; leave blank to keep it' : 'Paste VirusTotal API key';
window.streamProfiles = config.graylog_stream_profiles || []; window.streamProfiles = config.graylog_stream_profiles || [];
if (config.graylog_mcp_token_configured && config.graylog_mcp_url) loadStreams(); if (config.graylog_mcp_token_configured && config.graylog_mcp_url) loadStreams();
loadOllamaModels();
}
async function loadOllamaModels() {
const target = document.getElementById('ollamaModels');
try {
const payload = await (await fetch('/api/ollama/models', {cache: 'no-store'})).json();
if (!payload.models || payload.models.length === 0) {
target.textContent = payload.error || 'No local Ollama models found.';
return;
}
target.innerHTML = `<div class="model-list">${payload.models.map(model => `<button type="button" class="model-pill" data-model="${esc(model.name)}">${esc(model.name)}</button>`).join('')}</div><div class="muted">Click a model to fill Ollama model and profile advisor model fields.</div>`;
target.querySelectorAll('.model-pill').forEach(button => button.addEventListener('click', () => {
document.querySelector('[name="llm_model"]').value = button.dataset.model;
document.querySelector('[name="profile_advisor_model"]').value = button.dataset.model;
}));
} catch (error) {
target.textContent = `Could not load Ollama models: ${error}`;
}
} }
async function loadStreams() { async function loadStreams() {
const response = await fetch('/api/graylog/streams'); const response = await fetch('/api/graylog/streams');
@@ -407,6 +452,7 @@ document.getElementById('settingsForm').addEventListener('submit', async event =
const form = event.currentTarget; const form = event.currentTarget;
const values = Object.fromEntries(new FormData(form)); const values = Object.fromEntries(new FormData(form));
values.llm_enabled = form.elements.llm_enabled.checked; values.llm_enabled = form.elements.llm_enabled.checked;
values.profile_advisor_enabled = form.elements.profile_advisor_enabled.checked;
values.threat_intel_enabled = form.elements.threat_intel_enabled.checked; values.threat_intel_enabled = form.elements.threat_intel_enabled.checked;
values.graylog_streams = [...document.querySelectorAll('.graylog-stream')].map(item => ({id:item.dataset.id, title:item.dataset.title, enabled:item.checked})); values.graylog_streams = [...document.querySelectorAll('.graylog-stream')].map(item => ({id:item.dataset.id, title:item.dataset.title, enabled:item.checked}));
if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const entityFields=[...form.querySelectorAll('.profile-entity:checked')].map(item=>item.value); if (!entityFields.length) { document.getElementById('settingsResult').textContent='Select at least one Entity field for the active profile.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:entityFields[0]||'',entity_fields:entityFields,timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; } if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const entityFields=[...form.querySelectorAll('.profile-entity:checked')].map(item=>item.value); if (!entityFields.length) { document.getElementById('settingsResult').textContent='Select at least one Entity field for the active profile.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:entityFields[0]||'',entity_fields:entityFields,timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
@@ -448,6 +494,9 @@ def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str |
if self.path == "/api/incidents": if self.path == "/api/incidents":
self._send(200, "application/json", json.dumps(IncidentStore().entries()).encode("utf-8")) self._send(200, "application/json", json.dumps(IncidentStore().entries()).encode("utf-8"))
return return
if self.path == "/api/ollama/models":
self._send(200, "application/json", json.dumps(_ollama_models()).encode("utf-8"))
return
if self.path == "/api/graylog/streams": if self.path == "/api/graylog/streams":
config = config_store.read() config = config_store.read()
try: try:

28
tests/test_dashboard.py Normal file
View File

@@ -0,0 +1,28 @@
import json
import unittest
from unittest.mock import patch
from fgai.dashboard import _ollama_models
class DashboardTests(unittest.TestCase):
def test_ollama_models_returns_sorted_model_names(self):
class Response:
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self):
return json.dumps({"models": [{"name": "qwen3:8b", "size": 2}, {"name": "llama3.1", "size": 1}]}).encode("utf-8")
with patch("fgai.dashboard.request.urlopen", return_value=Response()):
result = _ollama_models()
self.assertEqual(result["status"], "ok")
self.assertEqual([item["name"] for item in result["models"]], ["llama3.1", "qwen3:8b"])
if __name__ == "__main__":
unittest.main()