from __future__ import annotations import json from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib import error, request from urllib.parse import parse_qs, urlparse from .config import ConfigStore from .exports import investigation_report, investigation_report_markdown from .graylog_mcp import GraylogMcpClient from .metrics import prometheus_metrics from .feedback import FeedbackStore from .incidents import IncidentStore def _ollama_models(host: str = "http://127.0.0.1:11434") -> dict[str, object]: try: req = request.Request(f"{host.rstrip('/')}/api/tags", method="GET") with request.urlopen(req, timeout=5) as response: payload = json.loads(response.read().decode("utf-8")) except error.URLError as exc: return {"status": "error", "error": str(exc), "models": []} except (json.JSONDecodeError, OSError) as exc: return {"status": "error", "error": str(exc), "models": []} models = payload.get("models", []) if not isinstance(models, list): models = [] output = [] for item in models: if not isinstance(item, dict): continue output.append({ "name": str(item.get("name", "")), "modified_at": str(item.get("modified_at", "")), "size": int(item.get("size", 0) or 0), }) return {"status": "ok", "models": sorted(output, key=lambda model: model["name"])} HTML = """ SignalScope Monitor

SignalScope Monitor

FortiGate AI/ML Analyzer

Live Status

Waiting for monitor data.

Events and Anomalies

Baseline and Stream Health

Operator Guidance

Waiting for monitor data.

Correlation Map

Waiting for correlated entities.

Investigation Incidents

Anomalies

Recommendations

AI Assessment

LLM assessment disabled.

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

How To Use SignalScope

1. Normal workflow

  1. Settings: connect Graylog MCP and enable streams.
  2. Settings: apply recommended profiles for missing streams.
  3. Diagnostics: confirm raw samples, aggregate counts, and profile readiness.
  4. Overview: use Operator Guidance, incidents, trends, and correlation map.
  5. Findings: review only high-signal deviations first, then mark decisions.

2. Stream health

  • ready: profile exists, events are arriving, and baseline fields are ready.
  • learning: profile exists and events arrive, but baseline age or buckets are still too low.
  • missing_profile: stream is enabled but no profile exists. Apply or edit one.
  • no_events: stream is enabled but the current poll has no raw sample events.
  • partial_fetch: Graylog returned only part of the requested raw sample.

3. Ready fields

0/8 means 8 profile fields are tracked but none are mature enough yet. A field needs at least 12 baseline buckets and the configured Baseline training days before it is ready.

During learning, treat findings as signals to tune profiles, not as final alerts.

4. Profiles

  • Entity fields define who or what behavior is tracked, such as user, host, source IP, or application actor.
  • Baseline fields define the changing behavior to learn, such as action, event ID, service, URL, status, or counters.
  • Relationships learn pairs such as username to srcip or host to process.
  • Detectors add burst checks for auth failures, DNS queries, and deny actions.

5. Findings

  • Start with Triage Queue and incidents, not raw long tables.
  • Open evidence details before confirming a finding.
  • Use Expected for known behavior, False positive for bad signal, Confirmed for real investigation items.
  • Use expiry when a behavior is expected only temporarily.

6. High EPS / MCP

  • Use aggregate or auto fetch mode for high EPS streams.
  • Keep raw samples small enough for context; aggregate counts represent the full window.
  • Sample capped is normal in aggregate mode. Truncated raw mode means you may miss context.
  • If MCP is stale, the UI shows cached status so you can still inspect previous findings.

7. Ollama

  • Dashboard assessment summarizes current evidence.
  • Profile advisor maps unknown/custom fields and suggests relationships.
  • Ollama advice is constrained to fields discovered from Graylog; unknown fields are rejected.

8. Baseline DB size

  • Baseline DB size is the local SQLite behavior baseline.
  • Retention deletes old rows, but SQLite only returns disk space after manual VACUUM.
  • If the DB grows fast, lower bucket retention, value retention, or max values per entity field in Settings.
  • Stop the service before running signalscope baseline-maintenance --vacuum.

9. What to fix first

  1. No streams enabled.
  2. Enabled streams with missing profiles.
  3. Enabled streams with zero raw events.
  4. Profiles stuck at 0 ready fields after the training window.
  5. Too many repeated findings without review feedback.
  6. Baseline DB growing without maintenance.
""" def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | None = None, config_file: str = "state/fgai-config.json") -> None: status_path = Path(status_file) image_root = Path(image_dir) if image_dir else None config_store = ConfigStore(config_file) def read_status() -> dict[str, object]: try: status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {} except json.JSONDecodeError: status = {} if not isinstance(status, dict): status = {} status.setdefault("status_schema", 2) for row in status.get("stream_coverage", []) if isinstance(status.get("stream_coverage"), list) else []: if isinstance(row, dict): row.setdefault("aggregate_error", "") row.setdefault("raw_error", "") if row.get("aggregate_status") == "error" and not row.get("aggregate_error") and row.get("error"): row["aggregate_error"] = str(row.get("error", "")) if row.get("partial") and not row.get("raw_error") and row.get("error"): row["raw_error"] = str(row.get("error", "")) return status class Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: if self.path == "/": self._send(200, "text/html; charset=utf-8", HTML.encode("utf-8")) return if self.path == "/api/config": self._send(200, "application/json", json.dumps(config_store.public()).encode("utf-8")) return if self.path == "/api/feedback": self._send(200, "application/json", json.dumps(FeedbackStore().entries()).encode("utf-8")) return if self.path == "/api/incidents": self._send(200, "application/json", json.dumps(IncidentStore().entries()).encode("utf-8")) return if self.path == "/api/ollama/models": self._send(200, "application/json", json.dumps(_ollama_models()).encode("utf-8")) return if self.path == "/api/graylog/streams": config = config_store.read() try: client = GraylogMcpClient(str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")), verify_tls=bool(config.get("graylog_tls_verify", True))) client.probe() result = client.call_tool("list_streams", {}) content = result.get("result", {}).get("content", []) text = next((item.get("text", "") for item in content if isinstance(item, dict)), "") if isinstance(result.get("result"), dict) and result["result"].get("isError"): fallback = client.call_tool("list_resource", {"resource_type": "stream"}) resources = fallback.get("result", {}).get("structuredContent", {}).get("resources", []) streams = [ {"id": str(item.get("uri", "")).rsplit(":", 1)[-1], "title": item.get("title", item.get("name", "")), "description": item.get("description", "")} for item in resources if isinstance(item, dict) and item.get("uri") ] else: if not text: raise RuntimeError("Graylog list_streams returned no text content") streams = json.loads(text) body = {"streams": streams, "selected": config.get("graylog_streams", [])} self._send(200, "application/json", json.dumps(body).encode("utf-8")) except Exception as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return if self.path.startswith("/api/graylog/fields?"): stream_id = self.path.split("stream_id=", 1)[-1].split("&", 1)[0] config = config_store.read() try: client = GraylogMcpClient(str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")), verify_tls=bool(config.get("graylog_tls_verify", True))) client.probe() result = client.call_tool("list_fields", {"streams": [stream_id]}) content = result.get("result", {}).get("content", []) text = next((item.get("text", "") for item in content if isinstance(item, dict)), "") payload = json.loads(text) fields = payload.get("fields", payload) if isinstance(payload, dict) else payload self._send(200, "application/json", json.dumps({"fields": fields}).encode("utf-8")) except Exception as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return if self.path == "/api/status": self._send(200, "application/json", json.dumps(read_status()).encode("utf-8")) return if self.path.startswith("/api/export/incidents"): parsed = urlparse(self.path) params = parse_qs(parsed.query) fmt = params.get("format", ["markdown"])[0] incident_id = params.get("incident_id", [""])[0] or None status = read_status() report = investigation_report(status, incident_id=incident_id) if fmt == "json": self._send(200, "application/json", json.dumps(report, indent=2, sort_keys=True).encode("utf-8")) else: self._send(200, "text/markdown; charset=utf-8", investigation_report_markdown(report).encode("utf-8")) return if self.path == "/metrics": status = read_status() self._send(200, "text/plain; version=0.0.4; charset=utf-8", prometheus_metrics(status).encode("utf-8")) return if self.path.startswith("/images/") and image_root: image_path = image_root / Path(self.path).name if image_path.exists() and image_path.is_file(): content_type = "image/png" if image_path.suffix.lower() == ".png" else "application/octet-stream" self._send(200, content_type, image_path.read_bytes()) return self._send(404, "text/plain; charset=utf-8", b"not found") def do_POST(self) -> None: if self.path == "/api/feedback" and self._is_loopback_client(): try: payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8")) self._send(200, "application/json", json.dumps(FeedbackStore().add(payload)).encode("utf-8")) except (ValueError, json.JSONDecodeError) as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return if self.path == "/api/incidents" and self._is_loopback_client(): try: payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8")) self._send(200, "application/json", json.dumps(IncidentStore().update(str(payload.get("id", "")), str(payload.get("status", "")), str(payload.get("note", "")))).encode("utf-8")) except (ValueError, json.JSONDecodeError) as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return if self.path != "/api/config" or not self._is_loopback_client(): self._send(403, "application/json", b'{"error":"configuration is local-only"}') return try: length = int(self.headers.get("Content-Length", "0")) payload = json.loads(self.rfile.read(min(length, 2_097_152)).decode("utf-8")) if not isinstance(payload, dict): raise ValueError("configuration must be an object") body = json.dumps(config_store.update(payload)).encode("utf-8") except (ValueError, json.JSONDecodeError) as exc: self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) return self._send(200, "application/json", body) def _is_loopback_client(self) -> bool: return self.client_address[0] in {"127.0.0.1", "::1"} def log_message(self, format: str, *args: object) -> None: return def _send(self, status: int, content_type: str, body: bytes) -> None: self.send_response(status) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) server = ThreadingHTTPServer((host, port), Handler) print(f"Dashboard listening on http://{host}:{port}") print(f"Reading status from {status_path}") server.serve_forever()