from __future__ import annotations

import json
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib import error, request
from urllib.parse import parse_qs, urlparse

from .config import ConfigStore
from .exports import investigation_report, investigation_report_markdown
from .graylog_mcp import GraylogMcpClient
from .metrics import prometheus_metrics
from .feedback import FeedbackStore
from .incidents import IncidentStore


def _ollama_models(host: str = "http://127.0.0.1:11434") -> dict[str, object]:
    try:
        req = request.Request(f"{host.rstrip('/')}/api/tags", method="GET")
        with request.urlopen(req, timeout=5) as response:
            payload = json.loads(response.read().decode("utf-8"))
    except error.URLError as exc:
        return {"status": "error", "error": str(exc), "models": []}
    except (json.JSONDecodeError, OSError) as exc:
        return {"status": "error", "error": str(exc), "models": []}
    models = payload.get("models", [])
    if not isinstance(models, list):
        models = []
    output = []
    for item in models:
        if not isinstance(item, dict):
            continue
        output.append({
            "name": str(item.get("name", "")),
            "modified_at": str(item.get("modified_at", "")),
            "size": int(item.get("size", 0) or 0),
        })
    return {"status": "ok", "models": sorted(output, key=lambda model: model["name"])}


HTML = """<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>SignalScope Monitor</title>
  <style>
    :root { color-scheme: dark; font-family: Arial, sans-serif; background: #020d1b; color: #d9e8f7; }
    body { margin: 0; background: radial-gradient(circle at 20% 0%, #062448 0, #020d1b 36%, #020914 100%); }
    header { background: rgba(4,24,45,.86); color: white; padding: 10px 24px; border-bottom: 1px solid #1c4b70; backdrop-filter: blur(10px); }
    h1 { margin: 0; font-size: 22px; }
    main { padding: 18px; max-width: 1760px; margin: 0 auto; }
    .app-shell { display: grid; grid-template-columns: minmax(300px, 420px) minmax(0, 1fr); gap: 18px; align-items: start; }
    .brand-panel { position: sticky; top: 16px; display: grid; gap: 12px; }
    .brand-mark { display: grid; grid-template-columns: 64px 1fr; gap: 14px; align-items: center; }
    .shield { width: 58px; height: 58px; border: 2px solid #338dff; border-radius: 18px; display: grid; place-items: center; color: #62b7ff; font-size: 26px; box-shadow: inset 0 0 24px rgba(51,141,255,.18); }
    .brand-title { font-size: 40px; line-height: .95; font-weight: 800; color: #f7fbff; }
    .brand-subtitle { color: #21b8ff; letter-spacing: .18em; font-size: 12px; margin-top: 6px; text-transform: uppercase; }
    .brand-copy { color: #cfe0ef; font-size: 16px; line-height: 1.45; margin: 6px 0; }
    .brand-copy b { color: #27b9ff; }
    .feature-card { display: grid; grid-template-columns: 48px 1fr; gap: 12px; align-items: center; border: 1px solid #1c4b70; border-radius: 8px; padding: 12px; background: linear-gradient(135deg, rgba(7,29,51,.92), rgba(3,20,38,.86)); }
    .feature-icon { width: 38px; height: 38px; border: 1px solid #2b7fd0; border-radius: 50%; display: grid; place-items: center; color: #27d3ff; }
    .feature-card h3 { margin: 0 0 4px; color: #27b9ff; font-size: 13px; letter-spacing: .08em; text-transform: uppercase; }
    .feature-card p { margin: 0; color: #c3d5e5; font-size: 13px; line-height: 1.35; }
    .flow-card { border: 1px solid #1c4b70; border-radius: 8px; padding: 12px; background: rgba(4,24,45,.72); }
    .flow-card h3 { margin: 0 0 10px; text-align: center; font-size: 13px; color: #dceeff; letter-spacing: .06em; text-transform: uppercase; }
    .flow-steps { display: grid; grid-template-columns: repeat(4, 1fr); gap: 8px; align-items: stretch; }
    .flow-steps div { border: 1px solid #236291; border-radius: 6px; padding: 10px 6px; text-align: center; font-size: 11px; color: #c6d6e6; background: #061a2e; }
    .local-note { color: #91abc4; font-size: 12px; border-top: 1px solid #163b59; padding-top: 10px; }
    .workspace { min-width: 0; border: 1px solid #1c4b70; border-radius: 10px; padding: 12px; background: rgba(4,19,36,.72); box-shadow: 0 20px 60px rgba(0,0,0,.28); }
    .hero { display: grid; grid-template-columns: 1fr; gap: 12px; align-items: stretch; }
    .hero img { display: none; }
    .hero .panel { margin-bottom: 0; }
    .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; }
    .panel { background: linear-gradient(180deg, rgba(8,31,56,.94), rgba(5,24,44,.94)); border: 1px solid #1c4b70; border-radius: 8px; padding: 14px; margin-bottom: 12px; box-shadow: 0 8px 24px rgba(0,0,0,.16); }
    .metric { font-size: 28px; font-weight: 700; }
    .label, .muted { color: #91abc4; font-size: 13px; margin-top: 4px; }
    table { width: 100%; border-collapse: collapse; font-size: 14px; }
    th, td { border-bottom: 1px solid #163b59; padding: 8px; text-align: left; vertical-align: top; }
    th { color: #83bce9; font-weight: 600; }
    .sev-critical { color: #b00020; font-weight: 700; }
    .sev-high { color: #b54708; font-weight: 700; }
    .sev-medium { color: #8a6d00; font-weight: 700; }
    .sev-low { color: #345995; font-weight: 700; }
    code { background: #0b2944; color: #b9e3ff; padding: 2px 4px; border-radius: 4px; }
    .capabilities { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 8px; }
    .capability { border: 1px solid #1c4b70; background: #08243e; padding: 5px 8px; font-size: 12px; }
    .capability.on { border-color: #1d9b72; color: #72e2ae; background: #082f2b; }
    .capability.warn { border-color: #c89436; color: #ffd36e; background: #33260b; }
    .tabs { display: flex; border-bottom: 1px solid #1c4b70; margin: 14px 0 12px; gap: 4px; }
    .tab { border: 0; border-bottom: 3px solid transparent; background: transparent; padding: 10px 14px; color: #91abc4; cursor: pointer; }
    .tab.active { border-bottom-color: #1ea9ff; color: #f2f8ff; font-weight: 700; }
    [data-view] { display: none; } [data-view].active { display: block; }
    .split { display: grid; grid-template-columns: minmax(0, 1.3fr) minmax(320px, 0.7fr); gap: 12px; }
    .table-wrap { overflow-x: auto; }
    #settingsForm label:has(#streamPicker), #settingsForm label:has(#fieldPicker) { grid-column: 1 / -1; }
    #streamPicker { margin-top: 8px; border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 10px; }
    .stream-tools { display: grid; grid-template-columns: minmax(240px, 1fr) repeat(3, auto); gap: 8px; align-items: center; margin-bottom: 10px; }
    .stream-tools input[type="search"] { width: 100%; }
    .stream-list { display: grid; grid-template-columns: repeat(auto-fill, minmax(310px, 1fr)); gap: 6px; max-height: 430px; overflow: auto; padding-right: 4px; }
    .stream-counts { color: #91abc4; font-size: 12px; margin-bottom: 8px; }
    #fieldPicker { margin-top: 8px; max-height: 460px; overflow: auto; border: 1px solid #d9e0e7; background: #fbfcfd; }
    .field-header, .field-row { display: grid; grid-template-columns: minmax(190px, 1.2fr) 100px minmax(180px, 1fr) minmax(320px, 1.4fr); gap: 10px; align-items: center; padding: 8px 10px; }
    .field-header { position: sticky; top: 0; background: #edf3f7; color: #536170; font-size: 12px; font-weight: 700; z-index: 1; }
    .field-row { border-top: 1px solid #e4e9ee; font-size: 13px; }
    .field-row code { width: fit-content; }
    .field-controls { display: flex; flex-wrap: wrap; gap: 10px; }
    .field-controls label { white-space: nowrap; }
    .stream-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; align-items: center; gap: 8px; padding: 7px 8px; border: 1px solid #163b59; background: #071d33; border-radius: 4px; }
    .stream-row label { min-width: 0; display: flex; align-items: center; gap: 7px; }
    .stream-title { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
    .stream-row button, .stream-tools button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 5px 8px; cursor: pointer; white-space: nowrap; }
    .toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 8px 0 12px; color: #91abc4; }
    .toolbar label { display: inline-flex; align-items: center; gap: 6px; }
    .summary-card { border: 1px solid #163b59; background: #061a2e; padding: 10px; margin-bottom: 8px; border-radius: 6px; }
    .summary-card h3 { margin: 0 0 6px; font-size: 16px; }
    .review-actions { display: flex; flex-wrap: wrap; gap: 6px; min-width: 250px; }
    .review-actions button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 6px 8px; cursor: pointer; }
    .review-actions button[data-status="false_positive"] { border-color: #b7823a; color: #ffd36e; }
    .review-actions button[data-status="confirmed"] { border-color: #2a9b6e; color: #7be3ae; }
    .chart { width: 100%; height: 280px; background: #04182d; border: 1px solid #163b59; }
    .graph { width: 100%; height: 500px; background: #04182d; border: 1px solid #163b59; }
    .correlation-layout { display: grid; grid-template-columns: minmax(0, 1.6fr) minmax(340px, .8fr); gap: 12px; align-items: start; }
    .entity-picker { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 10px; }
    .entity-chip { border: 1px solid #39709a; background: #08243e; color: #d9e8f7; padding: 6px 8px; cursor: pointer; border-radius: 4px; }
    .entity-chip.active { border-color: #1ea9ff; color: #f2f8ff; background: #0b3358; }
    .entity-summary { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 8px; margin: 10px 0; }
    .entity-stat { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 8px; }
    .entity-stat b { display: block; color: #f2f8ff; font-size: 18px; }
    .entity-stat span { color: #91abc4; font-size: 12px; }
    .evidence-list { display: grid; gap: 8px; max-height: 420px; overflow: auto; padding-right: 4px; }
    .evidence-item { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 8px; }
    .evidence-head { display: flex; gap: 8px; justify-content: space-between; align-items: center; margin-bottom: 5px; }
    .evidence-stream { color: #83bce9; font-weight: 700; }
    .evidence-action { border: 1px solid #39709a; border-radius: 999px; padding: 2px 6px; color: #d9e8f7; font-size: 12px; }
    .evidence-context { color: #c7d7e7; font-size: 13px; line-height: 1.35; word-break: break-word; }
    .evidence-item code { display: inline-block; margin-top: 4px; }
    .ai-box { max-height: 260px; overflow: auto; }
    .ai-box details { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 8px; }
    .ai-box summary { cursor: pointer; color: #83bce9; font-weight: 700; }
    .sort-button { border: 0; background: transparent; color: #83bce9; cursor: pointer; font: inherit; font-weight: 600; padding: 0; }
    .sort-button:hover { color: #d9e8f7; }
    .model-list { display: flex; flex-wrap: wrap; gap: 8px; }
    .model-pill { border: 1px solid #39709a; background: #08243e; color: #d9e8f7; padding: 5px 8px; cursor: pointer; }
    .howto-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 12px; }
    .howto-card { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 12px; }
    .howto-card h3 { margin: 0 0 8px; font-size: 16px; color: #83bce9; }
    .howto-card ol, .howto-card ul { margin: 0; padding-left: 20px; }
    .howto-card li { margin: 6px 0; }
    .action-list { display: grid; gap: 8px; }
    .action-item { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 10px; }
    .action-item strong { color: #f2f8ff; }
    .action-item.warn { border-color: #b7823a; }
    .action-item.critical { border-color: #b00020; }
    .status-guide { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 8px; }
    .status-guide div { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 10px; }
    @media (max-width: 1100px) { .app-shell, .correlation-layout { grid-template-columns: 1fr; } .brand-panel { position: static; } .flow-steps { grid-template-columns: repeat(2, 1fr); } }
    @media (max-width: 860px) { .hero, .split { grid-template-columns: 1fr; } .hero img { display: none; } .brand-title { font-size: 32px; } }
  </style>
</head>
<body>
  <header><h1>SignalScope Monitor</h1><div id="stamp" class="muted"></div><div id="capabilities" class="capabilities"></div></header>
  <main>
    <section class="app-shell">
      <aside class="brand-panel" aria-label="SignalScope overview">
        <div class="brand-mark"><div class="shield">⌁</div><div><div class="brand-title">SignalScope</div><div class="brand-subtitle">Local Security Analytics Agent</div></div></div>
        <p class="brand-copy">Local multi-source analytics for <b>Graylog MCP</b>, stream profiles, behavior baselines, entity correlation, and local LLM explanations.</p>
        <div class="feature-card"><div class="feature-icon">MCP</div><div><h3>Multi-source integration</h3><p>Discovers Graylog streams and fields already available in your environment.</p></div></div>
        <div class="feature-card"><div class="feature-icon">P</div><div><h3>Stream profiles</h3><p>Normalizes entities, fields, relationships, and detectors per log source.</p></div></div>
        <div class="feature-card"><div class="feature-icon">B</div><div><h3>Baselines</h3><p>Learns normal rates, values, and field relationships before scoring deviations.</p></div></div>
        <div class="feature-card"><div class="feature-icon">C</div><div><h3>Entity correlation</h3><p>Connects users, hosts, IPs, services, and applications across streams.</p></div></div>
        <div class="feature-card"><div class="feature-icon">AI</div><div><h3>Local LLM explanations</h3><p>Uses Ollama locally to explain evidence and improve profile recommendations.</p></div></div>
        <div class="flow-card"><h3>How it works</h3><div class="flow-steps"><div>Data sources</div><div>Graylog MCP</div><div>SignalScope</div><div>Outputs</div></div><div class="local-note">All processing is performed locally. Your data stays in your environment.</div></div>
      </aside>
      <section class="workspace">
    <section class="hero">
      <img src="/images/FGinspectionagent.png" alt="FortiGate AI/ML Analyzer">
      <div>
        <section class="grid" id="metrics"></section>
        <section class="panel"><h2>Live Status</h2><div id="liveStatus" class="muted">Waiting for monitor data.</div></section>
      </div>
    </section>
    <nav class="tabs" aria-label="Dashboard views"><button class="tab active" data-tab="overview">Overview</button><button class="tab" data-tab="findings">Findings</button><button class="tab" data-tab="diagnostics">Diagnostics</button><button class="tab" data-tab="settings">Settings</button><button class="tab" data-tab="howto">How To</button></nav>
    <div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>Operator Guidance</h2><div id="actionPlan" class="muted">Waiting for monitor data.</div></section><section class="panel"><h2>Correlation Map</h2><div class="correlation-layout"><div><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div id="correlationExplorer" class="muted">Waiting for correlated entities.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div class="toolbar"><label><input id="showResolvedIncidents" type="checkbox"> show resolved</label><span id="incidentSummary"></span></div><div id="incidentNotice" class="muted" role="status"></div><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section></div>
    <div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
    <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
    <div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div class="toolbar"><label><input id="showExistingProfiles" type="checkbox"> show existing profiles</label><span id="profileSuggestionSummary"></span></div><div id="profileApplyResult" class="muted" role="status"></div><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Installed Ollama Models</h2><div id="ollamaModels" class="muted">Loading local Ollama models.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Behavior relationships (JSON)<br><textarea name="profile_relationship_fields" placeholder='[{"left":"username","right":"srcip","name":"user source IP"}]'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog fetch mode<br><select name="graylog_fetch_mode"><option value="auto">Auto aggregate</option><option value="aggregate">Aggregate + sample</option><option value="raw">Raw only</option></select></label><label>Graylog MCP poll window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="300"></label><label>Graylog max events per stream<br><input name="graylog_max_events_per_stream" type="number" min="1" step="1000" placeholder="5000"></label><label>Raw sample events per stream<br><input name="graylog_raw_sample_events" type="number" min="1" step="1000" placeholder="5000"></label><label>MCP call timeout seconds<br><input name="graylog_mcp_call_timeout_seconds" type="number" min="1" step="1" placeholder="8"></label><label>MCP poll budget seconds<br><input name="graylog_mcp_poll_timeout_seconds" type="number" min="60" step="30" placeholder="120"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label><input name="graylog_tls_verify" type="checkbox"> Verify Graylog TLS certificate</label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
    <div data-view="howto"><section class="panel"><h2>How To Use SignalScope</h2><div class="howto-grid"><div class="howto-card"><h3>1. Normal workflow</h3><ol><li>Settings: connect Graylog MCP and enable streams.</li><li>Settings: apply recommended profiles for missing streams.</li><li>Diagnostics: confirm raw samples, aggregate counts, and profile readiness.</li><li>Overview: use Operator Guidance, incidents, trends, and correlation map.</li><li>Findings: review only high-signal deviations first, then mark decisions.</li></ol></div><div class="howto-card"><h3>2. Stream health</h3><ul><li><b>ready</b>: profile exists, events are arriving, and baseline fields are ready.</li><li><b>learning</b>: profile exists and events arrive, but baseline age or buckets are still too low.</li><li><b>missing_profile</b>: stream is enabled but no profile exists. Apply or edit one.</li><li><b>no_events</b>: stream is enabled but the current poll has no raw sample events.</li><li><b>partial_fetch</b>: Graylog returned only part of the requested raw sample.</li></ul></div><div class="howto-card"><h3>3. Ready fields</h3><p><code>0/8</code> means 8 profile fields are tracked but none are mature enough yet. A field needs at least 12 baseline buckets and the configured Baseline training days before it is ready.</p><p>During learning, treat findings as signals to tune profiles, not as final alerts.</p></div><div class="howto-card"><h3>4. Profiles</h3><ul><li><b>Entity fields</b> define who or what behavior is tracked, such as user, host, source IP, or application actor.</li><li><b>Baseline fields</b> define the changing behavior to learn, such as action, event ID, service, URL, status, or counters.</li><li><b>Relationships</b> learn pairs such as username to srcip or host to process.</li><li><b>Detectors</b> add burst checks for auth failures, DNS queries, and deny actions.</li></ul></div><div class="howto-card"><h3>5. Findings</h3><ul><li>Start with Triage Queue and incidents, not raw long tables.</li><li>Open evidence details before confirming a finding.</li><li>Use Expected for known behavior, False positive for bad signal, Confirmed for real investigation items.</li><li>Use expiry when a behavior is expected only temporarily.</li></ul></div><div class="howto-card"><h3>6. High EPS / MCP</h3><ul><li>Use aggregate or auto fetch mode for high EPS streams.</li><li>Keep raw samples small enough for context; aggregate counts represent the full window.</li><li>Sample capped is normal in aggregate mode. Truncated raw mode means you may miss context.</li><li>If MCP is stale, the UI shows cached status so you can still inspect previous findings.</li></ul></div><div class="howto-card"><h3>7. Ollama</h3><ul><li>Dashboard assessment summarizes current evidence.</li><li>Profile advisor maps unknown/custom fields and suggests relationships.</li><li>Ollama advice is constrained to fields discovered from Graylog; unknown fields are rejected.</li></ul></div><div class="howto-card"><h3>8. What to fix first</h3><ol><li>No streams enabled.</li><li>Enabled streams with missing profiles.</li><li>Enabled streams with zero raw events.</li><li>Profiles stuck at 0 ready fields after the training window.</li><li>Too many repeated findings without review feedback.</li></ol></div></div></section></div>
      </section>
    </section>
  </main>
<script>
function esc(value) {
  return String(value ?? "").replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
}
function metric(label, value) {
  return `<div class="panel"><div class="metric">${esc(value)}</div><div class="label">${esc(label)}</div></div>`;
}
function bytes(value) { const n=Number(value||0); if (!n) return '0 B'; const units=['B','KB','MB','GB','TB']; const i=Math.min(units.length-1, Math.floor(Math.log(n)/Math.log(1024))); return `${(n/Math.pow(1024,i)).toFixed(i?1:0)} ${units[i]}`; }
function graylogBaseUrl(configuration) {
  let url = String(configuration.graylog_mcp_url || '');
  if (!url) return '';
  if (url.endsWith('/')) url = url.slice(0, -1);
  if (url.endsWith('/api/mcp')) url = url.slice(0, -8);
  else if (url.endsWith('/api')) url = url.slice(0, -4);
  return url;
}
function graylogEvidenceLink(query, configuration, label='Open in Graylog') {
  const base = graylogBaseUrl(configuration);
  if (!base || !query) return '';
  const href = `${base}/search?rangetype=relative&relative=3600&q=${encodeURIComponent(query)}`;
  return ` <a href="${esc(href)}" target="_blank" rel="noreferrer">${esc(label)}</a>`;
}
const tableSort = {};
const uiCache = {correlations: [], fieldRows: [], correlationHitboxes: [], selectedCorrelationKey: '', entityLabels: {}};
window.availableStreams = [];
window.streamSelection = {};
window.configuredStreams = [];
window.streamProfiles = [];
window.currentCorrelations = [];
window.currentConfiguration = {};
function table(rows, columns, id = '') {
  if (!rows || rows.length === 0) return '<p class="muted">No data.</p>';
  const sort = tableSort[id];
  const sorted = sort ? [...rows].sort((left, right) => { const a=left[sort.key] ?? '', b=right[sort.key] ?? ''; const numeric=Number(a), numericB=Number(b); const compare=Number.isFinite(numeric) && Number.isFinite(numericB) && String(a).trim() !== '' && String(b).trim() !== '' ? numeric-numericB : String(a).localeCompare(String(b)); return sort.direction * compare; }) : rows;
  const head = columns.map(c => `<th>${c.key && id ? `<button class="sort-button" data-sort-table="${esc(id)}" data-sort-key="${esc(c.key)}">${esc(c.label)}${sort && sort.key === c.key ? (sort.direction === 1 ? ' ▲' : ' ▼') : ''}</button>` : esc(c.label)}</th>`).join('');
  const body = sorted.map(row => `<tr>${columns.map(c => `<td>${c.render ? c.render(row) : esc(row[c.key])}</td>`).join('')}</tr>`).join('');
  return `<div class="table-wrap"><table><thead><tr>${head}</tr></thead><tbody>${body}</tbody></table></div>`;
}
function capability(label, state, detail) { return `<span class="capability ${state}">${esc(label)}: ${esc(detail)}</span>`; }
function mcpCapability(mcp, configuration, pollRunningSeconds=0) {
  if (configuration.log_source !== 'graylog_mcp') return {state: 'warn', detail: 'not selected'};
  const status = String(mcp.status || 'checking');
  if (status === 'connected') return {state: 'on', detail: 'connected'};
  if (status === 'partial') return {state: 'warn', detail: 'partial'};
  if (status === 'refreshing') {
    const pollBudget = Number(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 120);
    if (pollRunningSeconds > pollBudget) return {state: 'warn', detail: `poll over budget ${pollRunningSeconds}s`};
    const previous = String(mcp.previous_status || '');
    if (['connected', 'partial'].includes(previous) || Number(mcp.previous_events_fetched || mcp.events_fetched || 0) > 0 || Number(mcp.previous_aggregate_events || mcp.aggregate_events || 0) > 0) {
      const suffix = pollRunningSeconds ? `polling ${pollRunningSeconds}s` : 'polling';
      return {state: 'on', detail: previous === 'partial' ? `partial, ${suffix}` : `connected, ${suffix}`};
    }
    return {state: 'warn', detail: 'starting poll'};
  }
  return {state: 'warn', detail: status};
}
function llmCapability(llm, configuration) {
  const enabled = Boolean(llm.enabled || configuration.llm_enabled);
  if (!enabled) return {state: 'warn', detail: 'disabled'};
  const status = String(llm.status || 'starting');
  if (status === 'ok') return {state: 'on', detail: 'ok'};
  if (status === 'cached') return {state: 'on', detail: 'cached'};
  if (status === 'error' && String(llm.text || '').trim()) return {state: 'warn', detail: 'cached, assessment error'};
  if (status === 'error') return {state: 'warn', detail: 'assessment error'};
  return {state: 'warn', detail: status};
}
function actionItem(severity, title, detail, action) {
  return `<div class="action-item ${esc(severity)}"><strong>${esc(title)}</strong><div>${esc(detail)}</div><div class="muted">Next: ${esc(action)}</div></div>`;
}
function buildActionPlan({configuration, mcp, enabledStreams, streamsMissingProfile, pendingProfileSuggestions, displayedRawEvents, displayedAggregateEvents, fieldRows, incidents, correlations, baseline}) {
  const items = [];
  if (configuration.log_source === 'graylog_mcp' && (!enabledStreams || enabledStreams.length === 0)) {
    items.push(actionItem('critical', 'No streams enabled', 'SignalScope is connected to Graylog configuration, but no streams are selected for monitoring.', 'Open Settings, Load streams, tick the streams you want, then Save streams.'));
  }
  if (mcp.status === 'error') {
    items.push(actionItem('critical', 'Graylog MCP is not healthy', mcp.error || 'The monitor cannot currently query Graylog MCP.', 'Check token, URL, DNS/TLS, and Graylog MCP permissions. Cached data may still be shown.'));
  }
  if (streamsMissingProfile > 0) {
    items.push(actionItem('warn', 'Streams need profiles', `${streamsMissingProfile} enabled stream(s) do not have a usable profile.`, 'Open Settings, review Recommended Stream Profiles, then Apply or Update profile.'));
  }
  const learning = (enabledStreams || []).filter(item => item.profile_ready && item.health === 'learning');
  if (learning.length) {
    items.push(actionItem('warn', 'Baselines are still learning', `${learning.length} profiled stream(s) have fields that are not ready yet.`, 'Use Diagnostics -> Profile Baseline Readiness. Treat early findings as tuning signals until ready fields increase.'));
  }
  const noRaw = (enabledStreams || []).filter(item => item.enabled && Number(item.events_fetched || 0) === 0);
  if (noRaw.length && mcp.status !== 'refreshing') {
    items.push(actionItem('warn', 'Some streams have no raw sample', `${noRaw.length} enabled stream(s) returned no raw events in this poll.`, 'Check Graylog query, stream permissions, poll window, or whether the stream is quiet.'));
  }
  if (Number(mcp.truncated_streams || 0) > 0 || Number(mcp.partial_streams || 0) > 0) {
    items.push(actionItem('warn', 'Raw fetch is incomplete', `${mcp.partial_streams || 0} partial and ${mcp.truncated_streams || 0} truncated stream(s).`, 'Use aggregate or auto fetch mode with a smaller raw sample for high-EPS streams.'));
  }
  if (fieldRows.length > 0) {
    items.push(actionItem('warn', 'Findings need review', `${fieldRows.length} visible field deviation(s) need triage.`, 'Open Findings, expand evidence, then mark Expected, False positive, or Confirmed.'));
  }
  if ((incidents || []).length > 0) {
    items.push(actionItem('warn', 'Investigation incidents exist', `${incidents.length} incident(s) are grouped from anomalies, field deviations, and correlations.`, 'Start with Overview -> Investigation Incidents before looking through raw findings.'));
  }
  if (!items.length && displayedRawEvents > 0) {
    items.push(actionItem('', 'System looks usable', `${displayedRawEvents.toLocaleString()} raw sample event(s), ${displayedAggregateEvents.toLocaleString()} aggregate event(s), and ${correlations.length} correlated entity record(s).`, 'Continue monitoring. Use Settings -> show existing profiles when you add fields or streams.'));
  }
  if (!items.length) {
    items.push(actionItem('warn', 'Waiting for useful data', 'No immediate action was detected, but the current status has little evidence.', 'Check Diagnostics for stream coverage and verify events are arriving.'));
  }
  return `<div class="action-list">${items.slice(0, 6).join('')}</div>`;
}
function compactRelatedActivity(rows) {
  const groups = new Map();
  for (const row of rows || []) {
    const key = `${row.entity || row.source_ip || '-'}|${row.stream || '-'}|${row.type || '-'}`;
    const group = groups.get(key) || {entity: row.entity || row.source_ip || '-', entity_label: row.entity_label || entityDisplay(row.entity || row.source_ip, row), stream: row.stream || '-', type: row.type || '-', count: 0, security: 0, actions: new Set(), destinations: new Set(), services: new Set(), first: row.timestamp || '', last: row.timestamp || '', samples: []};
    group.count += 1;
    if (row.severity && !['-','info','notice','low'].includes(String(row.severity).toLowerCase())) group.security += 1;
    if (row.action && row.action !== '-') group.actions.add(row.action);
    if (row.destination && row.destination !== '-') group.destinations.add(row.destination);
    if (row.service && row.service !== '-') group.services.add(row.service);
    if (row.timestamp && (!group.first || row.timestamp < group.first)) group.first = row.timestamp;
    if (row.timestamp && (!group.last || row.timestamp > group.last)) group.last = row.timestamp;
    if (group.samples.length < 6) group.samples.push(row);
    groups.set(key, group);
  }
  return [...groups.values()].sort((left,right) => (right.security-left.security) || (right.count-left.count)).slice(0,25);
}
function correlationKey(item) {
  return String(item.entity || item.source_ip || '');
}
function correlationLabel(item) {
  return String(item.entity_label || item.entity_display || item.entity || item.source_ip || '-');
}
function correlationShortLabel(item) {
  const label = correlationLabel(item);
  const key = correlationKey(item);
  return label && label !== key ? label : key || '-';
}
function rememberEntityLabel(key, label, detail='') {
  key = String(key || '');
  label = String(label || key || '');
  if (!key || !label) return;
  uiCache.entityLabels[key] = {label, detail: String(detail || '')};
}
function entityDisplay(value, row={}) {
  const key = String(row.entity || row.source_ip || row.src_ip || row.subject || value || '');
  const explicit = row.entity_label || row.entity_display;
  if (explicit) return String(row.entity_label || row.entity_display);
  return (uiCache.entityLabels[key] || {}).label || key || String(value || '-');
}
function entityCell(row, key='entity') {
  const value = row[key] || row.entity || row.source_ip || row.src_ip || row.subject || '';
  const label = entityDisplay(value, row);
  return esc(label || '-');
}
function refreshEntityLabels(correlations, context) {
  for (const item of correlations || []) {
    rememberEntityLabel(correlationKey(item), correlationShortLabel(item), item.entity_detail || item.source_ip || '');
  }
  for (const item of (context.source_profiles || [])) {
    rememberEntityLabel(item.entity, item.entity_label || item.entity_display || item.entity, item.entity_detail || '');
  }
  for (const item of (context.security_event_samples || [])) {
    rememberEntityLabel(item.entity, item.entity_label || item.entity_display || item.entity, item.entity_detail || '');
  }
}
function incidentActions(row) {
  const state = String(row.lifecycle_status || 'open');
  const id = esc(row.id);
  const actions = [];
  if (state !== 'acknowledged') actions.push(`<button class="incident-action" data-id="${id}" data-status="acknowledged">Ack</button>`);
  if (state !== 'resolved') actions.push(`<button class="incident-action" data-id="${id}" data-status="resolved">Resolve</button>`);
  if (state !== 'open') actions.push(`<button class="incident-action" data-id="${id}" data-status="open">Reopen</button>`);
  return `<div class="review-actions">${actions.join('')}</div>${row.note ? `<div class="muted">${esc(row.note)}</div>` : ''}`;
}
function renderCorrelationExplorer(correlations, configuration) {
  const target = document.getElementById('correlationExplorer');
  const rows = [...(correlations || [])].sort((a,b)=>(Number(b.security_events)||0)-(Number(a.security_events)||0) || (Number(b.events)||0)-(Number(a.events)||0)).slice(0,10);
  if (!rows.length) {
    target.textContent = 'No correlated entities in the current analysis window.';
    return;
  }
  if (!rows.some(item => correlationKey(item) === uiCache.selectedCorrelationKey)) {
    uiCache.selectedCorrelationKey = correlationKey(rows[0]);
  }
  const selected = rows.find(item => correlationKey(item) === uiCache.selectedCorrelationKey) || rows[0];
  const chips = rows.map(item => {
    const key = correlationKey(item);
    const label = `${correlationShortLabel(item)} (${Number(item.security_events || 0)}/${Number(item.events || 0)})`;
    return `<button type="button" class="entity-chip ${key === uiCache.selectedCorrelationKey ? 'active' : ''}" data-correlation-key="${esc(key)}">${esc(label)}</button>`;
  }).join('');
  const samples = (selected.samples || []).slice(0,8).map(item => {
    const context = item.context || item.message || item.destination || item.service || '-';
    const meta = [item.timestamp, item.destination, item.service].filter(Boolean).join(' · ');
    return `<div class="evidence-item"><div class="evidence-head"><span class="evidence-stream">${esc(item.stream || 'stream')}</span><span class="evidence-action">${esc(item.action || item.severity || '-')}</span></div><div class="evidence-context">${esc(context)}</div><div class="muted">${esc(meta || '-')}</div>${item.graylog_query ? `<details data-detail-id="${esc(`query:${item.graylog_query}`)}"><summary>Graylog query</summary><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}</details>` : ''}</div>`;
  }).join('');
  const selectedKey = correlationKey(selected);
  const selectedLabel = correlationShortLabel(selected);
  const keyNote = selectedLabel !== selectedKey ? ` · key ${selectedKey}` : '';
  target.innerHTML = `<div class="entity-picker">${chips}</div><div><strong>${esc(selectedLabel || '-')}</strong><div class="muted">${esc((selected.streams || []).join(', ') || 'single stream')}${esc(keyNote)}</div></div><div class="entity-summary"><div class="entity-stat"><b>${esc(selected.events || 0)}</b><span>events</span></div><div class="entity-stat"><b>${esc(selected.security_events || 0)}</b><span>security</span></div><div class="entity-stat"><b>${esc((selected.streams || []).length)}</b><span>streams</span></div></div><div class="evidence-list">${samples || '<p class="muted">No sample evidence for this entity.</p>'}</div>`;
  target.querySelectorAll('.entity-chip').forEach(button => button.addEventListener('click', () => {
    uiCache.selectedCorrelationKey = button.dataset.correlationKey || '';
    renderCorrelationExplorer(window.currentCorrelations || [], window.currentConfiguration || {});
  }));
}
function inferredProfileType(row) {
  const explicit = row.detected_log_type || (row.discovery || {}).log_type;
  if (explicit) return explicit;
  const haystack = `${row.stream_name || ''} ${(row.common_fields || []).map(item => item.field || '').join(' ')} ${(row.categorical_fields || []).join(' ')}`.toLowerCase();
  if (/(windows|winlog|sysmon|powershell|security event)/.test(haystack)) return 'windows';
  if (/(fortigate|firewall|policyid|dstport|srcport)/.test(haystack)) return 'firewall';
  if (/(dns|adguard|query_domain|dns_query|\\bqh\\b)/.test(haystack)) return 'dns';
  if (/(nginx|apache|iis|squid|proxy|url|request_uri)/.test(haystack)) return 'web_proxy';
  if (/(audit|app|application|serverlog|prod)/.test(haystack)) return 'application';
  return 'unknown';
}
function profileDiscoveryDetails(row) {
  const d = row.discovery || {};
  const detailId = `profile-discovery:${row.stream_id || row.stream_name || ''}:${d.field_count || 0}`;
  if (!d.field_count) {
    const common = (row.common_fields || []).slice(0,8).map(item => `${item.field} ${(Number(item.coverage || 0)*100).toFixed(0)}%/${item.unique_values}`).join(', ');
    return `<details data-detail-id="${esc(detailId)}"><summary>waiting for discovery data</summary><p>This row was produced before profile discovery metadata was added. Wait for the next monitor cycle after updating, or re-apply/edit the profile.</p><p><b>Observed common fields</b><br>${esc(common || '-')}</p></details>`;
  }
  const selected = d.selected_fields || {};
  const top = (d.top_fields || []).slice(0,8).map(item => `${item.field} ${(Number(item.coverage || 0)*100).toFixed(0)}%/${item.unique_values}`).join(', ');
  const shared = (d.shared_fields || row.shared_fields || []).slice(0,8).map(sharedFieldLabel).join(', ');
  const rejected = (d.rejected_fields || []).slice(0,8).map(item => `${item.field}: ${item.reason}`).join('; ');
  const reasons = (d.reasons || []).join('; ');
  const advisor = row.profile_advisor || {};
  const fieldRoles = Object.entries(advisor.field_roles || {}).slice(0,12).map(([field, role]) => `${field}: ${role}`).join(', ');
  const correlationRoles = Object.entries(advisor.correlation_roles || {}).slice(0,8).map(([role, fields]) => `${role}: ${(fields || []).join(', ')}`).join('; ');
  return `<details data-detail-id="${esc(detailId)}"><summary>${esc(d.field_count)} fields analyzed</summary><p><b>Selected</b><br>Entity: ${esc((selected.entity || []).join(', ') || '-')}<br>Time: ${esc((selected.time || []).join(', ') || '-')}<br>Categorical: ${esc((selected.categorical || []).join(', ') || '-')}<br>Numeric: ${esc((selected.numeric || []).join(', ') || '-')}<br>Relationships: ${esc((selected.relationships || []).join(', ') || '-')}</p><p><b>Shared across streams</b><br>${esc(shared || '-')}</p><p><b>Advisor roles</b><br>${esc(fieldRoles || '-')}<br>${esc(correlationRoles || '')}</p><p><b>Top fields</b><br>${esc(top || '-')}</p><p><b>Rejected</b><br>${esc(rejected || '-')}</p><p>${esc(reasons || '')}</p></details>`;
}
function sharedFieldLabel(item) {
  const aliases = (item.aliases || []).filter(alias => alias !== item.field).slice(0,4);
  const suffix = aliases.length ? ` via ${aliases.join('/')}` : '';
  const kind = item.kind ? ` ${item.kind}` : '';
  return `${item.field}${kind} (${item.streams} streams${suffix})`;
}
function profileAdvisorLabel(row, advisor) {
  const rowAdvisor = row.profile_advisor || {};
  if (rowAdvisor.status === 'heuristic' && rowAdvisor.error) return 'heuristic fallback';
  if (rowAdvisor.status) return rowAdvisor.status;
  if (advisor.enabled && advisor.status === 'heuristic_fallback') return 'heuristic fallback';
  if (advisor.enabled && advisor.status === 'error') return 'advisor error';
  return advisor.enabled ? (advisor.status || 'checking') : 'heuristic';
}
function drawTrend(history) {
  const canvas=document.getElementById('trendChart'), ctx=canvas.getContext('2d'), ratio=window.devicePixelRatio||1, cw=canvas.clientWidth, ch=canvas.clientHeight;
  canvas.width=cw*ratio; canvas.height=ch*ratio; ctx.scale(ratio,ratio); ctx.clearRect(0,0,cw,ch);
  const rows=(history||[]).slice(-72);
  if (!rows.length) { ctx.fillStyle='#91abc4'; ctx.font='14px Arial'; ctx.fillText('Waiting for monitor history.', 16, 28); return; }
  const left=58,right=58,top=34,bottom=36,w=cw-left-right,h=ch-top-bottom;
  const eventsMax=Math.max(1,...rows.map(item=>Number(item.events)||0));
  const anomalyMax=Math.max(1,...rows.map(item=>Number(item.anomalies)||0),...rows.map(item=>Number(item.high||0)+Number(item.critical||0)));
  ctx.strokeStyle='#163b59'; ctx.lineWidth=1; ctx.font='11px Arial'; ctx.textAlign='right'; ctx.fillStyle='#91abc4';
  for(let tick=0; tick<=4; tick++){
    const y=top+h-(tick/4)*h;
    ctx.beginPath(); ctx.moveTo(left,y); ctx.lineTo(cw-right,y); ctx.stroke();
    ctx.fillText(Math.round(eventsMax*tick/4).toLocaleString(), left-8, y+4);
    ctx.textAlign='left'; ctx.fillText(Math.round(anomalyMax*tick/4).toLocaleString(), cw-right+8, y+4); ctx.textAlign='right';
  }
  const points=(key,max) => rows.map((item,index)=>({x:left+index*w/Math.max(1,rows.length-1), y:top+h-((Number(item[key])||0)/max)*h}));
  const drawLine=(pts,color,width=2.5) => { ctx.strokeStyle=color; ctx.lineWidth=width; ctx.beginPath(); pts.forEach((point,index)=>index?ctx.lineTo(point.x,point.y):ctx.moveTo(point.x,point.y)); ctx.stroke(); };
  const eventsPts=points('events',eventsMax), anomaliesPts=points('anomalies',anomalyMax), highPts=rows.map((item,index)=>({x:left+index*w/Math.max(1,rows.length-1), y:top+h-(((Number(item.high)||0)+(Number(item.critical)||0))/anomalyMax)*h}));
  ctx.fillStyle='rgba(30,169,255,.12)';
  ctx.beginPath(); eventsPts.forEach((point,index)=>index?ctx.lineTo(point.x,point.y):ctx.moveTo(point.x,point.y)); ctx.lineTo(left+w,top+h); ctx.lineTo(left,top+h); ctx.closePath(); ctx.fill();
  drawLine(eventsPts,'#1ea9ff',2.5);
  drawLine(highPts,'#ffcf5a',2);
  drawLine(anomaliesPts,'#ff6666',2.5);
  const last=rows[rows.length-1] || {};
  ctx.strokeStyle='#285071'; ctx.strokeRect(left,top,w,h);
  ctx.textAlign='left'; ctx.fillStyle='#1ea9ff'; ctx.fillText(`events max ${eventsMax.toLocaleString()}`, left, 18);
  ctx.fillStyle='#ffcf5a'; ctx.fillText(`high+ max ${anomalyMax.toLocaleString()}`, left+150, 18);
  ctx.fillStyle='#ff6666'; ctx.fillText(`anomalies latest ${Number(last.anomalies||0).toLocaleString()}`, left+280, 18);
  ctx.textAlign='right'; ctx.fillStyle='#91abc4'; ctx.fillText(`${rows.length} samples`, cw-right, ch-10); ctx.textAlign='left';
}
function drawCorrelationGraph(correlations) {
  const canvas=document.getElementById('correlationGraph'), ctx=canvas.getContext('2d'), ratio=window.devicePixelRatio||1, cw=canvas.clientWidth, ch=canvas.clientHeight;
  canvas.width=cw*ratio; canvas.height=ch*ratio; ctx.scale(ratio,ratio); ctx.clearRect(0,0,cw,ch);
  const short=(value,max=22)=>String(value||'-').length>max?`${String(value).slice(0,max-3)}...`:String(value||'-');
  const usefulResource = value => {
    const text = String(value || '').trim();
    if (!text || ['-', '0', 'unknown', 'n/a', 'none', 'null'].includes(text.toLowerCase())) return '';
    if (/^\\d+(\\.\\d+)?$/.test(text)) return '';
    if (/^\\d{4}-\\d{2}-\\d{2}t/i.test(text)) return '';
    return text;
  };
  const sampleResource = item => usefulResource(item.destination) || usefulResource(item.service) || usefulResource(item.type) || usefulResource(item.context);
  const items=[...(correlations||[])].sort((a,b)=>(Number(b.security_events)||0)-(Number(a.security_events)||0) || (Number(b.events)||0)-(Number(a.events)||0) || (b.streams||[]).length-(a.streams||[]).length).slice(0,8);
  uiCache.correlationHitboxes = [];
  if (!items.length) { ctx.fillStyle='#91abc4'; ctx.font='14px Arial'; ctx.fillText('No multi-stream entities in the current analysis window.', 16, 28); return; }
  if (!items.some(item => correlationKey(item) === uiCache.selectedCorrelationKey)) {
    uiCache.selectedCorrelationKey = correlationKey(items[0]);
  }
  const selected = items.find(item => correlationKey(item) === uiCache.selectedCorrelationKey) || items[0];
  const center={x:cw*.48,y:ch*.48};
  const streams=(selected.streams||[]).slice(0,7);
  const samples=(selected.samples||[]).slice(0,8);
  const resources=[...new Set(samples.map(sampleResource).filter(Boolean))].slice(0,5);
  const outer=[...streams.map(name=>({kind:'stream', name})), ...resources.map(name=>({kind:'resource', name}))];
  const radius=Math.max(86, Math.min(cw,ch)*.32);
  ctx.strokeStyle='#163b59';
  ctx.lineWidth=1;
  for (let ring=1; ring<=2; ring++) {
    ctx.beginPath();
    ctx.arc(center.x, center.y, radius*ring/2, 0, Math.PI*2);
    ctx.stroke();
  }
  const nodePositions=[];
  outer.forEach((node,index)=>{
    const angle=(-Math.PI/2)+(index*Math.PI*2/Math.max(1,outer.length));
    const x=center.x+Math.cos(angle)*radius;
    const y=center.y+Math.sin(angle)*radius;
    nodePositions.push({...node,x,y});
    const security=samples.some(sample => sample.stream===node.name && (/(deny|block|fail|drop|reject)/i.test(String(sample.action||'')) || !['','-','info','notice','low'].includes(String(sample.severity||'').toLowerCase())));
    ctx.strokeStyle=security ? 'rgba(255,95,95,.72)' : 'rgba(51,145,202,.55)';
    ctx.lineWidth=security ? 2.2 : 1.2;
    ctx.beginPath();
    ctx.moveTo(center.x, center.y);
    ctx.lineTo(x, y);
    ctx.stroke();
  });
  const drawNode=(x,y,r,color,label,sub,key='') => {
    ctx.fillStyle=color;
    ctx.beginPath();
    ctx.arc(x,y,r,0,Math.PI*2);
    ctx.fill();
    ctx.strokeStyle='#9ecfff';
    ctx.lineWidth=1.2;
    ctx.stroke();
    if (key) uiCache.correlationHitboxes.push({key,x,y,radius:r+8});
    ctx.textAlign='center';
    ctx.font='12px Arial';
    ctx.fillStyle='#f2f8ff';
    ctx.fillText(short(label,24), x, y+r+14);
    if (sub) { ctx.font='11px Arial'; ctx.fillStyle='#91abc4'; ctx.fillText(short(sub,28), x, y+r+28); }
  };
  drawNode(center.x, center.y, 26, Number(selected.security_events||0) ? '#d95f5f' : '#2389cc', correlationShortLabel(selected), `${Number(selected.events)||0} events / ${Number(selected.security_events)||0} security`, correlationKey(selected));
  nodePositions.forEach(node => {
    const color=node.kind==='stream' ? '#238b5d' : '#6f55c8';
    const count=samples.filter(sample => sample.stream===node.name || sampleResource(sample)===node.name).length;
    drawNode(node.x,node.y,node.kind==='stream'?15:12,color,node.name,count?`${count}`:node.kind,'');
  });
  const sideX=20, sideY=28;
  if (cw > 760) {
    ctx.textAlign='left';
    ctx.font='12px Arial';
    ctx.fillStyle='#91abc4';
    ctx.fillText('Top correlated', sideX, sideY);
    items.slice(0,5).forEach((item,index)=>{
      const key=correlationKey(item);
      const y=sideY+22+index*23;
      ctx.fillStyle=key===correlationKey(selected)?'#1ea9ff':'#91abc4';
      ctx.fillText(`${short(correlationShortLabel(item),24)}  ${Number(item.security_events||0)}/${Number(item.events||0)}`, sideX, y);
      uiCache.correlationHitboxes.push({key,x:sideX-4,y:y-14,w:210,h:20});
    });
  }
  ctx.textAlign='left'; ctx.fillStyle='#91abc4'; ctx.font='12px Arial'; ctx.fillText('Selected entity is centered. Green nodes are streams; purple nodes are destinations/services. Click a listed entity or center node evidence chip to change focus.', 8, ch-14);
}
function showRefreshError(error) {
  const message = error && error.stack ? error.stack : String(error || 'unknown refresh error');
  console.error(error);
  document.getElementById('stamp').textContent = 'Dashboard render error';
  document.getElementById('liveStatus').innerHTML = `<span class="sev-high">Dashboard render error:</span><br><code>${esc(message)}</code>`;
}
async function refresh() {
  try {
  const openDetails = new Set([...document.querySelectorAll('details[open][data-detail-id]')].map(item => item.dataset.detailId));
  const res = await fetch('/api/status', {cache: 'no-store'});
  if (!res.ok) throw new Error(`GET /api/status failed: ${res.status} ${res.statusText}`);
  const data = await res.json();
  const s = data.summary || {};
  const a = data.anomaly_summary || {};
  const baseline = data.baseline || {};
  const threat = (data.capabilities || {}).threat_intel || {};
  const mcp = (data.capabilities || {}).graylog_mcp || {};
  const advisor = ((data.capabilities || {}).profile_advisor || {});
  const configuration = data.configuration || {};
  const cache = data.status_cache || {};
  const streamCoverage = data.stream_coverage || [];
  const context = data.event_context || {};
  const enabledConfigStreams = (configuration.graylog_streams || []).filter(item => item.enabled);
  const enabledStreams = streamCoverage.some(item => item.enabled) ? streamCoverage.filter(item => item.enabled) : enabledConfigStreams;
  const streamsMissingProfile = enabledStreams.filter(item => !item.profile_ready).length;
  const enabledWithNoRawEvents = enabledStreams.filter(item => Number(item.events_fetched || 0) === 0);
  const displayedAggregateEvents = mcp.status === 'refreshing' ? Number(mcp.previous_aggregate_events || 0) : Number(mcp.aggregate_events || 0);
  const displayedRawEvents = mcp.status === 'refreshing' ? Number(mcp.previous_raw_events_fetched || mcp.previous_events_fetched || 0) : Number(mcp.raw_events_fetched || mcp.events_fetched || 0);
  const displayedFetchMode = mcp.fetch_mode || mcp.previous_fetch_mode || configuration.graylog_fetch_mode || 'raw';
  const displayedCoverage = mcp.coverage_status || mcp.previous_coverage_status || 'unknown';
  const zeroRawWindow = mcp.status === 'refreshing' ? 'previous completed poll' : 'this poll';
  const nowSeconds = Math.floor(Date.now() / 1000);
  const pollStartedAt = Number(mcp.poll_started_at || 0);
  const pollCompletedAt = Number(mcp.poll_completed_at || mcp.previous_poll_completed_at || 0);
  const pollRunningSeconds = pollStartedAt ? Math.max(0, nowSeconds - pollStartedAt) : 0;
  const pollCompletedAge = pollCompletedAt ? Math.max(0, nowSeconds - pollCompletedAt) : 0;
  const rawCorrelations = data.cross_source_correlations || [];
  const correlationsCached = rawCorrelations.length === 0 && uiCache.correlations.length > 0;
  const correlations = rawCorrelations.length ? rawCorrelations : uiCache.correlations;
  if (rawCorrelations.length) uiCache.correlations = rawCorrelations;
  refreshEntityLabels(correlations, context);
  window.currentCorrelations = correlations;
  window.currentConfiguration = configuration;
  drawTrend(data.history || []);
  drawCorrelationGraph(correlations);
  renderCorrelationExplorer(correlations, configuration);
  document.getElementById('correlationGraphInfo').textContent = `${correlations.length} entities correlated across enabled streams${correlationsCached ? ' (cached from previous non-empty poll)' : ''}. Focus graph shows the selected entity, connected streams, and sampled destinations/services; click entity labels or chips to inspect evidence.`;
  document.getElementById('stamp').textContent = data.generated_at ? `Updated ${new Date(data.generated_at * 1000).toLocaleString()}` : 'Waiting for monitor data';
  document.getElementById('metrics').innerHTML = [
    metric('Total events', s.total || 0),
    metric('UTM events', s.utm || 0),
    metric('Threat actions', s.threat_actions || 0),
    metric('Anomalies high+', (a.high || 0) + (a.critical || 0))
  ].join('');
  const llm = data.llm_assessment || {};
  const llmEnabled = Boolean(llm.enabled || configuration.llm_enabled);
  const mcpBadge = mcpCapability(mcp, configuration, pollRunningSeconds);
  const llmBadge = llmCapability(llm, configuration);
  document.getElementById('capabilities').innerHTML = [
    capability('Baseline', baseline.enabled ? 'on' : 'warn', baseline.enabled ? `${baseline.sources_ready || 0} sources ready` : 'disabled'),
    capability('Ollama', llmBadge.state, llmBadge.detail),
    capability('Threat Intel', threat.enabled && threat.configured ? 'on' : 'warn', threat.enabled ? `${threat.provider || 'unknown'}${threat.configured ? '' : ', key missing'}` : 'disabled'),
    capability('Graylog MCP', mcpBadge.state, mcpBadge.detail)
  ].join('');
  document.getElementById('liveStatus').innerHTML = [
    data.stale ? `<span class="sev-high">Showing cached dashboard data because live MCP fetch failed.</span>` : '',
    cache.stored_at ? `Cached snapshot: ${esc(new Date(cache.stored_at * 1000).toLocaleString())}` : '',
    `Status schema: ${esc(data.status_schema || 'old')}`,
    `Critical anomalies: ${esc((a.critical || 0))}`,
    `High anomalies: ${esc((a.high || 0))}`,
    `Baseline sources ready: ${esc((data.baseline || {}).sources_ready || 0)}`,
    `Baseline training days: ${esc((data.baseline || {}).training_days || 0)}`,
    `Baseline DB size: ${esc(bytes((data.baseline || {}).size_bytes || 0))}`,
    `Discovery fields recorded: ${esc((data.baseline || {}).discovery_fields_recorded || 0)}`,
    `Discovery cache events: ${esc((data.baseline || {}).discovery_cache_events || 0)}`,
    llm.status === 'error' ? `<span class="sev-high">Ollama assessment error: ${esc(llm.error || 'unknown error')}</span>` : '',
    `MCP status: ${esc(mcp.status || 'unknown')}`,
    mcp.status === 'refreshing' && pollStartedAt ? `MCP poll running: ${esc(pollRunningSeconds)}s` : '',
    pollCompletedAt ? `Last completed MCP poll: ${esc(pollCompletedAge)}s ago` : '',
    mcp.status === 'refreshing' && pollRunningSeconds > Number(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 120) ? `<span class="sev-high">MCP poll has been refreshing for ${esc(pollRunningSeconds)}s, which is over the configured poll budget. Restart monitor or lower enabled streams/sample size if this keeps happening.</span>` : '',
    mcp.error ? `<span class="sev-high">MCP error: ${esc(mcp.error)}</span>` : '',
    `Enabled streams: ${esc(enabledStreams.length)}`,
    `MCP fetch mode: ${esc(displayedFetchMode)}${mcp.status === 'refreshing' ? ' (refreshing, showing previous counters)' : ''}`,
    `MCP poll window: ${esc(mcp.range_seconds || configuration.graylog_range_seconds || 0)}s`,
    `MCP max events/stream: ${esc(mcp.max_events_per_stream || configuration.graylog_max_events_per_stream || 0)}`,
    `MCP call timeout: ${esc(mcp.call_timeout_seconds || configuration.graylog_mcp_call_timeout_seconds || 0)}s`,
    `MCP poll budget: ${esc(mcp.poll_timeout_seconds || configuration.graylog_mcp_poll_timeout_seconds || 0)}s`,
    mcp.poll_duration_seconds ? `MCP last poll duration: ${esc(mcp.poll_duration_seconds)}s` : '',
    Number(mcp.max_events_per_stream || configuration.graylog_max_events_per_stream || 0) > 100000 ? `<span class="sev-high">MCP max events/stream is very high. Use aggregate mode with a smaller raw sample, for example max 5000 and raw sample 1000-5000.</span>` : '',
    `MCP aggregate events: ${esc(displayedAggregateEvents)}`,
    `MCP raw sample events: ${esc(displayedRawEvents)}`,
    `MCP coverage: ${esc(displayedCoverage)}${mcp.partial_streams ? ` (${esc(mcp.partial_streams)} partial)` : ''}${mcp.truncated_streams ? ` (${esc(mcp.truncated_streams)} truncated)` : ''}${mcp.sample_limited_streams ? ` (${esc(mcp.sample_limited_streams)} sample capped)` : ''}`,
    mcp.skipped_streams ? `<span class="sev-high">${esc(mcp.skipped_streams)} stream(s) skipped because the MCP poll budget was reached.</span>` : '',
    configuration.log_source === 'graylog_mcp' && mcp.status === 'no_streams_enabled' ? `<span class="sev-high">No Graylog streams are enabled. Open Settings, Load streams, tick streams, then click Save streams.</span>` : '',
    configuration.log_source === 'graylog_mcp' && mcp.status !== 'refreshing' && enabledStreams.length > 0 && displayedRawEvents === 0 ? `<span class="sev-high">No raw events fetched from enabled streams. Check poll window, Graylog query, stream permissions, and Diagnostics -> Stream Coverage.</span>` : '',
    enabledWithNoRawEvents.length && mcp.status !== 'refreshing' ? `<span class="sev-high">${esc(enabledWithNoRawEvents.length)} enabled stream(s) returned zero raw events in ${esc(zeroRawWindow)}.</span>` : '',
    mcp.coverage_warning ? `<span class="sev-high">${esc(mcp.coverage_warning)}</span>` : '',
    advisor.status === 'heuristic_fallback' ? `Profile advisor: heuristic fallback (${esc(advisor.error || 'advisor unavailable')})` : '',
    advisor.status === 'error' ? `<span class="sev-high">Profile advisor error: ${esc(advisor.error || 'unknown error')}</span>` : ''
  ].filter(Boolean).join('<br>');
  document.getElementById('health').innerHTML = [
    metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP aggregate events', displayedAggregateEvents), metric('MCP raw sample', displayedRawEvents), metric('Partial streams', mcp.partial_streams || 0), metric('Sample capped', mcp.sample_limited_streams || 0), metric('Truncated streams', mcp.truncated_streams || 0), metric('Correlated entities', correlations.length)
  ].join('');
  window.profileSuggestions = data.profile_suggestions || [];
  const showExistingProfiles = document.getElementById('showExistingProfiles')?.checked;
  const pendingProfileSuggestions = window.profileSuggestions.filter(item => !item.profile_exists);
  const visibleProfileSuggestions = showExistingProfiles ? window.profileSuggestions : pendingProfileSuggestions;
  document.getElementById('profileSuggestionSummary').textContent = `${pendingProfileSuggestions.length} missing profile(s), ${window.profileSuggestions.length - pendingProfileSuggestions.length} existing.`;
  document.getElementById('profileSuggestions').innerHTML = visibleProfileSuggestions.length ? table(visibleProfileSuggestions, [
    {label:'Stream', key:'stream_name'},
    {label:'Events', key:'events'},
    {label:'Type', render:r => esc(inferredProfileType(r))},
    {label:'Confidence', key:'confidence'},
    {label:'Advisor', render:r => esc(profileAdvisorLabel(r, advisor))},
    {label:'Profile', render:r => r.profile_exists ? 'exists' : 'new'},
    {label:'Entity fields', render:r => esc((r.entity_fields || []).join(', ') || '-')},
    {label:'Time', key:'timestamp_field'},
    {label:'Baseline fields', render:r => esc([...(r.categorical_fields || []), ...(r.numeric_fields || [])].slice(0,8).join(', ') || '-')},
    {label:'Relationships', render:r => esc((r.relationship_fields || []).slice(0,4).map(item => `${item.left}->${item.right}`).join(', ') || '-')},
    {label:'Detectors', render:r => esc(Object.keys(r.detectors || {}).join(', ') || '-')},
    {label:'Common denominators', render:r => esc((r.common_fields || []).slice(0,5).map(item => `${item.field} ${(item.coverage*100).toFixed(0)}%`).join(', ') || '-')},
    {label:'Shared fields', render:r => esc((r.shared_fields || []).slice(0,4).map(sharedFieldLabel).join(', ') || '-')},
    {label:'Discovery', render:r => profileDiscoveryDetails(r)},
    {label:'Action', render:r => `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">${r.profile_exists ? 'Update profile' : 'Apply profile'}</button>`}
  ], 'profile-suggestions') : '<p class="muted">No missing stream profiles. Enable "show existing profiles" to inspect already configured profiles.</p>';
  const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
  document.getElementById('llmAssessment').innerHTML = `<div class="ai-box"><details><summary>Status: ${esc(llm.status || (llmEnabled ? 'starting' : 'disabled'))}</summary><p>${llmText}</p></details></div>`;
  document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [
    {label:'Source', key:'subject', render:r => entityCell(r, 'subject')},
    {label:'Score', key:'score'},
    {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
    {label:'Confidence', key:'confidence'},
    {label:'Rate / ports / hits', render:r => {
      const e = r.evidence || {};
      const rate = e.timed_events > 1 ? `${e.events_per_minute} events/min` : 'no timestamps';
      return esc(`${rate}; dst ports: ${e.distinct_dst_ports || 0}; src ports: ${e.distinct_src_ports || 0}; hitcount: ${e.hitcount_total || 0}`);
    }},
    {label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
  ], 'anomalies');
  document.getElementById('recommendations').innerHTML = table(data.recommendations || [], [
    {label:'Subject', key:'subject', render:r => entityCell(r, 'subject')},
    {label:'Score', key:'score'},
    {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
    {label:'Title', key:'title'},
    {label:'Recommendation', key:'recommendation'},
    {label:'Policies', render:r => esc((r.related_policy_ids || []).join(', '))},
    {label:'Services', render:r => esc((r.related_services || []).join(', '))}
  ], 'recommendations');
  const showResolvedIncidents = document.getElementById('showResolvedIncidents')?.checked;
  const visibleIncidents = (data.incidents || []).filter(item => showResolvedIncidents || (item.lifecycle_status || 'open') !== 'resolved');
  const resolvedIncidentCount = (data.incidents || []).filter(item => (item.lifecycle_status || 'open') === 'resolved').length;
  document.getElementById('incidentSummary').textContent = `${visibleIncidents.length} shown, ${resolvedIncidentCount} resolved hidden.`;
  document.getElementById('incidents').innerHTML = table(visibleIncidents, [
    {label:'Entity', key:'entity', render:r => esc(`${entityDisplay(r.entity, r)} (${r.entity_type || 'entity'})`)},
    {label:'Score', key:'score'},
    {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
    {label:'State', render:r => esc(r.lifecycle_status || 'open')},
    {label:'Streams', render:r => esc((r.correlated_streams || []).join(', ') || 'single stream')},
    {label:'Evidence', render:r => esc((r.evidence || []).join('; '))},
    {label:'Timeline', render:r => { const rows=(r.timeline||[]).map(item => `${esc(`${item.timestamp || ''} | ${item.stream || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.context || item.message || ''}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`incident:${r.id || r.entity}:${r.first_seen || ''}`; return rows ? `<details data-detail-id="${esc(id)}"><summary>${esc(`${r.first_seen || '-'} to ${r.last_seen || '-'}`)}</summary><p>${rows}</p></details>` : '-'; }},
    {label:'Action', render:r => incidentActions(r)}
  ], 'incidents');
  document.querySelectorAll('.incident-action').forEach(button => button.addEventListener('click', async event => {
    event.preventDefault();
    const notice = document.getElementById('incidentNotice');
    const note = prompt('Incident note (optional):') || '';
    button.disabled = true;
    if (notice) notice.textContent = `Saving incident state: ${button.dataset.status || 'open'}...`;
    try {
      const response = await fetch('/api/incidents', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({id:button.dataset.id, status:button.dataset.status, note})});
      let payload = {};
      try { payload = await response.json(); } catch (_err) {}
      if (!response.ok) throw new Error(payload.error || `HTTP ${response.status}`);
      if (notice) notice.textContent = `Incident marked ${payload.status || button.dataset.status || 'open'}.`;
      await refresh();
    } catch (err) {
      if (notice) notice.textContent = `Could not save incident state: ${err.message || err}`;
      button.disabled = false;
    }
  }));
  document.getElementById('blocks').innerHTML = table(data.block_candidates || [], [
    {label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')},
    {label:'Score', key:'score'},
    {label:'Reasons', render:r => esc((r.reasons || []).join('; '))}
  ], 'blocks');
  const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
  const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, entity_label: correlation.entity_label || correlation.entity_display || correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
  const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
  document.getElementById('triageQueue').innerHTML = table(data.triage_queue || [], [
    {label:'Entity', key:'entity', render:r => entityCell(r)},
    {label:'Score', key:'score'},
    {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
    {label:'State', key:'state'},
    {label:'Streams', render:r => esc((r.streams || []).join(', ') || '-')},
    {label:'Detectors', render:r => esc((r.detectors || []).join(', ') || '-')},
    {label:'Open signals', key:'active_deviation_count'},
    {label:'Why', render:r => esc((r.evidence || []).join('; ') || '-')},
    {label:'Next', key:'next_action'}
  ], 'triage-queue');
  const rawFieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, ...item, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})));
  const fieldRowsCached = rawFieldRows.length === 0 && uiCache.fieldRows.length > 0;
  const sourceFieldRows = rawFieldRows.length ? rawFieldRows : uiCache.fieldRows;
  const showReviewed = document.getElementById('showReviewedFindings')?.checked;
  const showLow = document.getElementById('showLowFindings')?.checked;
  const fieldRows = sourceFieldRows
    .filter(row => showReviewed || !['expected','false_positive'].includes(row.feedback || ''))
    .filter(row => showLow || Number(row.score || 0) >= 35 || row.feedback === 'confirmed')
    .sort((left, right) => Number(right.score || 0) - Number(left.score || 0))
    .slice(0, 50);
  document.getElementById('actionPlan').innerHTML = buildActionPlan({configuration, mcp, enabledStreams, streamsMissingProfile, pendingProfileSuggestions, displayedRawEvents, displayedAggregateEvents, fieldRows, incidents: data.incidents || [], correlations, baseline});
  document.getElementById('findingSummary').textContent = `${fieldRows.length} shown from ${sourceFieldRows.length} raw deviations. Reviewed and low-score findings are hidden by default.`;
  if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
  document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [
    {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">False positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Confirm</button></div>`}
  ], 'field-deviations');
  document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
    const note = prompt('Review note (optional):') || '';
    const days = prompt('Expiry in days (0 = no expiry):', '0') || '0';
    const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, value:button.dataset.value, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
    document.getElementById('feedbackNotice').textContent = response.ok ? 'Review saved. The matching pattern will be labeled on the next refresh.' : 'Could not save review.';
    refresh();
  }));
  const relatedGroups = compactRelatedActivity(relatedRows);
  document.getElementById('relatedActivity').innerHTML = `<p class="muted">${relatedGroups.length} grouped rows shown from ${relatedRows.length} raw related events.</p>` + table(relatedGroups, [
    {label:'Entity', key:'entity', render:r => entityCell(r)},
    {label:'Stream', key:'stream'},
    {label:'Type', key:'type'},
    {label:'Events', key:'count'},
    {label:'Security', key:'security'},
    {label:'Time range', render:r => esc(`${r.first || '-'} to ${r.last || '-'}`)},
    {label:'Actions', render:r => esc([...r.actions].slice(0,4).join(', ') || '-')},
    {label:'Destinations', render:r => esc([...r.destinations].slice(0,4).join(', ') || '-')},
    {label:'Samples', render:r => { const rows=(r.samples || []).map(item => `${esc(`${item.timestamp || ''} | ${item.action || ''} | ${item.destination || ''} | ${item.service || ''} | ${item.context || ''}`)}${item.graylog_query ? `<br><code>${esc(item.graylog_query)}</code>${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('<br>'); const id=`related:${r.entity}:${r.stream}:${r.type}`; return rows ? `<details data-detail-id="${esc(id)}"><summary>show samples</summary><p>${rows}</p></details>` : '-'; }}
  ], 'related-activity');
  document.getElementById('reputation').innerHTML = table(reputationRows, [
    {label:'IP', key:'ip'},
    {label:'Provider', key:'provider'},
    {label:'Status', key:'status'},
    {label:'Score', key:'score'},
    {label:'Malicious', key:'malicious'},
    {label:'Suspicious', key:'suspicious'}
  ], 'reputation');
  document.getElementById('policies').innerHTML = table(data.policy_findings || [], [
    {label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
    {label:'Reference', key:'reference'},
    {label:'Title', key:'title'},
    {label:'Detail', key:'detail'}
  ], 'policies');
  const d = data.diagnostics || {};
  const quality = data.data_quality || {};
  const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id]));
  const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id}));
  document.getElementById('diagnostics').innerHTML =
    '<h3>Stream Coverage</h3>' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') +
    '<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
    '<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +
    '<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
    '<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
    '<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
    '<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}]) +
    '<h3>Top Destinations</h3>' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
    '<h3>Top Policy IDs</h3>' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
    '<h3>Top Destination Ports</h3>' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
    '<h3>Top Source Ports</h3>' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
    '<h3>Top Services</h3>' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
    '<h3>Local-in Failures</h3>' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]);
  document.querySelectorAll('details[data-detail-id]').forEach(item => { if (openDetails.has(item.dataset.detailId)) item.open = true; });
  document.querySelectorAll('[data-sort-table]').forEach(button => button.addEventListener('click', () => { const current=tableSort[button.dataset.sortTable]; tableSort[button.dataset.sortTable]={key:button.dataset.sortKey,direction:current && current.key===button.dataset.sortKey ? -current.direction : 1}; refresh(); }));
  document.querySelectorAll('.apply-suggested-profile').forEach(button => button.addEventListener('click', () => applySuggestedProfile(button.dataset.streamId)));
  } catch (error) {
    showRefreshError(error);
  }
}
async function loadSettings() {
  const config = await (await fetch('/api/config', {cache: 'no-store'})).json();
  const form = document.getElementById('settingsForm');
  for (const [key, value] of Object.entries(config)) {
    const field = form.elements.namedItem(key);
    if (!field) continue;
    if (field.type === 'checkbox') field.checked = Boolean(value); else field.value = value || '';
  }
  const token = form.elements.namedItem('graylog_mcp_token');
  token.placeholder = config.graylog_mcp_token_configured ? 'Token configured; leave blank to keep it' : 'Paste a read-only token';
  const abuseKey = form.elements.namedItem('abuseipdb_api_key');
  abuseKey.placeholder = config.abuseipdb_api_key_configured ? 'Key configured; leave blank to keep it' : 'Paste AbuseIPDB API key';
  const vtKey = form.elements.namedItem('virustotal_api_key');
  vtKey.placeholder = config.virustotal_api_key_configured ? 'Key configured; leave blank to keep it' : 'Paste VirusTotal API key';
  window.streamProfiles = config.graylog_stream_profiles || [];
  window.configuredStreams = config.graylog_streams || [];
  if (config.graylog_mcp_token_configured && config.graylog_mcp_url) loadStreams();
  loadOllamaModels();
}
async function loadOllamaModels() {
  const target = document.getElementById('ollamaModels');
  try {
    const payload = await (await fetch('/api/ollama/models', {cache: 'no-store'})).json();
    if (!payload.models || payload.models.length === 0) {
      target.textContent = payload.error || 'No local Ollama models found.';
      return;
    }
    target.innerHTML = `<div class="model-list">${payload.models.map(model => `<button type="button" class="model-pill" data-model="${esc(model.name)}">${esc(model.name)}</button>`).join('')}</div><div class="muted">Click a model to fill Ollama model and profile advisor model fields.</div>`;
    target.querySelectorAll('.model-pill').forEach(button => button.addEventListener('click', () => {
      document.querySelector('[name="llm_model"]').value = button.dataset.model;
      document.querySelector('[name="profile_advisor_model"]').value = button.dataset.model;
    }));
  } catch (error) {
    target.textContent = `Could not load Ollama models: ${error}`;
  }
}
async function loadStreams() {
  const response = await fetch('/api/graylog/streams');
  const payload = await response.json();
  if (!response.ok) {
    renderStreamPicker(payload.error || `Could not load streams: ${response.status}`);
    return;
  }
  const configuredStreams = payload.selected || window.configuredStreams || [];
  let selected = new Set(configuredStreams.filter(item => item.enabled).map(item => item.id));
  let recoveryText = '';
  if (!selected.size && (window.streamProfiles || []).length) {
    selected = new Set((window.streamProfiles || []).map(item => item.stream_id).filter(Boolean));
    recoveryText = 'No enabled streams were saved; streams with existing profiles are selected for recovery. Click Save streams to keep this selection.';
  }
  window.availableStreams = payload.streams || [];
  window.streamSelection = Object.fromEntries(window.availableStreams.map(stream => [stream.id, {id:stream.id, title:stream.title, enabled:selected.has(stream.id)}]));
  renderStreamPicker(recoveryText || payload.error || '');
}
document.getElementById('loadStreams').addEventListener('click', loadStreams);
async function saveStreamSelection() {
  const result = document.getElementById('settingsResult');
  const streams = Object.values(window.streamSelection || {});
  if (!streams.length) {
    result.textContent = 'Load streams before saving stream selection.';
    return false;
  }
  const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({graylog_streams: streams})});
  let payload = {};
  try { payload = await response.json(); } catch {}
  if (!response.ok) {
    result.textContent = `Could not save enabled streams: ${payload.error || response.statusText || response.status}`;
    return false;
  }
  result.textContent = `Saved ${streams.filter(item => item.enabled).length} enabled stream(s). Monitor will use them on the next cycle.`;
  return true;
}
function renderStreamPicker(errorText='') {
  const target = document.getElementById('streamPicker');
  if (!window.availableStreams.length) {
    target.textContent = errorText || 'No streams found.';
    return;
  }
  const previousFilter = document.getElementById('streamFilter')?.value || '';
  const previousEnabledOnly = document.getElementById('streamEnabledOnly')?.checked || false;
  const filter = previousFilter.toLowerCase().trim();
  const streams = window.availableStreams
    .filter(stream => !filter || `${stream.title || ''} ${stream.id || ''}`.toLowerCase().includes(filter))
    .filter(stream => !previousEnabledOnly || window.streamSelection[stream.id]?.enabled);
  const enabledCount = Object.values(window.streamSelection).filter(item => item.enabled).length;
  target.innerHTML = `
    ${errorText ? `<div class="notice warn">${esc(errorText)}</div>` : ''}
    <div class="stream-tools">
      <input id="streamFilter" type="search" placeholder="Search streams by name or id" value="${esc(previousFilter)}">
      <label><input id="streamEnabledOnly" type="checkbox" ${previousEnabledOnly ? 'checked' : ''}> enabled only</label>
      <button type="button" id="enableVisibleStreams">Enable visible</button>
      <button type="button" id="disableVisibleStreams">Disable visible</button>
      <button type="button" id="saveVisibleStreams">Save streams</button>
    </div>
    <div class="stream-counts">${esc(enabledCount)} enabled, ${esc(streams.length)} visible, ${esc(window.availableStreams.length)} total</div>
    <div class="stream-list">${streams.map(stream => {
      const selected = window.streamSelection[stream.id] || {id:stream.id, title:stream.title, enabled:false};
      return `<div class="stream-row" data-stream-row="${esc(stream.id)}"><label title="${esc(stream.title || stream.id)}"><input type="checkbox" class="graylog-stream" data-id="${esc(stream.id)}" data-title="${esc(stream.title)}" ${selected.enabled ? 'checked' : ''}> <span class="stream-title">${esc(stream.title || stream.id)}</span></label><button type="button" class="edit-profile" data-id="${esc(stream.id)}" data-title="${esc(stream.title)}">Edit profile</button></div>`;
    }).join('')}</div>`;
  document.getElementById('streamFilter').addEventListener('input', () => renderStreamPicker());
  document.getElementById('streamEnabledOnly').addEventListener('change', () => renderStreamPicker());
  document.getElementById('enableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = true; }); renderStreamPicker(); });
  document.getElementById('disableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = false; }); renderStreamPicker(); });
  document.getElementById('saveVisibleStreams').addEventListener('click', saveStreamSelection);
}
function mergeProfile(existing, recommended) {
  const mergeList = (left, right) => [...new Set([...(left || []), ...(right || [])].filter(Boolean))];
  const mergeRelationships = (left, right) => {
    const rows = [...(left || []), ...(right || [])].filter(item => item && item.left && item.right);
    const seen = new Set();
    return rows.filter(item => {
      const key = `${String(item.left).toLowerCase()}->${String(item.right).toLowerCase()}`;
      if (seen.has(key)) return false;
      seen.add(key);
      return true;
    });
  };
  const detectors = {...(recommended.detectors || {}), ...(existing.detectors || {})};
  return {
    ...recommended,
    ...existing,
    name: existing.name || recommended.name,
    entity_field: (existing.entity_field || recommended.entity_field || (recommended.entity_fields || [])[0] || ''),
    entity_fields: mergeList(existing.entity_fields || (existing.entity_field ? [existing.entity_field] : []), recommended.entity_fields || (recommended.entity_field ? [recommended.entity_field] : [])),
    timestamp_field: existing.timestamp_field || recommended.timestamp_field || 'timestamp',
    categorical_fields: mergeList(existing.categorical_fields, recommended.categorical_fields),
    numeric_fields: mergeList(existing.numeric_fields, recommended.numeric_fields),
    detectors,
    field_weights: {...(recommended.field_weights || {}), ...(existing.field_weights || {})},
    relationship_fields: mergeRelationships(existing.relationship_fields, recommended.relationship_fields)
  };
}
async function applySuggestedProfile(streamId) {
  const notice = document.getElementById('profileApplyResult');
  const button = document.querySelector(`.apply-suggested-profile[data-stream-id="${CSS.escape(streamId)}"]`);
  const suggestion = (window.profileSuggestions || []).find(item => String(item.stream_id) === String(streamId));
  if (!suggestion || !suggestion.profile) {
    notice.textContent = `No recommended profile found for ${streamId}.`;
    return;
  }
  try {
    if (button) button.disabled = true;
    notice.textContent = `Applying profile for ${suggestion.stream_name || streamId}...`;
    const configResponse = await fetch('/api/config', {cache: 'no-store'});
    const config = await configResponse.json();
    const existingProfile = (config.graylog_stream_profiles || []).find(item => String(item.stream_id) === String(suggestion.profile.stream_id));
    const profile = existingProfile ? mergeProfile(existingProfile, suggestion.profile) : suggestion.profile;
    const payload = {
      graylog_stream_profiles: [...(config.graylog_stream_profiles || []).filter(item => item.stream_id !== profile.stream_id), profile],
      graylog_streams: config.graylog_streams || [],
      log_source: config.log_source || 'graylog_mcp',
      graylog_mcp_url: config.graylog_mcp_url || '',
      graylog_query: config.graylog_query || '*',
      graylog_fetch_mode: config.graylog_fetch_mode || 'auto',
      graylog_tls_verify: config.graylog_tls_verify !== false,
      graylog_range_seconds: config.graylog_range_seconds || 300,
      graylog_max_events_per_stream: config.graylog_max_events_per_stream || 5000,
      graylog_raw_sample_events: config.graylog_raw_sample_events || 5000,
      graylog_mcp_call_timeout_seconds: config.graylog_mcp_call_timeout_seconds || 8,
      graylog_mcp_poll_timeout_seconds: config.graylog_mcp_poll_timeout_seconds || 120,
      graylog_field_mapping: config.graylog_field_mapping || '',
      baseline_training_days: config.baseline_training_days || 7,
      baseline_retention_days: config.baseline_retention_days || 14,
      baseline_value_retention_days: config.baseline_value_retention_days || 7,
      baseline_max_values_per_field: config.baseline_max_values_per_field || 2000,
      llm_enabled: Boolean(config.llm_enabled),
      llm_model: config.llm_model || '',
      profile_advisor_enabled: Boolean(config.profile_advisor_enabled),
      profile_advisor_model: config.profile_advisor_model || 'qwen3:8b',
      profile_advisor_timeout: config.profile_advisor_timeout || 120,
      threat_intel_enabled: Boolean(config.threat_intel_enabled),
      threat_intel_provider: config.threat_intel_provider || 'auto',
      threat_intel_daily_limit: config.threat_intel_daily_limit || 100,
      threat_intel_ttl_seconds: config.threat_intel_ttl_seconds || 604800,
      threat_intel_error_ttl_seconds: config.threat_intel_error_ttl_seconds || 3600,
      abuseipdb_max_age_days: config.abuseipdb_max_age_days || 90
    };
    const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload)});
    let result = {};
    try { result = await response.json(); } catch {}
    if (!response.ok) {
      notice.textContent = `Could not apply profile: ${result.error || response.statusText || response.status}`;
      return;
    }
    notice.textContent = `${existingProfile ? 'Updated' : 'Applied'} recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`;
    document.getElementById('settingsResult').textContent = notice.textContent;
    window.streamProfiles = result.graylog_stream_profiles || payload.graylog_stream_profiles;
    loadSettings();
  } catch (error) {
    notice.textContent = `Could not apply profile: ${error}`;
  } finally {
    if (button) button.disabled = false;
  }
}
async function editStreamProfile(streamId, title) {
  document.getElementById('profileEditorStatus').innerHTML = `Editing profile for <code>${esc(title || streamId)}</code>`;
  document.getElementById('fieldPicker').textContent = 'Loading stream fields...';
  const payload = await (await fetch(`/api/graylog/fields?stream_id=${encodeURIComponent(streamId)}`)).json();
  const profile = (window.streamProfiles || []).find(item => item.stream_id === streamId) || {};
  document.querySelector('[name="profile_name"]').value = profile.name || `${title || streamId} profile`;
  document.querySelector('[name="profile_detectors"]').value = Object.keys(profile.detectors || {}).length ? JSON.stringify(profile.detectors, null, 2) : '';
  document.querySelector('[name="profile_field_weights"]').value = Object.keys(profile.field_weights || {}).length ? JSON.stringify(profile.field_weights, null, 2) : '';
  document.querySelector('[name="profile_relationship_fields"]').value = (profile.relationship_fields || []).length ? JSON.stringify(profile.relationship_fields, null, 2) : '';
  const entityFields = new Set(profile.entity_fields || (profile.entity_field ? [profile.entity_field] : []));
  const rows = (payload.fields || []).map(field => { const name=field.name||field.field, type=(field.type||{}).type||'', props=(field.type||{}).properties||[]; return `<div class="field-row"><code>${esc(name)}</code><span>${esc(type)}</span><span>${esc(props.join(', '))}</span><div class="field-controls"><label><input class="profile-entity" type="checkbox" value="${esc(name)}" ${entityFields.has(name)?'checked':''}> Entity</label><label><input type="radio" name="profile_timestamp" value="${esc(name)}" ${profile.timestamp_field===name?'checked':''}> Time</label>${props.includes('enumerable')?`<label><input class="profile-categorical" type="checkbox" value="${esc(name)}" ${(profile.categorical_fields||[]).includes(name)?'checked':''}> Categorical</label>`:''}${props.includes('numeric')?`<label><input class="profile-numeric" type="checkbox" value="${esc(name)}" ${(profile.numeric_fields||[]).includes(name)?'checked':''}> Numeric</label>`:''}</div></div>`; }).join('');
  document.getElementById('fieldPicker').innerHTML = rows ? `<div class="field-header"><span>Field</span><span>Type</span><span>Capabilities</span><span>Use In Profile</span></div>${rows}` : esc(payload.error || 'No fields found.');
  window.activeProfileStream = streamId;
  window.activeProfileTitle = title || streamId;
}
document.getElementById('streamPicker').addEventListener('click', event => {
  const button = event.target.closest('.edit-profile');
  if (!button) return;
  editStreamProfile(button.dataset.id, button.dataset.title);
});
document.getElementById('streamPicker').addEventListener('change', event => {
  const checkbox = event.target.closest('.graylog-stream');
  if (!checkbox || !window.streamSelection[checkbox.dataset.id]) return;
  window.streamSelection[checkbox.dataset.id].enabled = checkbox.checked;
  document.querySelector('.stream-counts').textContent = `${Object.values(window.streamSelection).filter(item => item.enabled).length} enabled, ${document.querySelectorAll('.stream-row').length} visible, ${window.availableStreams.length} total`;
  document.getElementById('settingsResult').textContent = 'Stream selection changed. Click Save streams or Save configuration.';
});
document.getElementById('loadFields').addEventListener('click', async () => {
  if (window.activeProfileStream) { editStreamProfile(window.activeProfileStream, window.activeProfileTitle); return; }
  const checked = Object.values(window.streamSelection || {}).filter(item => item.enabled);
  if (checked.length !== 1) { document.getElementById('fieldPicker').textContent = 'Click Edit profile on the stream you want to edit, or enable exactly one stream first.'; return; }
  editStreamProfile(checked[0].id, checked[0].title);
});
document.getElementById('settingsForm').addEventListener('submit', async event => {
  event.preventDefault();
  const form = event.currentTarget;
  const values = Object.fromEntries(new FormData(form));
  values.llm_enabled = form.elements.llm_enabled.checked;
  values.profile_advisor_enabled = form.elements.profile_advisor_enabled.checked;
  values.threat_intel_enabled = form.elements.threat_intel_enabled.checked;
  values.graylog_tls_verify = form.elements.graylog_tls_verify.checked;
  const streamValues = Object.values(window.streamSelection || {});
  if (streamValues.length) {
    values.graylog_streams = streamValues;
  }
  if (window.activeProfileStream) { let detectors={},fieldWeights={},relationshipFields=[]; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } try { relationshipFields=form.elements.profile_relationship_fields.value.trim() ? JSON.parse(form.elements.profile_relationship_fields.value) : []; } catch { document.getElementById('settingsResult').textContent='Behavior relationships must be valid JSON.'; return; } if (!Array.isArray(relationshipFields)) { document.getElementById('settingsResult').textContent='Behavior relationships must be a JSON list.'; return; } const entityFields=[...form.querySelectorAll('.profile-entity:checked')].map(item=>item.value); if (!entityFields.length) { document.getElementById('settingsResult').textContent='Select at least one Entity field for the active profile.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:entityFields[0]||'',entity_fields:entityFields,timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights,relationship_fields:relationshipFields}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
  const savedProfile = window.activeProfileStream ? ` Profile saved for ${window.activeProfileTitle || window.activeProfileStream}.` : ' No profile editor active, so profiles were not changed.';
  const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(values)});
  document.getElementById('settingsResult').textContent = response.ok ? `Saved enabled streams and global settings.${savedProfile} Monitor applies supported settings on its next cycle.` : 'Could not save configuration.';
  if (response.ok) loadSettings();
});
document.querySelectorAll('.tab').forEach(button => button.addEventListener('click', () => {
  document.querySelectorAll('.tab').forEach(item => item.classList.toggle('active', item === button));
  document.querySelectorAll('[data-view]').forEach(view => view.classList.toggle('active', view.dataset.view === button.dataset.tab));
}));
['showReviewedFindings','showLowFindings','showExistingProfiles','showResolvedIncidents'].forEach(id => document.getElementById(id)?.addEventListener('change', refresh));
document.getElementById('correlationGraph')?.addEventListener('click', event => {
  const rect = event.currentTarget.getBoundingClientRect();
  const x = event.clientX - rect.left;
  const y = event.clientY - rect.top;
  const hit = (uiCache.correlationHitboxes || []).find(item => {
    if (Number.isFinite(item.w) && Number.isFinite(item.h)) return x >= item.x && x <= item.x + item.w && y >= item.y && y <= item.y + item.h;
    return Math.hypot(item.x - x, item.y - y) <= item.radius;
  });
  if (!hit) return;
  uiCache.selectedCorrelationKey = hit.key;
  renderCorrelationExplorer(window.currentCorrelations || [], window.currentConfiguration || {});
});
refresh();
loadSettings();
setInterval(refresh, 5000);
</script>
</body>
</html>
"""


def serve_dashboard(host: str, port: int, status_file: str, *, image_dir: str | None = None, config_file: str = "state/fgai-config.json") -> None:
    status_path = Path(status_file)
    image_root = Path(image_dir) if image_dir else None
    config_store = ConfigStore(config_file)

    def read_status() -> dict[str, object]:
        try:
            status = json.loads(status_path.read_text(encoding="utf-8")) if status_path.exists() else {}
        except json.JSONDecodeError:
            status = {}
        if not isinstance(status, dict):
            status = {}
        status.setdefault("status_schema", 2)
        for row in status.get("stream_coverage", []) if isinstance(status.get("stream_coverage"), list) else []:
            if isinstance(row, dict):
                row.setdefault("aggregate_error", "")
                row.setdefault("raw_error", "")
                if row.get("aggregate_status") == "error" and not row.get("aggregate_error") and row.get("error"):
                    row["aggregate_error"] = str(row.get("error", ""))
                if row.get("partial") and not row.get("raw_error") and row.get("error"):
                    row["raw_error"] = str(row.get("error", ""))
        return status

    class Handler(BaseHTTPRequestHandler):
        def do_GET(self) -> None:
            if self.path == "/":
                self._send(200, "text/html; charset=utf-8", HTML.encode("utf-8"))
                return
            if self.path == "/api/config":
                self._send(200, "application/json", json.dumps(config_store.public()).encode("utf-8"))
                return
            if self.path == "/api/feedback":
                self._send(200, "application/json", json.dumps(FeedbackStore().entries()).encode("utf-8"))
                return
            if self.path == "/api/incidents":
                self._send(200, "application/json", json.dumps(IncidentStore().entries()).encode("utf-8"))
                return
            if self.path == "/api/ollama/models":
                self._send(200, "application/json", json.dumps(_ollama_models()).encode("utf-8"))
                return
            if self.path == "/api/graylog/streams":
                config = config_store.read()
                try:
                    client = GraylogMcpClient(str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")), verify_tls=bool(config.get("graylog_tls_verify", True)))
                    client.probe()
                    result = client.call_tool("list_streams", {})
                    content = result.get("result", {}).get("content", [])
                    text = next((item.get("text", "") for item in content if isinstance(item, dict)), "")
                    if isinstance(result.get("result"), dict) and result["result"].get("isError"):
                        fallback = client.call_tool("list_resource", {"resource_type": "stream"})
                        resources = fallback.get("result", {}).get("structuredContent", {}).get("resources", [])
                        streams = [
                            {"id": str(item.get("uri", "")).rsplit(":", 1)[-1], "title": item.get("title", item.get("name", "")), "description": item.get("description", "")}
                            for item in resources if isinstance(item, dict) and item.get("uri")
                        ]
                    else:
                        if not text:
                            raise RuntimeError("Graylog list_streams returned no text content")
                        streams = json.loads(text)
                    body = {"streams": streams, "selected": config.get("graylog_streams", [])}
                    self._send(200, "application/json", json.dumps(body).encode("utf-8"))
                except Exception as exc:
                    self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
                return
            if self.path.startswith("/api/graylog/fields?"):
                stream_id = self.path.split("stream_id=", 1)[-1].split("&", 1)[0]
                config = config_store.read()
                try:
                    client = GraylogMcpClient(str(config.get("graylog_mcp_url", "")), str(config.get("graylog_mcp_token", "")), verify_tls=bool(config.get("graylog_tls_verify", True)))
                    client.probe()
                    result = client.call_tool("list_fields", {"streams": [stream_id]})
                    content = result.get("result", {}).get("content", [])
                    text = next((item.get("text", "") for item in content if isinstance(item, dict)), "")
                    payload = json.loads(text)
                    fields = payload.get("fields", payload) if isinstance(payload, dict) else payload
                    self._send(200, "application/json", json.dumps({"fields": fields}).encode("utf-8"))
                except Exception as exc:
                    self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
                return
            if self.path == "/api/status":
                self._send(200, "application/json", json.dumps(read_status()).encode("utf-8"))
                return
            if self.path.startswith("/api/export/incidents"):
                parsed = urlparse(self.path)
                params = parse_qs(parsed.query)
                fmt = params.get("format", ["markdown"])[0]
                incident_id = params.get("incident_id", [""])[0] or None
                status = read_status()
                report = investigation_report(status, incident_id=incident_id)
                if fmt == "json":
                    self._send(200, "application/json", json.dumps(report, indent=2, sort_keys=True).encode("utf-8"))
                else:
                    self._send(200, "text/markdown; charset=utf-8", investigation_report_markdown(report).encode("utf-8"))
                return
            if self.path == "/metrics":
                status = read_status()
                self._send(200, "text/plain; version=0.0.4; charset=utf-8", prometheus_metrics(status).encode("utf-8"))
                return
            if self.path.startswith("/images/") and image_root:
                image_path = image_root / Path(self.path).name
                if image_path.exists() and image_path.is_file():
                    content_type = "image/png" if image_path.suffix.lower() == ".png" else "application/octet-stream"
                    self._send(200, content_type, image_path.read_bytes())
                    return
            self._send(404, "text/plain; charset=utf-8", b"not found")

        def do_POST(self) -> None:
            if self.path == "/api/feedback" and self._is_loopback_client():
                try:
                    payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8"))
                    self._send(200, "application/json", json.dumps(FeedbackStore().add(payload)).encode("utf-8"))
                except (ValueError, json.JSONDecodeError) as exc:
                    self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
                return
            if self.path == "/api/incidents" and self._is_loopback_client():
                try:
                    payload = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length", "0")), 16_384)).decode("utf-8"))
                    self._send(200, "application/json", json.dumps(IncidentStore().update(str(payload.get("id", "")), str(payload.get("status", "")), str(payload.get("note", "")))).encode("utf-8"))
                except (ValueError, json.JSONDecodeError) as exc:
                    self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
                return
            if self.path != "/api/config" or not self._is_loopback_client():
                self._send(403, "application/json", b'{"error":"configuration is local-only"}')
                return
            try:
                length = int(self.headers.get("Content-Length", "0"))
                payload = json.loads(self.rfile.read(min(length, 2_097_152)).decode("utf-8"))
                if not isinstance(payload, dict):
                    raise ValueError("configuration must be an object")
                body = json.dumps(config_store.update(payload)).encode("utf-8")
            except (ValueError, json.JSONDecodeError) as exc:
                self._send(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8"))
                return
            self._send(200, "application/json", body)

        def _is_loopback_client(self) -> bool:
            return self.client_address[0] in {"127.0.0.1", "::1"}

        def log_message(self, format: str, *args: object) -> None:
            return

        def _send(self, status: int, content_type: str, body: bytes) -> None:
            self.send_response(status)
            self.send_header("Content-Type", content_type)
            self.send_header("Content-Length", str(len(body)))
            self.end_headers()
            self.wfile.write(body)

    server = ThreadingHTTPServer((host, port), Handler)
    print(f"Dashboard listening on http://{host}:{port}")
    print(f"Reading status from {status_path}")
    server.serve_forever()
