add chache for last known good in sqlite..

This commit is contained in:
larssand
2026-06-30 10:30:59 +02:00
parent 50463aa4d6
commit 2aaa2e28f3
6 changed files with 91 additions and 2 deletions

View File

@@ -213,6 +213,7 @@ def run_monitor(args: argparse.Namespace) -> int:
baseline_path=args.baseline_db,
config_path=args.config_file,
history_path=args.history_db,
status_cache_path=args.status_cache_db,
)
return 0
@@ -421,6 +422,7 @@ def build_parser() -> argparse.ArgumentParser:
monitor.add_argument("--baseline-db", default="state/fgai-baseline.sqlite3", help="SQLite database for historical behavior baselines")
monitor.add_argument("--config-file", default="state/fgai-config.json", help="Runtime configuration written by dashboard")
monitor.add_argument("--history-db", default="state/signalscope-history.sqlite3", help="Aggregate history database for dashboard trends")
monitor.add_argument("--status-cache-db", default="state/signalscope-status-cache.sqlite3", help="SQLite cache for last successful dashboard status")
monitor.add_argument("--llm", action="store_true", help="Generate cached Ollama analyst note for dashboard")
monitor.add_argument("--llm-interval", type=int, default=300, help="Seconds between Ollama dashboard assessments")
monitor.add_argument("--model", default=None, help="Ollama model name")

View File

@@ -233,6 +233,7 @@ async function refresh() {
const threat = (data.capabilities || {}).threat_intel || {};
const mcp = (data.capabilities || {}).graylog_mcp || {};
const configuration = data.configuration || {};
const cache = data.status_cache || {};
const streamCoverage = data.stream_coverage || [];
const enabledStreams = streamCoverage.filter(item => item.enabled);
const streamsMissingProfile = enabledStreams.filter(item => !item.profile_ready).length;
@@ -258,6 +259,8 @@ async function refresh() {
capability('Graylog MCP', mcp.status === 'connected' ? 'on' : 'warn', configuration.log_source === 'graylog_mcp' ? (mcp.status || 'checking') : 'not selected')
].join('');
document.getElementById('liveStatus').innerHTML = [
data.stale ? `<span class="sev-high">Showing cached dashboard data because live MCP fetch failed.</span>` : '',
cache.stored_at ? `Cached snapshot: ${esc(new Date(cache.stored_at * 1000).toLocaleString())}` : '',
`Log file: <code>${esc(data.log_path || '')}</code>`,
`Policy file: <code>${esc(data.policy_path || 'none')}</code>`,
`Critical anomalies: ${esc((a.critical || 0))}`,

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import json
import sqlite3
import time
from pathlib import Path
@@ -26,3 +27,34 @@ class HistoryStore:
with sqlite3.connect(self.path) as connection:
rows = connection.execute("select at, events, anomalies, critical, high, baseline_ready, mcp_events from snapshots order by at desc limit ?", (limit,)).fetchall()
return [dict(zip(("at", "events", "anomalies", "critical", "high", "baseline_ready", "mcp_events"), row)) for row in reversed(rows)]
class StatusSnapshotStore:
def __init__(self, path: str) -> None:
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
with sqlite3.connect(self.path) as connection:
connection.execute("create table if not exists status_snapshots (name text primary key, at integer not null, payload text not null)")
def save(self, name: str, status: dict[str, object]) -> None:
with sqlite3.connect(self.path) as connection:
connection.execute(
"insert or replace into status_snapshots values (?, ?, ?)",
(name, int(time.time()), json.dumps(status, sort_keys=True)),
)
def load(self, name: str) -> dict[str, object] | None:
with sqlite3.connect(self.path) as connection:
row = connection.execute("select at, payload from status_snapshots where name = ?", (name,)).fetchone()
if not row:
return None
try:
payload = json.loads(str(row[1]))
except json.JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
payload.setdefault("status_cache", {})
if isinstance(payload["status_cache"], dict):
payload["status_cache"]["stored_at"] = int(row[0])
return payload

View File

@@ -12,7 +12,7 @@ from .event_context import build_event_context
from .feedback import FeedbackStore
from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource
from .history import HistoryStore
from .history import HistoryStore, StatusSnapshotStore
from .incidents import IncidentStore, build_incidents
from .data_quality import assess_data_quality
from .llm import ollama_dashboard_assessment, ollama_profile_advice
@@ -107,6 +107,7 @@ def build_status(
config_path: str | None = None,
history_path: str | None = None,
incident_path: str | None = None,
status_cache_path: str | None = None,
) -> dict[str, object]:
config_store = ConfigStore(config_path) if config_path else None
config_exists = bool(config_store and config_store.path.exists())
@@ -342,9 +343,27 @@ def build_status(
history = HistoryStore(history_path)
history.record(status)
status["history"] = history.recent()
if status_cache_path and not (isinstance(mcp_status, dict) and mcp_status.get("status") == "error"):
StatusSnapshotStore(status_cache_path).save("last_good", status)
return status
def cached_status_with_error(cache_path: str, error_status: dict[str, object]) -> dict[str, object] | None:
cached = StatusSnapshotStore(cache_path).load("last_good")
if not cached:
return None
cached["generated_at"] = int(time.time())
cached["stale"] = True
cached["stale_reason"] = "live_mcp_error"
capabilities = cached.setdefault("capabilities", {})
if isinstance(capabilities, dict):
capabilities["graylog_mcp"] = error_status
cached.setdefault("status_cache", {})
if isinstance(cached["status_cache"], dict):
cached["status_cache"].update({"served_from_cache": True, "reason": "live_mcp_error"})
return cached
def add_llm_assessment(status: dict[str, object], *, previous: str | None = None, model: str | None = None, timeout: int | None = None) -> None:
try:
status["llm_assessment"] = {
@@ -385,6 +404,7 @@ def monitor_loop(
baseline_path: str | None = None,
config_path: str | None = None,
history_path: str | None = None,
status_cache_path: str | None = None,
) -> None:
print(f"Monitoring {log_path}")
print(f"Writing status to {output}")
@@ -397,7 +417,13 @@ def monitor_loop(
status = build_status(
log_path, policy_path=policy_path, anomaly_limit=anomaly_limit,
baseline_path=baseline_path, config_path=config_path, history_path=history_path,
status_cache_path=status_cache_path,
)
mcp = status.get("capabilities", {}).get("graylog_mcp", {}) if isinstance(status.get("capabilities"), dict) else {}
if status_cache_path and isinstance(mcp, dict) and mcp.get("status") == "error":
cached = cached_status_with_error(status_cache_path, mcp)
if cached:
status = cached
if effective_llm:
now = int(time.time())
if now - last_llm_at >= llm_interval: