from __future__ import annotations import json import time from pathlib import Path from .anomaly import anomaly_summary, detect_source_anomalies from .baseline import BaselineStore from .config import ConfigStore from .correlation import correlate_source_ips from .event_context import build_event_context from .feedback import FeedbackStore from .graylog_mcp import GraylogMcpClient from .graylog_source import GraylogStreamSource from .history import HistoryStore from .incidents import IncidentStore, build_incidents from .data_quality import assess_data_quality from .llm import ollama_dashboard_assessment, ollama_profile_advice from .logs import local_in_failures, read_events, summarize_events, top_field_values from .mitigation import parse_allowlist, suggest_block_candidates from .policies import audit_policies, read_policies from .profile_suggestions import apply_profile_advice, suggest_stream_profiles from .recommendations import build_recommendations from .sequences import detect_sequences from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip from .triage import build_triage_queue from .stream_profiles import parse_profiles def _stream_titles(config: dict[str, object]) -> dict[str, str]: return { str(item.get("id", "")): str(item.get("title", "") or item.get("id", "")) for item in config.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") } def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str: return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id def _profile_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str: name = str(getattr(profile, "name", "") or "").strip() if not name or name == stream_id: return f"{_stream_name(stream_id, stream_titles, profile)} profile" return name def _range_seconds(value: object) -> int: try: return max(60, int(value)) except (TypeError, ValueError): return 300 def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[str, object], stream_status: dict[str, object], profile_readiness: list[dict[str, object]], stream_titles: dict[str, str]) -> list[dict[str, object]]: configured = [ item for item in runtime_values.get("graylog_streams", []) if isinstance(item, dict) and item.get("id") ] status_by_id = { str(item.get("stream_id", "")): item for item in stream_status.get("streams", []) if isinstance(item, dict) and item.get("stream_id") } readiness_by_stream: dict[str, list[dict[str, object]]] = {} for item in profile_readiness: readiness_by_stream.setdefault(str(item.get("stream_id", "")), []).append(item) ids = list(dict.fromkeys([str(item.get("id", "")) for item in configured] + list(stream_profiles) + list(status_by_id))) rows = [] for stream_id in ids: profile = stream_profiles.get(stream_id) readiness = readiness_by_stream.get(stream_id, []) ready_fields = sum(1 for item in readiness if item.get("ready")) total_fields = len(readiness) status = status_by_id.get(stream_id, {}) enabled = next((bool(item.get("enabled")) for item in configured if str(item.get("id", "")) == stream_id), False) rows.append({ "stream_id": stream_id, "stream_name": _stream_name(stream_id, stream_titles, profile), "enabled": enabled, "profile": _profile_name(stream_id, stream_titles, profile) if profile else "", "profile_ready": bool(profile), "entity_field": ", ".join(getattr(profile, "entity_fields", ()) or (str(getattr(profile, "entity_field", "")),)) if profile else "", "tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) if profile else 0, "ready_fields": ready_fields, "total_fields": total_fields, "readiness": f"{ready_fields}/{total_fields}" if total_fields else "0/0", "events_fetched": int(status.get("events_fetched", 0) or 0), "latest_event_time": str(status.get("latest_event_time", "")), "truncated": bool(status.get("truncated")), "partial": bool(status.get("partial")), "error": str(status.get("error", "")), "health": "not_enabled" if not enabled else "partial_fetch" if status.get("partial") else "missing_profile" if not profile else "no_events" if int(status.get("events_fetched", 0) or 0) == 0 else "learning" if total_fields and ready_fields < total_fields else "ready" if total_fields else "profile_needs_fields", }) return rows def build_status( log_path: str, *, policy_path: str | None = None, min_block_events: int = 3, min_block_score: int = 7, anomaly_limit: int = 20, baseline_path: str | None = None, config_path: str | None = None, history_path: str | None = None, incident_path: str | None = None, ) -> dict[str, object]: config_store = ConfigStore(config_path) if config_path else None config_exists = bool(config_store and config_store.path.exists()) runtime_values = config_store.read() if config_exists and config_store else {} runtime_config = config_store.public() if config_store else {} stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", [])) stream_titles = _stream_titles(runtime_values) events = read_events(log_path) if Path(log_path).exists() else [] mcp_status: dict[str, object] = {"status": "not_configured"} if runtime_values.get("log_source") == "graylog_mcp": url, token = str(runtime_values.get("graylog_mcp_url", "")), str(runtime_values.get("graylog_mcp_token", "")) if not url or not token: mcp_status = {"status": "missing_configuration"} events = [] else: try: configured_streams = runtime_values.get("graylog_streams", []) stream_configs = [item for item in configured_streams if isinstance(item, dict) and item.get("enabled") and item.get("id")] stream_ids = [str(item.get("id")) for item in stream_configs] if not stream_ids: stream_configs = [{"id": str(runtime_values.get("graylog_stream", "")), "title": "Graylog"}] stream_statuses = [] events = [] range_seconds = _range_seconds(runtime_values.get("graylog_range_seconds", 300)) max_events_per_stream = max(1, int(runtime_values.get("graylog_max_events_per_stream", 5000) or 5000)) for stream_config in stream_configs: stream_id = str(stream_config["id"]) profile = stream_profiles.get(stream_id) profile_fields = ( str(getattr(profile, "entity_field", "")), *tuple(str(field) for field in getattr(profile, "entity_fields", ())), str(getattr(profile, "timestamp_field", "")), *tuple(str(field) for field in getattr(profile, "categorical_fields", ())), *tuple(str(field) for field in getattr(profile, "numeric_fields", ())), ) if profile else () stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id) stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(max_events=max_events_per_stream, range_seconds=range_seconds) events.extend(stream_events) stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status}) truncated_streams = [item for item in stream_statuses if item.get("truncated")] partial_streams = [item for item in stream_statuses if item.get("partial")] warnings = [] if partial_streams: warnings.append(f"{len(partial_streams)} stream(s) returned a partial MCP fetch; Graylog likely timed out or rejected a large paged query.") if truncated_streams: warnings.append(f"{len(truncated_streams)} stream(s) hit max_events_per_stream; high EPS means the analysis window is only partially sampled.") mcp_status = { "status": "partial" if partial_streams else "connected", "streams": stream_statuses, "events_fetched": len(events), "range_seconds": range_seconds, "max_events_per_stream": max_events_per_stream, "partial_streams": len(partial_streams), "truncated_streams": len(truncated_streams), "coverage_status": "partial" if partial_streams else "truncated" if truncated_streams else "complete_window", "coverage_warning": " ".join(warnings), } except RuntimeError as exc: mcp_status = {"status": "error", "error": str(exc)} events = [] baseline = BaselineStore(baseline_path) if baseline_path else None profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {} baseline_training_days = int(runtime_values.get("baseline_training_days", 7) or 7) field_deviations = baseline.profile_deviations(events, stream_profiles, min_training_days=baseline_training_days) if baseline else {} sequence_findings = detect_sequences(events) for entity, findings in sequence_findings.items(): field_deviations.setdefault(entity, []).extend(findings) for deviations in field_deviations.values(): for deviation in deviations: stream_id = str(deviation.get("stream_id", "")) profile = stream_profiles.get(stream_id) name = _stream_name(stream_id, stream_titles, profile) deviation["stream_name"] = name deviation["stream_title"] = name deviation["profile_name"] = _profile_name(stream_id, stream_titles, profile) deviation["sample_events"] = [ {"stream": name, **sample} if isinstance(sample, dict) and not sample.get("stream") else sample for sample in deviation.get("sample_events", []) ] feedback = FeedbackStore().entries() for entity, deviations in field_deviations.items(): for deviation in deviations: match = next(( item for item in feedback if item.get("entity") == entity and item.get("stream_id") == deviation.get("stream_id") and item.get("field") == deviation.get("field") and (not item.get("value") or item.get("value") == deviation.get("value", "")) ), None) if match: deviation["feedback"] = match["status"] if match["status"] in {"false_positive", "expected"}: deviation["score"] = 0 anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations) baseline_events = baseline.ingest(events) if baseline else 0 profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0 baseline_maintenance = ( baseline.maintenance( retention_days=int(runtime_values.get("baseline_retention_days", 14) or 14), value_retention_days=int(runtime_values.get("baseline_value_retention_days", 7) or 7), max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 2000) or 2000), ) if baseline else {} ) profile_readiness = baseline.profile_readiness(stream_profiles, min_training_days=baseline_training_days) if baseline else [] profile_readiness = [ { **item, "profile_name": _profile_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))), "stream_name": _stream_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))), } for item in profile_readiness ] stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles) profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles) profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"} if runtime_values.get("profile_advisor_enabled") and profile_suggestions: try: advice = ollama_profile_advice( profile_suggestions, model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"), timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120), ) profile_suggestions = apply_profile_advice(profile_suggestions, advice) profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")} except Exception as exc: profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")} intel_ips = sorted( { ip for event in events for ip in (event.src_ip, event.dst_ip) if is_public_ip(ip) } ) threat_enabled = bool(runtime_values.get("threat_intel_enabled")) if runtime_values else None reputation = enrich_ips(intel_ips, limit=25, enabled=threat_enabled, config=runtime_values) threat_intel_status = ThreatIntelClient( enabled=threat_enabled, provider=str(runtime_values.get("threat_intel_provider", "auto")), abuseipdb_key=str(runtime_values.get("abuseipdb_api_key", "") or "") or None, virustotal_key=str(runtime_values.get("virustotal_api_key", "") or "") or None, daily_limit=int(runtime_values.get("threat_intel_daily_limit", 100) or 100), ttl_seconds=int(runtime_values.get("threat_intel_ttl_seconds", 604800) or 604800), error_ttl_seconds=int(runtime_values.get("threat_intel_error_ttl_seconds", 3600) or 3600), abuseipdb_max_age_days=int(runtime_values.get("abuseipdb_max_age_days", 90) or 90), ).status() recommendations = build_recommendations(events, anomalies, reputation) correlations = correlate_source_ips(events) incidents = IncidentStore(incident_path or "state/signalscope-incidents.json").apply(build_incidents(anomalies, field_deviations, correlations)) triage_queue = build_triage_queue(incidents, field_deviations, correlations, recommendations) block_candidates = suggest_block_candidates( events, min_events=min_block_events, min_score=min_block_score, allowlist=parse_allowlist(), ) policy_findings: list[dict[str, str | None]] = [] policy_error: str | None = None if policy_path and Path(policy_path).exists(): try: policy_findings = [finding.__dict__ for finding in audit_policies(read_policies(policy_path))] except Exception as exc: policy_error = str(exc) status = { "generated_at": int(time.time()), "log_path": log_path, "policy_path": policy_path, "summary": summarize_events(events), "anomaly_summary": anomaly_summary(anomalies), "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0}, "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status}, "configuration": runtime_config, "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()], "stream_coverage": stream_coverage, "profile_suggestions": profile_suggestions, "profile_readiness": profile_readiness, "diagnostics": { "top_source_ips": top_field_values(events, "srcip", limit=10), "top_destination_ips": top_field_values(events, "dstip", limit=10), "top_policy_ids": top_field_values(events, "policyid", limit=10), "top_destination_ports": top_field_values(events, "dstport", limit=10), "top_source_ports": top_field_values(events, "srcport", limit=10), "top_services": top_field_values(events, "service", limit=10), "top_actions": top_field_values(events, "action", limit=10), "top_subtypes": top_field_values(events, "subtype", limit=10), "local_in_failures": local_in_failures(events, limit=10), }, "event_context": build_event_context(events), "field_deviations": field_deviations, "triage_queue": triage_queue, "sequence_findings": sequence_findings, "feedback": feedback, "cross_source_correlations": correlations, "incidents": incidents, "data_quality": assess_data_quality(events, mcp_status), "anomalies": [ { "subject": finding.subject, "score": finding.score, "severity": finding.severity, "confidence": finding.confidence, "reasons": finding.reasons, "evidence": finding.evidence, } for finding in anomalies ], "recommendations": [ { "subject": item.subject, "score": item.score, "severity": item.severity, "title": item.title, "recommendation": item.recommendation, "reasons": item.reasons, "related_policy_ids": item.related_policy_ids, "related_services": item.related_services, } for item in recommendations ], "reputation": reputation, "block_candidates": [ {"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons} for candidate in block_candidates ], "policy_findings": policy_findings, "policy_error": policy_error, } if history_path: history = HistoryStore(history_path) history.record(status) status["history"] = history.recent() return status def add_llm_assessment(status: dict[str, object], *, previous: str | None = None, model: str | None = None, timeout: int | None = None) -> None: try: status["llm_assessment"] = { "enabled": True, "status": "ok", "generated_at": int(time.time()), "text": ollama_dashboard_assessment(status, model=model, timeout=timeout), } except Exception as exc: status["llm_assessment"] = { "enabled": True, "status": "error", "generated_at": int(time.time()), "error": str(exc), "text": previous or "", } def write_status(status: dict[str, object], output: str) -> None: output_path = Path(output) output_path.parent.mkdir(parents=True, exist_ok=True) tmp_path = output_path.with_suffix(f"{output_path.suffix}.tmp") tmp_path.write_text(json.dumps(status, indent=2, sort_keys=True), encoding="utf-8") tmp_path.replace(output_path) def monitor_loop( log_path: str, output: str, *, policy_path: str | None = None, interval: int = 10, anomaly_limit: int = 20, llm: bool = False, llm_interval: int = 300, llm_model: str | None = None, llm_timeout: int | None = None, baseline_path: str | None = None, config_path: str | None = None, history_path: str | None = None, ) -> None: print(f"Monitoring {log_path}") print(f"Writing status to {output}") last_llm_at = 0 last_llm_text: str | None = None while True: runtime = ConfigStore(config_path).read() if config_path and Path(config_path).exists() else {} effective_llm = bool(runtime.get("llm_enabled")) if runtime else llm effective_model = str(runtime.get("llm_model") or llm_model or "") status = build_status( log_path, policy_path=policy_path, anomaly_limit=anomaly_limit, baseline_path=baseline_path, config_path=config_path, history_path=history_path, ) if effective_llm: now = int(time.time()) if now - last_llm_at >= llm_interval: add_llm_assessment(status, previous=last_llm_text, model=effective_model or None, timeout=llm_timeout) assessment = status.get("llm_assessment", {}) if isinstance(assessment, dict): last_llm_text = str(assessment.get("text", "") or last_llm_text or "") last_llm_at = now else: status["llm_assessment"] = { "enabled": True, "status": "cached", "generated_at": last_llm_at, "text": last_llm_text or "", } else: status["llm_assessment"] = {"enabled": False, "status": "disabled", "text": ""} write_status(status, output) time.sleep(interval)