420 lines
22 KiB
Python
420 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from .anomaly import anomaly_summary, detect_source_anomalies
|
|
from .baseline import BaselineStore
|
|
from .config import ConfigStore
|
|
from .correlation import correlate_source_ips
|
|
from .event_context import build_event_context
|
|
from .feedback import FeedbackStore
|
|
from .graylog_mcp import GraylogMcpClient
|
|
from .graylog_source import GraylogStreamSource
|
|
from .history import HistoryStore
|
|
from .incidents import IncidentStore, build_incidents
|
|
from .data_quality import assess_data_quality
|
|
from .llm import ollama_dashboard_assessment, ollama_profile_advice
|
|
from .logs import local_in_failures, read_events, summarize_events, top_field_values
|
|
from .mitigation import parse_allowlist, suggest_block_candidates
|
|
from .policies import audit_policies, read_policies
|
|
from .profile_suggestions import apply_profile_advice, suggest_stream_profiles
|
|
from .recommendations import build_recommendations
|
|
from .sequences import detect_sequences
|
|
from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
|
|
from .triage import build_triage_queue
|
|
from .stream_profiles import parse_profiles
|
|
|
|
|
|
def _stream_titles(config: dict[str, object]) -> dict[str, str]:
|
|
return {
|
|
str(item.get("id", "")): str(item.get("title", "") or item.get("id", ""))
|
|
for item in config.get("graylog_streams", [])
|
|
if isinstance(item, dict) and item.get("id")
|
|
}
|
|
|
|
|
|
def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
|
|
return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
|
|
|
|
|
|
def _profile_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
|
|
name = str(getattr(profile, "name", "") or "").strip()
|
|
if not name or name == stream_id:
|
|
return f"{_stream_name(stream_id, stream_titles, profile)} profile"
|
|
return name
|
|
|
|
|
|
def _range_seconds(value: object) -> int:
|
|
try:
|
|
return max(60, int(value))
|
|
except (TypeError, ValueError):
|
|
return 300
|
|
|
|
|
|
def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[str, object], stream_status: dict[str, object], profile_readiness: list[dict[str, object]], stream_titles: dict[str, str]) -> list[dict[str, object]]:
|
|
configured = [
|
|
item for item in runtime_values.get("graylog_streams", [])
|
|
if isinstance(item, dict) and item.get("id")
|
|
]
|
|
status_by_id = {
|
|
str(item.get("stream_id", "")): item
|
|
for item in stream_status.get("streams", [])
|
|
if isinstance(item, dict) and item.get("stream_id")
|
|
}
|
|
readiness_by_stream: dict[str, list[dict[str, object]]] = {}
|
|
for item in profile_readiness:
|
|
readiness_by_stream.setdefault(str(item.get("stream_id", "")), []).append(item)
|
|
ids = list(dict.fromkeys([str(item.get("id", "")) for item in configured] + list(stream_profiles) + list(status_by_id)))
|
|
rows = []
|
|
for stream_id in ids:
|
|
profile = stream_profiles.get(stream_id)
|
|
readiness = readiness_by_stream.get(stream_id, [])
|
|
ready_fields = sum(1 for item in readiness if item.get("ready"))
|
|
total_fields = len(readiness)
|
|
status = status_by_id.get(stream_id, {})
|
|
enabled = next((bool(item.get("enabled")) for item in configured if str(item.get("id", "")) == stream_id), False)
|
|
rows.append({
|
|
"stream_id": stream_id,
|
|
"stream_name": _stream_name(stream_id, stream_titles, profile),
|
|
"enabled": enabled,
|
|
"profile": _profile_name(stream_id, stream_titles, profile) if profile else "",
|
|
"profile_ready": bool(profile),
|
|
"entity_field": ", ".join(getattr(profile, "entity_fields", ()) or (str(getattr(profile, "entity_field", "")),)) if profile else "",
|
|
"tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) if profile else 0,
|
|
"ready_fields": ready_fields,
|
|
"total_fields": total_fields,
|
|
"readiness": f"{ready_fields}/{total_fields}" if total_fields else "0/0",
|
|
"events_fetched": int(status.get("events_fetched", 0) or 0),
|
|
"latest_event_time": str(status.get("latest_event_time", "")),
|
|
"truncated": bool(status.get("truncated")),
|
|
"partial": bool(status.get("partial")),
|
|
"error": str(status.get("error", "")),
|
|
"health": "not_enabled" if not enabled else "partial_fetch" if status.get("partial") else "missing_profile" if not profile else "no_events" if int(status.get("events_fetched", 0) or 0) == 0 else "learning" if total_fields and ready_fields < total_fields else "ready" if total_fields else "profile_needs_fields",
|
|
})
|
|
return rows
|
|
|
|
|
|
def build_status(
|
|
log_path: str,
|
|
*,
|
|
policy_path: str | None = None,
|
|
min_block_events: int = 3,
|
|
min_block_score: int = 7,
|
|
anomaly_limit: int = 20,
|
|
baseline_path: str | None = None,
|
|
config_path: str | None = None,
|
|
history_path: str | None = None,
|
|
incident_path: str | None = None,
|
|
) -> dict[str, object]:
|
|
config_store = ConfigStore(config_path) if config_path else None
|
|
config_exists = bool(config_store and config_store.path.exists())
|
|
runtime_values = config_store.read() if config_exists and config_store else {}
|
|
runtime_config = config_store.public() if config_store else {}
|
|
stream_profiles = parse_profiles(runtime_values.get("graylog_stream_profiles", []))
|
|
stream_titles = _stream_titles(runtime_values)
|
|
events = read_events(log_path) if Path(log_path).exists() else []
|
|
mcp_status: dict[str, object] = {"status": "not_configured"}
|
|
if runtime_values.get("log_source") == "graylog_mcp":
|
|
url, token = str(runtime_values.get("graylog_mcp_url", "")), str(runtime_values.get("graylog_mcp_token", ""))
|
|
if not url or not token:
|
|
mcp_status = {"status": "missing_configuration"}
|
|
events = []
|
|
else:
|
|
try:
|
|
configured_streams = runtime_values.get("graylog_streams", [])
|
|
stream_configs = [item for item in configured_streams if isinstance(item, dict) and item.get("enabled") and item.get("id")]
|
|
stream_ids = [str(item.get("id")) for item in stream_configs]
|
|
if not stream_ids:
|
|
stream_configs = [{"id": str(runtime_values.get("graylog_stream", "")), "title": "Graylog"}]
|
|
stream_statuses = []
|
|
events = []
|
|
range_seconds = _range_seconds(runtime_values.get("graylog_range_seconds", 300))
|
|
max_events_per_stream = max(1, int(runtime_values.get("graylog_max_events_per_stream", 5000) or 5000))
|
|
for stream_config in stream_configs:
|
|
stream_id = str(stream_config["id"])
|
|
profile = stream_profiles.get(stream_id)
|
|
profile_fields = (
|
|
str(getattr(profile, "entity_field", "")),
|
|
*tuple(str(field) for field in getattr(profile, "entity_fields", ())),
|
|
str(getattr(profile, "timestamp_field", "")),
|
|
*tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
|
|
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
|
|
) if profile else ()
|
|
stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
|
|
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(max_events=max_events_per_stream, range_seconds=range_seconds)
|
|
events.extend(stream_events)
|
|
stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status})
|
|
truncated_streams = [item for item in stream_statuses if item.get("truncated")]
|
|
partial_streams = [item for item in stream_statuses if item.get("partial")]
|
|
warnings = []
|
|
if partial_streams:
|
|
warnings.append(f"{len(partial_streams)} stream(s) returned a partial MCP fetch; Graylog likely timed out or rejected a large paged query.")
|
|
if truncated_streams:
|
|
warnings.append(f"{len(truncated_streams)} stream(s) hit max_events_per_stream; high EPS means the analysis window is only partially sampled.")
|
|
mcp_status = {
|
|
"status": "partial" if partial_streams else "connected",
|
|
"streams": stream_statuses,
|
|
"events_fetched": len(events),
|
|
"range_seconds": range_seconds,
|
|
"max_events_per_stream": max_events_per_stream,
|
|
"partial_streams": len(partial_streams),
|
|
"truncated_streams": len(truncated_streams),
|
|
"coverage_status": "partial" if partial_streams else "truncated" if truncated_streams else "complete_window",
|
|
"coverage_warning": " ".join(warnings),
|
|
}
|
|
except RuntimeError as exc:
|
|
mcp_status = {"status": "error", "error": str(exc)}
|
|
events = []
|
|
baseline = BaselineStore(baseline_path) if baseline_path else None
|
|
profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
|
|
baseline_training_days = int(runtime_values.get("baseline_training_days", 7) or 7)
|
|
field_deviations = baseline.profile_deviations(events, stream_profiles, min_training_days=baseline_training_days) if baseline else {}
|
|
sequence_findings = detect_sequences(events)
|
|
for entity, findings in sequence_findings.items():
|
|
field_deviations.setdefault(entity, []).extend(findings)
|
|
for deviations in field_deviations.values():
|
|
for deviation in deviations:
|
|
stream_id = str(deviation.get("stream_id", ""))
|
|
profile = stream_profiles.get(stream_id)
|
|
name = _stream_name(stream_id, stream_titles, profile)
|
|
deviation["stream_name"] = name
|
|
deviation["stream_title"] = name
|
|
deviation["profile_name"] = _profile_name(stream_id, stream_titles, profile)
|
|
deviation["sample_events"] = [
|
|
{"stream": name, **sample} if isinstance(sample, dict) and not sample.get("stream") else sample
|
|
for sample in deviation.get("sample_events", [])
|
|
]
|
|
feedback = FeedbackStore().entries()
|
|
for entity, deviations in field_deviations.items():
|
|
for deviation in deviations:
|
|
match = next((
|
|
item for item in feedback
|
|
if item.get("entity") == entity
|
|
and item.get("stream_id") == deviation.get("stream_id")
|
|
and item.get("field") == deviation.get("field")
|
|
and (not item.get("value") or item.get("value") == deviation.get("value", ""))
|
|
), None)
|
|
if match:
|
|
deviation["feedback"] = match["status"]
|
|
if match["status"] in {"false_positive", "expected"}:
|
|
deviation["score"] = 0
|
|
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
|
|
baseline_events = baseline.ingest(events) if baseline else 0
|
|
profile_baseline_fields = baseline.ingest_profile_fields(events, stream_profiles) if baseline else 0
|
|
baseline_maintenance = (
|
|
baseline.maintenance(
|
|
retention_days=int(runtime_values.get("baseline_retention_days", 14) or 14),
|
|
value_retention_days=int(runtime_values.get("baseline_value_retention_days", 7) or 7),
|
|
max_values_per_field=int(runtime_values.get("baseline_max_values_per_field", 2000) or 2000),
|
|
)
|
|
if baseline
|
|
else {}
|
|
)
|
|
profile_readiness = baseline.profile_readiness(stream_profiles, min_training_days=baseline_training_days) if baseline else []
|
|
profile_readiness = [
|
|
{
|
|
**item,
|
|
"profile_name": _profile_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))),
|
|
"stream_name": _stream_name(str(item.get("stream_id", "")), stream_titles, stream_profiles.get(str(item.get("stream_id", "")))),
|
|
}
|
|
for item in profile_readiness
|
|
]
|
|
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
|
|
profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles)
|
|
profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"}
|
|
if runtime_values.get("profile_advisor_enabled") and profile_suggestions:
|
|
try:
|
|
advice = ollama_profile_advice(
|
|
profile_suggestions,
|
|
model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
|
|
timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120),
|
|
)
|
|
profile_suggestions = apply_profile_advice(profile_suggestions, advice)
|
|
profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
|
|
except Exception as exc:
|
|
profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
|
|
intel_ips = sorted(
|
|
{
|
|
ip
|
|
for event in events
|
|
for ip in (event.src_ip, event.dst_ip)
|
|
if is_public_ip(ip)
|
|
}
|
|
)
|
|
threat_enabled = bool(runtime_values.get("threat_intel_enabled")) if runtime_values else None
|
|
reputation = enrich_ips(intel_ips, limit=25, enabled=threat_enabled, config=runtime_values)
|
|
threat_intel_status = ThreatIntelClient(
|
|
enabled=threat_enabled,
|
|
provider=str(runtime_values.get("threat_intel_provider", "auto")),
|
|
abuseipdb_key=str(runtime_values.get("abuseipdb_api_key", "") or "") or None,
|
|
virustotal_key=str(runtime_values.get("virustotal_api_key", "") or "") or None,
|
|
daily_limit=int(runtime_values.get("threat_intel_daily_limit", 100) or 100),
|
|
ttl_seconds=int(runtime_values.get("threat_intel_ttl_seconds", 604800) or 604800),
|
|
error_ttl_seconds=int(runtime_values.get("threat_intel_error_ttl_seconds", 3600) or 3600),
|
|
abuseipdb_max_age_days=int(runtime_values.get("abuseipdb_max_age_days", 90) or 90),
|
|
).status()
|
|
recommendations = build_recommendations(events, anomalies, reputation)
|
|
correlations = correlate_source_ips(events)
|
|
incidents = IncidentStore(incident_path or "state/signalscope-incidents.json").apply(build_incidents(anomalies, field_deviations, correlations))
|
|
triage_queue = build_triage_queue(incidents, field_deviations, correlations, recommendations)
|
|
block_candidates = suggest_block_candidates(
|
|
events,
|
|
min_events=min_block_events,
|
|
min_score=min_block_score,
|
|
allowlist=parse_allowlist(),
|
|
)
|
|
|
|
policy_findings: list[dict[str, str | None]] = []
|
|
policy_error: str | None = None
|
|
if policy_path and Path(policy_path).exists():
|
|
try:
|
|
policy_findings = [finding.__dict__ for finding in audit_policies(read_policies(policy_path))]
|
|
except Exception as exc:
|
|
policy_error = str(exc)
|
|
|
|
status = {
|
|
"generated_at": int(time.time()),
|
|
"log_path": log_path,
|
|
"policy_path": policy_path,
|
|
"summary": summarize_events(events),
|
|
"anomaly_summary": anomaly_summary(anomalies),
|
|
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
|
|
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
|
|
"configuration": runtime_config,
|
|
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
|
|
"stream_coverage": stream_coverage,
|
|
"profile_suggestions": profile_suggestions,
|
|
"profile_readiness": profile_readiness,
|
|
"diagnostics": {
|
|
"top_source_ips": top_field_values(events, "srcip", limit=10),
|
|
"top_destination_ips": top_field_values(events, "dstip", limit=10),
|
|
"top_policy_ids": top_field_values(events, "policyid", limit=10),
|
|
"top_destination_ports": top_field_values(events, "dstport", limit=10),
|
|
"top_source_ports": top_field_values(events, "srcport", limit=10),
|
|
"top_services": top_field_values(events, "service", limit=10),
|
|
"top_actions": top_field_values(events, "action", limit=10),
|
|
"top_subtypes": top_field_values(events, "subtype", limit=10),
|
|
"local_in_failures": local_in_failures(events, limit=10),
|
|
},
|
|
"event_context": build_event_context(events),
|
|
"field_deviations": field_deviations,
|
|
"triage_queue": triage_queue,
|
|
"sequence_findings": sequence_findings,
|
|
"feedback": feedback,
|
|
"cross_source_correlations": correlations,
|
|
"incidents": incidents,
|
|
"data_quality": assess_data_quality(events, mcp_status),
|
|
"anomalies": [
|
|
{
|
|
"subject": finding.subject,
|
|
"score": finding.score,
|
|
"severity": finding.severity,
|
|
"confidence": finding.confidence,
|
|
"reasons": finding.reasons,
|
|
"evidence": finding.evidence,
|
|
}
|
|
for finding in anomalies
|
|
],
|
|
"recommendations": [
|
|
{
|
|
"subject": item.subject,
|
|
"score": item.score,
|
|
"severity": item.severity,
|
|
"title": item.title,
|
|
"recommendation": item.recommendation,
|
|
"reasons": item.reasons,
|
|
"related_policy_ids": item.related_policy_ids,
|
|
"related_services": item.related_services,
|
|
}
|
|
for item in recommendations
|
|
],
|
|
"reputation": reputation,
|
|
"block_candidates": [
|
|
{"src_ip": candidate.src_ip, "score": candidate.score, "reasons": candidate.reasons}
|
|
for candidate in block_candidates
|
|
],
|
|
"policy_findings": policy_findings,
|
|
"policy_error": policy_error,
|
|
}
|
|
if history_path:
|
|
history = HistoryStore(history_path)
|
|
history.record(status)
|
|
status["history"] = history.recent()
|
|
return status
|
|
|
|
|
|
def add_llm_assessment(status: dict[str, object], *, previous: str | None = None, model: str | None = None, timeout: int | None = None) -> None:
|
|
try:
|
|
status["llm_assessment"] = {
|
|
"enabled": True,
|
|
"status": "ok",
|
|
"generated_at": int(time.time()),
|
|
"text": ollama_dashboard_assessment(status, model=model, timeout=timeout),
|
|
}
|
|
except Exception as exc:
|
|
status["llm_assessment"] = {
|
|
"enabled": True,
|
|
"status": "error",
|
|
"generated_at": int(time.time()),
|
|
"error": str(exc),
|
|
"text": previous or "",
|
|
}
|
|
|
|
|
|
def write_status(status: dict[str, object], output: str) -> None:
|
|
output_path = Path(output)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp_path = output_path.with_suffix(f"{output_path.suffix}.tmp")
|
|
tmp_path.write_text(json.dumps(status, indent=2, sort_keys=True), encoding="utf-8")
|
|
tmp_path.replace(output_path)
|
|
|
|
|
|
def monitor_loop(
|
|
log_path: str,
|
|
output: str,
|
|
*,
|
|
policy_path: str | None = None,
|
|
interval: int = 10,
|
|
anomaly_limit: int = 20,
|
|
llm: bool = False,
|
|
llm_interval: int = 300,
|
|
llm_model: str | None = None,
|
|
llm_timeout: int | None = None,
|
|
baseline_path: str | None = None,
|
|
config_path: str | None = None,
|
|
history_path: str | None = None,
|
|
) -> None:
|
|
print(f"Monitoring {log_path}")
|
|
print(f"Writing status to {output}")
|
|
last_llm_at = 0
|
|
last_llm_text: str | None = None
|
|
while True:
|
|
runtime = ConfigStore(config_path).read() if config_path and Path(config_path).exists() else {}
|
|
effective_llm = bool(runtime.get("llm_enabled")) if runtime else llm
|
|
effective_model = str(runtime.get("llm_model") or llm_model or "")
|
|
status = build_status(
|
|
log_path, policy_path=policy_path, anomaly_limit=anomaly_limit,
|
|
baseline_path=baseline_path, config_path=config_path, history_path=history_path,
|
|
)
|
|
if effective_llm:
|
|
now = int(time.time())
|
|
if now - last_llm_at >= llm_interval:
|
|
add_llm_assessment(status, previous=last_llm_text, model=effective_model or None, timeout=llm_timeout)
|
|
assessment = status.get("llm_assessment", {})
|
|
if isinstance(assessment, dict):
|
|
last_llm_text = str(assessment.get("text", "") or last_llm_text or "")
|
|
last_llm_at = now
|
|
else:
|
|
status["llm_assessment"] = {
|
|
"enabled": True,
|
|
"status": "cached",
|
|
"generated_at": last_llm_at,
|
|
"text": last_llm_text or "",
|
|
}
|
|
else:
|
|
status["llm_assessment"] = {"enabled": False, "status": "disabled", "text": ""}
|
|
write_status(status, output)
|
|
time.sleep(interval)
|