82 lines
4.1 KiB
Python
82 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from .anomaly import anomaly_summary, detect_source_anomalies
|
|
from .baseline import BaselineStore
|
|
from .models import LogEvent
|
|
|
|
|
|
def replay_comparison(current: dict[str, object], candidate: dict[str, object]) -> dict[str, object]:
|
|
"""Summarize how two replay outputs differ."""
|
|
current_counts = current.get("field_detector_counts", {})
|
|
candidate_counts = candidate.get("field_detector_counts", {})
|
|
detectors = sorted(set(current_counts if isinstance(current_counts, dict) else {}) | set(candidate_counts if isinstance(candidate_counts, dict) else {}))
|
|
detector_deltas = {
|
|
detector: int((candidate_counts if isinstance(candidate_counts, dict) else {}).get(detector, 0)) - int((current_counts if isinstance(current_counts, dict) else {}).get(detector, 0))
|
|
for detector in detectors
|
|
}
|
|
return {
|
|
"events_delta": int(candidate.get("events", 0)) - int(current.get("events", 0)),
|
|
"field_findings_delta": len(candidate.get("field_findings", [])) - len(current.get("field_findings", [])),
|
|
"source_anomalies_delta": len(candidate.get("source_anomalies", [])) - len(current.get("source_anomalies", [])),
|
|
"detector_count_delta": detector_deltas,
|
|
}
|
|
|
|
|
|
def _timestamp(event: LogEvent, fallback: int) -> int:
|
|
value = event.fields.get("eventtime", event.fields.get("timestamp", ""))
|
|
try:
|
|
return int(float(value))
|
|
except (TypeError, ValueError):
|
|
try:
|
|
return int(datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp())
|
|
except ValueError:
|
|
return fallback
|
|
|
|
|
|
def _with_stream(events: list[LogEvent], stream_id: str, stream_name: str) -> list[LogEvent]:
|
|
if not stream_id:
|
|
return events
|
|
return [LogEvent(event.raw, {**event.fields, "fgai_stream_id": stream_id, "fgai_stream": stream_name or stream_id}) for event in events]
|
|
|
|
|
|
def replay_events(events: list[LogEvent], profiles: dict[str, object], *, stream_id: str = "", stream_name: str = "", bucket_seconds: int = 300) -> dict[str, object]:
|
|
"""Evaluate historical events using a temporary baseline without touching runtime state."""
|
|
normalized = _with_stream(events, stream_id, stream_name)
|
|
buckets: dict[int, list[LogEvent]] = defaultdict(list)
|
|
for index, event in enumerate(normalized):
|
|
timestamp = _timestamp(event, index * bucket_seconds)
|
|
buckets[timestamp - (timestamp % bucket_seconds)].append(event)
|
|
|
|
field_findings: list[dict[str, object]] = []
|
|
source_findings = []
|
|
with tempfile.TemporaryDirectory(prefix="signalscope-replay-") as directory:
|
|
baseline = BaselineStore(str(Path(directory) / "baseline.sqlite3"), bucket_seconds=bucket_seconds)
|
|
for bucket, batch in sorted(buckets.items()):
|
|
profiles_by_source = baseline.profiles({event.src_ip for event in batch if event.src_ip})
|
|
deviations = baseline.profile_deviations(batch, profiles)
|
|
anomalies = detect_source_anomalies(batch, baselines=profiles_by_source, field_deviations=deviations)
|
|
for entity, findings in deviations.items():
|
|
for finding in findings:
|
|
field_findings.append({"bucket_start": bucket, "entity": entity, **finding})
|
|
source_findings.extend(anomalies)
|
|
baseline.ingest(batch, observed_at=bucket)
|
|
baseline.ingest_profile_fields(batch, profiles, observed_at=bucket)
|
|
|
|
detector_counts = Counter(str(item.get("detector", "unknown")) for item in field_findings)
|
|
return {
|
|
"events": len(normalized),
|
|
"buckets": len(buckets),
|
|
"field_findings": field_findings,
|
|
"field_detector_counts": dict(sorted(detector_counts.items())),
|
|
"source_anomaly_summary": anomaly_summary(source_findings),
|
|
"source_anomalies": [
|
|
{"subject": item.subject, "score": item.score, "severity": item.severity, "confidence": item.confidence, "reasons": item.reasons, "evidence": item.evidence}
|
|
for item in source_findings
|
|
],
|
|
}
|