diff --git a/README.md b/README.md index f556890..8b0144c 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ Autoblocking is dry-run by default. The tool will not block RFC1918, loopback, m ### Findings ![Findings](images/findings.jpg) +### Correlation +![Correlation](images/correlation.jpg) + ## Quick Start ```bash @@ -85,6 +88,23 @@ baselines for authentication failures, DNS queries, and deny/block actions when those events are present. These are evaluated per configured entity, so a Windows account, DNS client, or firewall source is compared to its own history. +Stream profiles can also carry `field_weights` to tune scoring without changing +the baseline itself. Weights are multipliers from `0` to `5` and can target a +field, a detector, or a field+detector pair: + +```json +{ + "url": 1.5, + "auth_failure_burst": 2, + "query_domain": { + "rare_value": 1.8 + } +} +``` + +Use replay or `replay-graylog --compare-config-file` to test score changes before +applying them to the live profile. + Replay a historic JSONL or Graylog export without changing the live baseline: ```bash diff --git a/ROADMAP.md b/ROADMAP.md index f4cdb31..c8842df 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -26,7 +26,7 @@ Goal: make findings more accurate before adding more integrations. - [x] Add generic event-rate burst detection per stream/entity. - [x] Add rare-value detection with a minimum historical observation threshold. - [x] Add detector-specific authentication failure, DNS volume, and denied-traffic burst thresholds. -- [ ] Add configurable per-field detector weights. +- [x] Add configurable per-field detector weights. - [ ] Add sequence detection, for example DNS lookup -> outbound connection -> authentication event. - [x] Add per-stream detector enablement and thresholds in the UI. - [x] Add a dry-run replay command for historic JSONL or Graylog exports using temporary baselines. diff --git a/images/correlation.jpg b/images/correlation.jpg new file mode 100644 index 0000000..0d0e586 Binary files /dev/null and b/images/correlation.jpg differ diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index 3db6a34..3aaf4c6 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -42,6 +42,26 @@ def _baseline_confidence(samples: int, *, temporal: bool) -> str: return "low" +def _weighted_score(base: int, profile: object | None, field: str, detector: str) -> tuple[int, float]: + weights = getattr(profile, "field_weights", {}) if profile else {} + if not isinstance(weights, dict): + return base, 1.0 + field_key = str(field).lower() + detector_key = str(detector).lower() + multiplier = 1.0 + for key in (field_key, detector_key): + value = weights.get(key) + if isinstance(value, (int, float)): + multiplier *= float(value) + nested = weights.get(field_key) + if isinstance(nested, dict): + value = nested.get(detector_key) + if isinstance(value, (int, float)): + multiplier *= float(value) + multiplier = max(0.0, min(5.0, multiplier)) + return min(100, max(0, int(round(base * multiplier)))), round(multiplier, 2) + + class BaselineStore: """Persistent five-minute behavior baseline, implemented with stdlib SQLite.""" @@ -211,6 +231,7 @@ class BaselineStore: output: dict[str, list[dict[str, object]]] = defaultdict(list) with self._connect() as connection: for (stream, entity, field), values in current.items(): + profile = profiles.get(stream) matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)] current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time()) moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc) @@ -233,8 +254,9 @@ class BaselineStore: sample_values = sorted({event.fields.get(field, "") for event in matching})[:5] evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]] confidence = _baseline_confidence(len(rows), temporal=temporal) - score = 18 if confidence == "high" else 15 if confidence == "medium" else 10 - output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events}) + base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10 + score, weight = _weighted_score(base_score, profile, field, "numeric_baseline") + output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events}) # Event-rate burst is calculated once per stream/entity, rather than once per selected field. for (stream, entity), matching in entity_events.items(): @@ -260,9 +282,10 @@ class BaselineStore: if z_score < 3: continue confidence = _baseline_confidence(len(rows), temporal=temporal) - score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score)) + base_score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score)) + score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst") samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": "", "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]] - output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples}) + output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples}) for (stream, entity, detector), current_value in detector_current.items(): profile = profiles.get(stream) @@ -290,9 +313,10 @@ class BaselineStore: if z_score < z_threshold: continue confidence = _baseline_confidence(len(rows), temporal=temporal) - score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score)) + base_score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score)) + score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst") samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or event.fields.get("query_domain", ""), "action": event.action, "severity": event.severity, "service": event.fields.get("service", event.fields.get("query_type", "")), "value": detector, "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]] - output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples}) + output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples}) # Detect selected categorical values that have not appeared for this entity in prior data. for event in events: profile = profiles.get(event.fields.get("fgai_stream_id", "")) @@ -311,7 +335,8 @@ class BaselineStore: known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0] if known is None and known_total >= 30: samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value] - evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": 12, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]} + score, weight = _weighted_score(12, profile, field, "rare_value") + evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]} if evidence not in output[entity]: output[entity].append(evidence) return output diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 52ec548..4b5b2ea 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -80,7 +80,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Correlation Map

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Runtime Configuration

+

Runtime Configuration