diff --git a/README.md b/README.md index 0a05b7e..f265134 100644 --- a/README.md +++ b/README.md @@ -244,12 +244,18 @@ Tune these in the dashboard or in `state/fgai-config.json`: ```json { + "baseline_training_days": 7, "baseline_retention_days": 14, "baseline_value_retention_days": 7, "baseline_max_values_per_field": 2000 } ``` +`baseline_training_days` is the minimum baseline age before profile deviations +are promoted into live triage. For production data, set this to the amount of +history you trust, commonly `7` to `14` days. The dashboard shows fields as +`learning` until both bucket count and baseline age are sufficient. + If an existing baseline database has already grown large, stop the monitor and run a manual prune plus SQLite compaction: @@ -302,7 +308,12 @@ export VIRUSTOTAL_API_KEY='...' fgai recommend --logs logs/fg_syslog.jsonl --min-score 35 --threat-intel ``` -Threat intelligence responses are cached locally in `state/threat-intel-cache.json`. Successful results are reused for seven days by default, failures for one hour, and SignalScope permits at most 100 new provider lookups per UTC day. Cached responses are returned even after that budget is reached. Tune these safeguards with `FGAI_THREAT_INTEL_TTL_SECONDS`, `FGAI_THREAT_INTEL_ERROR_TTL_SECONDS`, and `FGAI_THREAT_INTEL_DAILY_LIMIT`. +Threat intelligence can also be configured in the dashboard settings. Choose +`auto`, `abuseipdb`, or `virustotal`, paste the provider API key, and set the +daily lookup budget and cache TTLs. API keys are stored only in the local config +file and are not returned back to the browser after saving. + +Threat intelligence responses are cached locally in `state/threat-intel-cache.json`. Successful results are reused for seven days by default, failures for one hour, and SignalScope permits at most 100 new provider lookups per UTC day. Cached responses are returned even after that budget is reached. Tune these safeguards in the dashboard or with `FGAI_THREAT_INTEL_TTL_SECONDS`, `FGAI_THREAT_INTEL_ERROR_TTL_SECONDS`, and `FGAI_THREAT_INTEL_DAILY_LIMIT`. Listen for FortiGate syslog locally: diff --git a/src/fgai/baseline.py b/src/fgai/baseline.py index c212e93..090ea59 100644 --- a/src/fgai/baseline.py +++ b/src/fgai/baseline.py @@ -18,6 +18,7 @@ from .query_details import event_query_details DEFAULT_RETENTION_DAYS = 14 DEFAULT_VALUE_RETENTION_DAYS = 7 DEFAULT_MAX_VALUES_PER_FIELD = 2000 +DEFAULT_TRAINING_DAYS = 7 MIN_REPORTED_DEVIATION_SCORE = 15 MAX_RARE_VALUES_PER_ENTITY = 5 @@ -50,6 +51,12 @@ def _baseline_confidence(samples: int, *, temporal: bool) -> str: return "low" +def _age_days(oldest_bucket: int | None, newest_timestamp: int) -> float: + if not oldest_bucket: + return 0.0 + return max(0.0, (newest_timestamp - int(oldest_bucket)) / 86400) + + def _weighted_score(base: int, profile: object | None, field: str, detector: str) -> tuple[int, float]: weights = getattr(profile, "field_weights", {}) if profile else {} if not isinstance(weights, dict): @@ -254,7 +261,7 @@ class BaselineStore: on conflict(stream_id, entity, detector, weekday, hour, bucket_start) do update set events=events+excluded.events""", (stream_id, entity, detector, weekday, hour, bucket, count)) return len(pending) - def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]: + def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object], *, min_training_days: int = 0) -> dict[str, list[dict[str, object]]]: current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0]) entity_events: dict[tuple[str, str], list[LogEvent]] = defaultdict(list) detector_current: Counter[tuple[str, str, str]] = Counter() @@ -291,6 +298,10 @@ class BaselineStore: baseline_scope = "all observed periods" if len(rows) < 12: continue + oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0] + baseline_age_days = _age_days(oldest, current_timestamp) + if baseline_age_days < min_training_days: + continue if values[1] == 0: continue history = [row[1] / row[0] if row[0] else 0 for row in rows] @@ -304,7 +315,7 @@ class BaselineStore: base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10 score, weight = _weighted_score(base_score, profile, field, "numeric_baseline") if score >= MIN_REPORTED_DEVIATION_SCORE: - output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events}) + output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events}) # Event-rate burst is calculated once per stream/entity, rather than once per selected field. for (stream, entity), matching in entity_events.items(): @@ -324,6 +335,10 @@ class BaselineStore: baseline_scope = "all observed periods" if len(rows) < 12: continue + oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, reference_field)).fetchone()[0] + baseline_age_days = _age_days(oldest, current_timestamp) + if baseline_age_days < min_training_days: + continue history = [row[0] for row in rows] current_value = len(matching) z_score = (current_value - mean(history)) / (pstdev(history) or 1.0) @@ -334,7 +349,7 @@ class BaselineStore: score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst") samples = [_sample_event(event) for event in matching[:5]] if score >= MIN_REPORTED_DEVIATION_SCORE: - output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples}) + output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples}) for (stream, entity, detector), current_value in detector_current.items(): profile = profiles.get(stream) @@ -357,6 +372,10 @@ class BaselineStore: baseline_scope = "all observed periods" if len(rows) < 12: continue + oldest = connection.execute("select min(bucket_start) from profile_detector_buckets where stream_id=? and entity=? and detector=?", (stream, entity, detector)).fetchone()[0] + baseline_age_days = _age_days(oldest, current_timestamp) + if baseline_age_days < min_training_days: + continue history = [row[0] for row in rows] z_score = (current_value - mean(history)) / (pstdev(history) or 1.0) if z_score < z_threshold: @@ -366,7 +385,7 @@ class BaselineStore: score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst") samples = [_sample_event(event, detector) for event in matching[:5]] if score >= MIN_REPORTED_DEVIATION_SCORE: - output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples}) + output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples}) # Detect selected categorical values that have not appeared for this entity in prior data. rare_counts: Counter[tuple[str, str]] = Counter() for event in events: @@ -386,6 +405,10 @@ class BaselineStore: known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone() known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0] if known is None and known_total >= 30: + oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0] + baseline_age_days = _age_days(oldest, _event_epoch(event, int(time.time()))) + if baseline_age_days < min_training_days: + continue if rare_counts[(stream, entity)] >= MAX_RARE_VALUES_PER_ENTITY: continue samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and entity in profile_entities(item, profile) and item.fields.get(field) == value] @@ -393,7 +416,7 @@ class BaselineStore: score, weight = _weighted_score(base_score, profile, field, "rare_value") if score < MIN_REPORTED_DEVIATION_SCORE: continue - evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]} + evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]} if evidence not in output[entity]: output[entity].append(evidence) rare_counts[(stream, entity)] += 1 @@ -449,14 +472,16 @@ class BaselineStore: connection.execute("vacuum") return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0} - def profile_readiness(self, profiles: dict[str, object]) -> list[dict[str, object]]: + def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]: rows: list[dict[str, object]] = [] with self._connect() as connection: for stream_id, profile in profiles.items(): fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())] for field in fields: count = connection.execute("select count(distinct bucket_start) from profile_buckets where stream_id=? and field=?", (stream_id, str(field).lower())).fetchone()[0] - rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "ready": count >= 12}) + oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and field=?", (stream_id, str(field).lower())).fetchone()[0] + age_days = _age_days(oldest, int(time.time())) + rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days}) return rows def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]: diff --git a/src/fgai/config.py b/src/fgai/config.py index cfb36b8..0a25ef5 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -16,10 +16,18 @@ DEFAULT_CONFIG: dict[str, object] = { "baseline_retention_days": 14, "baseline_value_retention_days": 7, "baseline_max_values_per_field": 2000, + "baseline_training_days": 7, "graylog_field_mapping": "", "llm_enabled": False, "llm_model": "", "threat_intel_enabled": False, + "threat_intel_provider": "auto", + "abuseipdb_api_key": "", + "virustotal_api_key": "", + "threat_intel_daily_limit": 100, + "threat_intel_ttl_seconds": 604800, + "threat_intel_error_ttl_seconds": 3600, + "abuseipdb_max_age_days": 90, } EDITABLE_FIELDS = set(DEFAULT_CONFIG) | {"graylog_mcp_token"} @@ -39,6 +47,8 @@ class ConfigStore: def public(self) -> dict[str, object]: config = self.read() config["graylog_mcp_token_configured"] = bool(config.pop("graylog_mcp_token", "")) + config["abuseipdb_api_key_configured"] = bool(config.pop("abuseipdb_api_key", "")) + config["virustotal_api_key_configured"] = bool(config.pop("virustotal_api_key", "")) return config def update(self, values: dict[str, object]) -> dict[str, object]: @@ -46,18 +56,20 @@ class ConfigStore: for key, value in values.items(): if key not in EDITABLE_FIELDS: continue - if key == "graylog_mcp_token" and value == "": + if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "": continue if key in {"llm_enabled", "threat_intel_enabled"}: current[key] = bool(value) elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: current[key] = value - elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field"}: + elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: minimum = 60 if key == "graylog_range_seconds" else 1 current[key] = max(minimum, int(value)) except (TypeError, ValueError): continue + elif key == "threat_intel_provider" and value in {"auto", "abuseipdb", "virustotal"}: + current[key] = value elif key == "graylog_streams" and isinstance(value, list): current[key] = [ {"id": str(item.get("id", "")), "title": str(item.get("title", "")), "enabled": bool(item.get("enabled"))} diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index b1ecc49..085b63a 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -60,6 +60,10 @@ HTML = """ .field-controls label { white-space: nowrap; } .stream-row { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 4px 0; } .stream-row button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 4px 8px; cursor: pointer; } + .toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 8px 0 12px; color: #91abc4; } + .toolbar label { display: inline-flex; align-items: center; gap: 6px; } + .summary-card { border: 1px solid #163b59; background: #061a2e; padding: 10px; margin-bottom: 8px; border-radius: 6px; } + .summary-card h3 { margin: 0 0 6px; font-size: 16px; } .review-actions { display: flex; flex-wrap: wrap; gap: 6px; min-width: 250px; } .review-actions button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 6px 8px; cursor: pointer; } .review-actions button[data-status="false_positive"] { border-color: #b7823a; color: #ffd36e; } @@ -83,9 +87,9 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Correlation Map

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

-

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

+

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Runtime Configuration

+

Runtime Configuration