add rule generalization, sized LLM defaults, ops docs, and UI polish
Feedback rules can now be suggested for merging when a human has already reviewed the same pattern multiple times (never guesses about new values). LLM defaults sized for constrained VRAM instead of llama3.3/llama3.1. Adds systemd units, a state backup script, and reverse-proxy auth guidance for production deployment. Findings view is collapsible and sorts suggested-review rows to the top for a quicker overview.
This commit is contained in:
@@ -28,11 +28,12 @@ DEFAULT_CONFIG: dict[str, object] = {
|
||||
"llm_model": "",
|
||||
"llm_timeout": 180,
|
||||
"profile_advisor_enabled": False,
|
||||
"profile_advisor_model": "qwen3:8b",
|
||||
"profile_advisor_model": "qwen2.5:3b-instruct",
|
||||
"profile_advisor_timeout": 240,
|
||||
"profile_advisor_interval_seconds": 300,
|
||||
"field_confidence_threshold": 60,
|
||||
"field_confidence_retention_days": 90,
|
||||
"feedback_generalization_min_values": 3,
|
||||
"auto_apply_profile_suggestions": False,
|
||||
"threat_intel_enabled": False,
|
||||
"threat_intel_provider": "auto",
|
||||
@@ -89,6 +90,11 @@ class ConfigStore:
|
||||
current[key] = min(100, max(0, int(value)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
elif key == "feedback_generalization_min_values":
|
||||
try:
|
||||
current[key] = min(20, max(2, int(value)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
elif key == "threat_intel_provider" and value in {"auto", "abuseipdb", "virustotal"}:
|
||||
current[key] = value
|
||||
elif key == "graylog_streams" and isinstance(value, list):
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -175,3 +175,36 @@ def feedback_matches(entry: dict[str, object], *, entity: str, stream_id: str, f
|
||||
|
||||
def select_match(candidates: list[dict[str, object]]) -> dict[str, object]:
|
||||
return min(candidates, key=lambda item: SCOPE_PRIORITY.get(str(item.get("scope") or _resolve_scope(item)), 1))
|
||||
|
||||
|
||||
def suggest_generalizations(
|
||||
entries: list[dict[str, object]], *, min_values: int = 3, sample_limit: int = 5
|
||||
) -> list[dict[str, object]]:
|
||||
"""Find groups of narrow value-scope rules a human could merge into one field-scope rule.
|
||||
|
||||
Only ever looks at entries a human already created (value scope) — never
|
||||
reasons about unseen values, so it can't pre-judge a new anomaly.
|
||||
"""
|
||||
groups: dict[tuple[str, str, str, str], list[dict[str, object]]] = {}
|
||||
for entry in entries:
|
||||
scope = str(entry.get("scope") or _resolve_scope(entry))
|
||||
if scope != "value":
|
||||
continue
|
||||
status = str(entry.get("status", ""))
|
||||
if status not in VALID_STATUSES:
|
||||
continue
|
||||
key = (str(entry.get("stream_id", "")), str(entry.get("entity", "")), str(entry.get("field", "")), status)
|
||||
groups.setdefault(key, []).append(entry)
|
||||
|
||||
suggestions = []
|
||||
for (stream_id, entity, field, status), group in groups.items():
|
||||
distinct_values = sorted({str(item.get("value", "")) for item in group})
|
||||
if len(distinct_values) < min_values:
|
||||
continue
|
||||
suggestions.append({
|
||||
"stream_id": stream_id, "entity": entity, "field": field, "status": status,
|
||||
"value_count": len(distinct_values), "sample_values": distinct_values[:sample_limit],
|
||||
"entry_ids": [str(item.get("id", "")) for item in group],
|
||||
})
|
||||
suggestions.sort(key=lambda item: -item["value_count"])
|
||||
return suggestions
|
||||
|
||||
@@ -41,7 +41,7 @@ def ollama_summary(
|
||||
timeout: int | None = None,
|
||||
) -> str:
|
||||
host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
|
||||
selected_model = model or os.getenv("OLLAMA_MODEL", "llama3.1")
|
||||
selected_model = model or os.getenv("OLLAMA_MODEL", "qwen2.5:7b-instruct")
|
||||
selected_timeout = timeout or int(os.getenv("OLLAMA_TIMEOUT", "180"))
|
||||
prompt = {
|
||||
"analysis": analysis or {},
|
||||
@@ -378,7 +378,7 @@ def ollama_profile_advice(
|
||||
ambiguous_fields_by_stream: dict[str, list[str]] | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
|
||||
selected_model = model or os.getenv("FGAI_PROFILE_ADVISOR_MODEL", "qwen3:8b")
|
||||
selected_model = model or os.getenv("FGAI_PROFILE_ADVISOR_MODEL", "qwen2.5:3b-instruct")
|
||||
selected_timeout = timeout or int(os.getenv("FGAI_PROFILE_ADVISOR_TIMEOUT", "240"))
|
||||
scoped = bool(ambiguous_fields_by_stream)
|
||||
compact = [
|
||||
|
||||
@@ -12,7 +12,7 @@ from .baseline import BaselineStore
|
||||
from .config import ConfigStore
|
||||
from .correlation import correlate_source_ips
|
||||
from .event_context import build_event_context
|
||||
from .feedback import FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match
|
||||
from .feedback import FeedbackStore, SUPPRESSING_STATUSES, feedback_matches, select_match, suggest_generalizations
|
||||
from .graylog_aggregate import GraylogAggregateSource
|
||||
from .graylog_mcp import GraylogMcpClient
|
||||
from .graylog_source import GraylogStreamSource
|
||||
@@ -524,6 +524,8 @@ def build_status(
|
||||
deviation["feedback_rule_id"] = match.get("id", "")
|
||||
if match["status"] in SUPPRESSING_STATUSES:
|
||||
deviation["score"] = 0
|
||||
generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3)
|
||||
feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values)
|
||||
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations)
|
||||
baseline_events = baseline.ingest(events) if baseline else 0
|
||||
profile_baseline_fields = (
|
||||
@@ -691,6 +693,7 @@ def build_status(
|
||||
"triage_queue": triage_queue,
|
||||
"sequence_findings": sequence_findings,
|
||||
"feedback": feedback,
|
||||
"feedback_generalization_suggestions": feedback_generalization_suggestions,
|
||||
"cross_source_correlations": correlations,
|
||||
"incidents": incidents,
|
||||
"data_quality": assess_data_quality(events, mcp_status),
|
||||
|
||||
Reference in New Issue
Block a user