diff --git a/README.md b/README.md index f265134..c0aa6bb 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,15 @@ datatype/capability metadata and lets you select one or more entity fields, a time field, and categorical/numeric fields for the stream profile. Profiles are stored under `graylog_stream_profiles` in `state/fgai-config.json`. +The Settings page also shows recommended stream profiles built from observed +field coverage and cardinality. These recommendations use deterministic +discovery first, then can optionally be refined by a local Ollama profile advisor +model such as `qwen3:8b` or `qwen3:14b`. Enable `Ollama profile advisor` and set +`Profile advisor model` in Settings. Advisor output must be valid JSON and is +validated against fields actually seen in the stream before it can be applied. +Unknown fields, raw message fields, internal `fgai_*` fields, and unknown +detectors are rejected. + The settings page treats stream enablement and profile editing separately. The checkboxes decide which streams are monitored. Click `Edit profile` on one stream to load its fields and edit only that stream's profile; saving with no active diff --git a/ROADMAP.md b/ROADMAP.md index 80d9d21..af5e156 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,6 +36,7 @@ Goal: make findings more accurate before adding more integrations. - [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold. - [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status. - [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings. +- [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping. Acceptance: each finding shows its detector, confidence, baseline sample count, current value, expected value, and a bounded set of raw-event references. diff --git a/src/fgai/config.py b/src/fgai/config.py index 0a25ef5..3e607df 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -20,6 +20,9 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_field_mapping": "", "llm_enabled": False, "llm_model": "", + "profile_advisor_enabled": False, + "profile_advisor_model": "qwen3:8b", + "profile_advisor_timeout": 120, "threat_intel_enabled": False, "threat_intel_provider": "auto", "abuseipdb_api_key": "", @@ -58,11 +61,11 @@ class ConfigStore: continue if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "": continue - if key in {"llm_enabled", "threat_intel_enabled"}: + if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled"}: current[key] = bool(value) elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: current[key] = value - elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: + elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: minimum = 60 if key == "graylog_range_seconds" else 1 current[key] = max(minimum, int(value)) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 913f6f3..ab0472e 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -89,7 +89,7 @@ HTML = """