diff --git a/README.md b/README.md index f265134..c0aa6bb 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,15 @@ datatype/capability metadata and lets you select one or more entity fields, a time field, and categorical/numeric fields for the stream profile. Profiles are stored under `graylog_stream_profiles` in `state/fgai-config.json`. +The Settings page also shows recommended stream profiles built from observed +field coverage and cardinality. These recommendations use deterministic +discovery first, then can optionally be refined by a local Ollama profile advisor +model such as `qwen3:8b` or `qwen3:14b`. Enable `Ollama profile advisor` and set +`Profile advisor model` in Settings. Advisor output must be valid JSON and is +validated against fields actually seen in the stream before it can be applied. +Unknown fields, raw message fields, internal `fgai_*` fields, and unknown +detectors are rejected. + The settings page treats stream enablement and profile editing separately. The checkboxes decide which streams are monitored. Click `Edit profile` on one stream to load its fields and edit only that stream's profile; saving with no active diff --git a/ROADMAP.md b/ROADMAP.md index 80d9d21..af5e156 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,6 +36,7 @@ Goal: make findings more accurate before adding more integrations. - [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold. - [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status. - [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings. +- [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping. Acceptance: each finding shows its detector, confidence, baseline sample count, current value, expected value, and a bounded set of raw-event references. diff --git a/src/fgai/config.py b/src/fgai/config.py index 0a25ef5..3e607df 100644 --- a/src/fgai/config.py +++ b/src/fgai/config.py @@ -20,6 +20,9 @@ DEFAULT_CONFIG: dict[str, object] = { "graylog_field_mapping": "", "llm_enabled": False, "llm_model": "", + "profile_advisor_enabled": False, + "profile_advisor_model": "qwen3:8b", + "profile_advisor_timeout": 120, "threat_intel_enabled": False, "threat_intel_provider": "auto", "abuseipdb_api_key": "", @@ -58,11 +61,11 @@ class ConfigStore: continue if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "": continue - if key in {"llm_enabled", "threat_intel_enabled"}: + if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled"}: current[key] = bool(value) elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: current[key] = value - elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: + elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: try: minimum = 60 if key == "graylog_range_seconds" else 1 current[key] = max(minimum, int(value)) diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 913f6f3..ab0472e 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -89,7 +89,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Correlation Map

AI Assessment

LLM assessment disabled.

Investigation Incidents

Anomalies

Recommendations

Triage Queue

Field Baseline Deviations

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

Diagnostics

-

Recommended Stream Profiles

Waiting for observed stream data.

Runtime Configuration

+

Recommended Stream Profiles

Waiting for observed stream data.

Runtime Configuration