Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added the optional Ollama profile advisor

Browse Source
This commit is contained in:
larssand
2026-06-30 09:09:04 +02:00
parent f2385a00a9
commit d7b8c494e9
9 changed files with 178 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -63,6 +63,15 @@ datatype/capability metadata and lets you select one or more entity fields, a
time field, and categorical/numeric fields for the stream profile. Profiles are time field, and categorical/numeric fields for the stream profile. Profiles are
stored under `graylog_stream_profiles` in `state/fgai-config.json`. stored under `graylog_stream_profiles` in `state/fgai-config.json`.
The Settings page also shows recommended stream profiles built from observed
field coverage and cardinality. These recommendations use deterministic
discovery first, then can optionally be refined by a local Ollama profile advisor
model such as `qwen3:8b` or `qwen3:14b`. Enable `Ollama profile advisor` and set
`Profile advisor model` in Settings. Advisor output must be valid JSON and is
validated against fields actually seen in the stream before it can be applied.
Unknown fields, raw message fields, internal `fgai_*` fields, and unknown
detectors are rejected.
The settings page treats stream enablement and profile editing separately. The The settings page treats stream enablement and profile editing separately. The
checkboxes decide which streams are monitored. Click `Edit profile` on one stream checkboxes decide which streams are monitored. Click `Edit profile` on one stream
to load its fields and edit only that stream's profile; saving with no active to load its fields and edit only that stream's profile; saving with no active

1
ROADMAP.md
View File

@@ -36,6 +36,7 @@ Goal: make findings more accurate before adding more integrations.
- [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold. - [ ] Add baseline confidence tooling: per detector learning state, expected false-positive rate, and why a deviation crossed threshold.
- [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status. - [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status.
- [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings. - [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings.
- [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping.
Acceptance: each finding shows its detector, confidence, baseline sample count, current value, expected value, and a bounded set of raw-event references. Acceptance: each finding shows its detector, confidence, baseline sample count, current value, expected value, and a bounded set of raw-event references.

7
src/fgai/config.py
View File

@@ -20,6 +20,9 @@ DEFAULT_CONFIG: dict[str, object] = {
"graylog_field_mapping": "", "graylog_field_mapping": "",
"llm_enabled": False, "llm_enabled": False,
"llm_model": "", "llm_model": "",
"profile_advisor_enabled": False,
"profile_advisor_model": "qwen3:8b",
"profile_advisor_timeout": 120,
"threat_intel_enabled": False, "threat_intel_enabled": False,
"threat_intel_provider": "auto", "threat_intel_provider": "auto",
"abuseipdb_api_key": "", "abuseipdb_api_key": "",
@@ -58,11 +61,11 @@ class ConfigStore:
continue continue
if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "": if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "":
continue continue
if key in {"llm_enabled", "threat_intel_enabled"}: if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled"}:
current[key] = bool(value) current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value current[key] = value
elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
try: try:
minimum = 60 if key == "graylog_range_seconds" else 1 minimum = 60 if key == "graylog_range_seconds" else 1
current[key] = max(minimum, int(value)) current[key] = max(minimum, int(value))

7
src/fgai/dashboard.py
View File

@@ -89,7 +89,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div> <div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div> <div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div> <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div> <div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main> </main>
<script> <script>
function esc(value) { function esc(value) {
@@ -176,10 +176,12 @@ async function refresh() {
metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Correlated entities', correlations.length) metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Correlated entities', correlations.length)
].join(''); ].join('');
window.profileSuggestions = data.profile_suggestions || []; window.profileSuggestions = data.profile_suggestions || [];
const advisor = ((data.capabilities || {}).profile_advisor || {});
document.getElementById('profileSuggestions').innerHTML = table(window.profileSuggestions, [ document.getElementById('profileSuggestions').innerHTML = table(window.profileSuggestions, [
{label:'Stream', key:'stream_name'}, {label:'Stream', key:'stream_name'},
{label:'Events', key:'events'}, {label:'Events', key:'events'},
{label:'Confidence', key:'confidence'}, {label:'Confidence', key:'confidence'},
{label:'Advisor', render:r => esc((r.profile_advisor || {}).status || (advisor.enabled ? advisor.status : 'heuristic'))},
{label:'Profile', render:r => r.profile_exists ? 'exists' : 'new'}, {label:'Profile', render:r => r.profile_exists ? 'exists' : 'new'},
{label:'Entity fields', render:r => esc((r.entity_fields || []).join(', ') || '-')}, {label:'Entity fields', render:r => esc((r.entity_fields || []).join(', ') || '-')},
{label:'Time', key:'timestamp_field'}, {label:'Time', key:'timestamp_field'},
@@ -361,6 +363,9 @@ async function applySuggestedProfile(streamId) {
baseline_max_values_per_field: config.baseline_max_values_per_field || 2000, baseline_max_values_per_field: config.baseline_max_values_per_field || 2000,
llm_enabled: Boolean(config.llm_enabled), llm_enabled: Boolean(config.llm_enabled),
llm_model: config.llm_model || '', llm_model: config.llm_model || '',
profile_advisor_enabled: Boolean(config.profile_advisor_enabled),
profile_advisor_model: config.profile_advisor_model || 'qwen3:8b',
profile_advisor_timeout: config.profile_advisor_timeout || 120,
threat_intel_enabled: Boolean(config.threat_intel_enabled), threat_intel_enabled: Boolean(config.threat_intel_enabled),
threat_intel_provider: config.threat_intel_provider || 'auto', threat_intel_provider: config.threat_intel_provider || 'auto',
threat_intel_daily_limit: config.threat_intel_daily_limit || 100, threat_intel_daily_limit: config.threat_intel_daily_limit || 100,

40
src/fgai/llm.py
View File

@@ -82,3 +82,43 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
}, },
timeout=timeout, timeout=timeout,
) )
def ollama_profile_advice(suggestions: list[dict[str, object]], model: str | None = None, timeout: int | None = None) -> list[dict[str, object]]:
host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
selected_model = model or os.getenv("FGAI_PROFILE_ADVISOR_MODEL", "qwen3:8b")
selected_timeout = timeout or int(os.getenv("FGAI_PROFILE_ADVISOR_TIMEOUT", "120"))
compact = [
{
"stream_id": item.get("stream_id"),
"stream_name": item.get("stream_name"),
"events": item.get("events"),
"common_fields": item.get("common_fields", [])[:20],
"heuristic_profile": item.get("profile", {}),
}
for item in suggestions[:10]
]
body = json.dumps(
{
"model": selected_model,
"stream": False,
"format": "json",
"options": {"num_predict": 1200, "temperature": 0.1},
"prompt": (
"You are SignalScope's local profile advisor. Infer stream profile mappings from observed field statistics. "
"Return only valid JSON with this schema: "
"{\"profiles\":[{\"stream_id\":\"...\",\"entity_fields\":[\"...\"],\"timestamp_field\":\"...\","
"\"categorical_fields\":[\"...\"],\"numeric_fields\":[\"...\"],\"detectors\":{\"auth_failure\":{\"enabled\":true,\"minimum\":5,\"z_threshold\":3}},"
"\"reason\":\"short reason\"}]}. "
"Use only field names present in common_fields or heuristic_profile. Do not include raw message/full_message fields. "
"Allowed detectors are auth_failure, dns_query, deny_action. Prefer canonical fields such as username, hostname, eventid, srcip, dstip when present. "
f"\n\nObserved streams:\n{json.dumps(compact, sort_keys=True)}"
),
}
).encode("utf-8")
req = request.Request(f"{host}/api/generate", data=body, method="POST", headers={"Content-Type": "application/json"})
with request.urlopen(req, timeout=selected_timeout) as response:
data = json.loads(response.read().decode("utf-8"))
payload = json.loads(str(data.get("response", "{}")))
profiles = payload.get("profiles", [])
return profiles if isinstance(profiles, list) else []

18
src/fgai/monitor.py
View File

@@ -15,11 +15,11 @@ from .graylog_source import GraylogStreamSource
from .history import HistoryStore from .history import HistoryStore
from .incidents import IncidentStore, build_incidents from .incidents import IncidentStore, build_incidents
from .data_quality import assess_data_quality from .data_quality import assess_data_quality
from .llm import ollama_dashboard_assessment from .llm import ollama_dashboard_assessment, ollama_profile_advice
from .logs import local_in_failures, read_events, summarize_events, top_field_values from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import parse_allowlist, suggest_block_candidates from .mitigation import parse_allowlist, suggest_block_candidates
from .policies import audit_policies, read_policies from .policies import audit_policies, read_policies
from .profile_suggestions import suggest_stream_profiles from .profile_suggestions import apply_profile_advice, suggest_stream_profiles
from .recommendations import build_recommendations from .recommendations import build_recommendations
from .sequences import detect_sequences from .sequences import detect_sequences
from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
@@ -203,6 +203,18 @@ def build_status(
] ]
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles) stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles) profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles)
profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"}
if runtime_values.get("profile_advisor_enabled") and profile_suggestions:
try:
advice = ollama_profile_advice(
profile_suggestions,
model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120),
)
profile_suggestions = apply_profile_advice(profile_suggestions, advice)
profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
except Exception as exc:
profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
intel_ips = sorted( intel_ips = sorted(
{ {
ip ip
@@ -249,7 +261,7 @@ def build_status(
"summary": summarize_events(events), "summary": summarize_events(events),
"anomaly_summary": anomaly_summary(anomalies), "anomaly_summary": anomaly_summary(anomalies),
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0}, "baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status}, "capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
"configuration": runtime_config, "configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()], "stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
"stream_coverage": stream_coverage, "stream_coverage": stream_coverage,

66
src/fgai/profile_suggestions.py
View File

@@ -57,6 +57,7 @@ CATEGORICAL_PRIORITY = (
"context", "context",
) )
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code") NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"}
IGNORED_DISCOVERY_FIELDS = { IGNORED_DISCOVERY_FIELDS = {
"message", "message",
"msg", "msg",
@@ -151,6 +152,71 @@ def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str]
return output return output
def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
profile = suggestion.get("profile", {}) if isinstance(suggestion.get("profile"), dict) else {}
fields = {str(item.get("field", "")) for item in suggestion.get("common_fields", []) if isinstance(item, dict)}
for key in ("entity_fields", "categorical_fields", "numeric_fields"):
value = profile.get(key, [])
if isinstance(value, list):
fields.update(str(item) for item in value if item)
if profile.get("timestamp_field"):
fields.add(str(profile["timestamp_field"]))
return {field for field in fields if field and field not in IGNORED_DISCOVERY_FIELDS}
def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict[str, object]]) -> list[dict[str, object]]:
advice_by_stream = {str(item.get("stream_id", "")): item for item in advice if isinstance(item, dict)}
output = []
for suggestion in suggestions:
stream_id = str(suggestion.get("stream_id", ""))
item = dict(suggestion)
advisor = advice_by_stream.get(stream_id)
if not advisor:
item["profile_advisor"] = {"status": "not_run"}
output.append(item)
continue
allowed_fields = _allowed_fields(item)
def valid_fields(key: str, fallback: list[str]) -> list[str]:
value = advisor.get(key, [])
fields = [str(field) for field in value if str(field) in allowed_fields] if isinstance(value, list) else []
return list(dict.fromkeys(fields or fallback))
heuristic = item.get("profile", {}) if isinstance(item.get("profile"), dict) else {}
entity_fields = valid_fields("entity_fields", list(heuristic.get("entity_fields", [])) if isinstance(heuristic.get("entity_fields"), list) else [])
timestamp = str(advisor.get("timestamp_field", ""))
if timestamp not in allowed_fields:
timestamp = str(heuristic.get("timestamp_field", "timestamp"))
categorical = valid_fields("categorical_fields", list(heuristic.get("categorical_fields", [])) if isinstance(heuristic.get("categorical_fields"), list) else [])
numeric = valid_fields("numeric_fields", list(heuristic.get("numeric_fields", [])) if isinstance(heuristic.get("numeric_fields"), list) else [])
detectors_raw = advisor.get("detectors", {})
detectors = {}
if isinstance(detectors_raw, dict):
for name, settings in detectors_raw.items():
if str(name) not in ALLOWED_DETECTORS or not isinstance(settings, dict):
continue
try:
detectors[str(name)] = {"enabled": bool(settings.get("enabled", True)), "minimum": max(1, int(settings.get("minimum", 1))), "z_threshold": max(1.0, float(settings.get("z_threshold", 3.0)))}
except (TypeError, ValueError):
continue
if not detectors:
detectors = dict(heuristic.get("detectors", {})) if isinstance(heuristic.get("detectors"), dict) else {}
if entity_fields:
profile = {
**heuristic,
"entity_field": entity_fields[0],
"entity_fields": entity_fields,
"timestamp_field": timestamp,
"categorical_fields": categorical,
"numeric_fields": numeric,
"detectors": detectors,
}
item.update({"profile": profile, "entity_fields": entity_fields, "timestamp_field": timestamp, "categorical_fields": categorical, "numeric_fields": numeric, "detectors": detectors})
item["profile_advisor"] = {"status": "ok", "model": str(advisor.get("model", "")), "reason": str(advisor.get("reason", ""))}
else:
item["profile_advisor"] = {"status": "invalid", "reason": "advisor returned no valid entity fields"}
output.append(item)
return output
def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]: def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]:
existing_profiles = existing_profiles or {} existing_profiles = existing_profiles or {}
grouped: dict[str, list[LogEvent]] = defaultdict(list) grouped: dict[str, list[LogEvent]] = defaultdict(list)

14
tests/test_monitor.py
View File

@@ -95,6 +95,20 @@ class MonitorTests(unittest.TestCase):
self.assertEqual(status["llm_assessment"]["status"], "ok") self.assertEqual(status["llm_assessment"]["status"], "ok")
self.assertEqual(status["llm_assessment"]["text"], "looks noisy") self.assertEqual(status["llm_assessment"]["text"], "looks noisy")
def test_profile_advisor_status_records_error(self):
with tempfile.TemporaryDirectory() as tmp:
log_path = Path(tmp) / "events.log"
log_path.write_text("fgai_stream_id=windows fgai_stream=Windows username=alice eventid=4625 action=failure\n", encoding="utf-8")
config_path = Path(tmp) / "config.json"
config_path.write_text(json.dumps({"profile_advisor_enabled": True, "profile_advisor_model": "qwen3:8b"}), encoding="utf-8")
with patch("fgai.monitor.ollama_profile_advice", side_effect=TimeoutError("timeout")):
status = build_status(str(log_path), config_path=str(config_path), incident_path=str(Path(tmp) / "incidents.json"))
advisor = status["capabilities"]["profile_advisor"]
self.assertEqual(advisor["status"], "error")
self.assertIn("timeout", advisor["error"])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

23
tests/test_profile_suggestions.py
View File

@@ -1,7 +1,7 @@
import unittest import unittest
from fgai.logs import parse_log_line from fgai.logs import parse_log_line
from fgai.profile_suggestions import suggest_stream_profiles from fgai.profile_suggestions import apply_profile_advice, suggest_stream_profiles
class ProfileSuggestionTests(unittest.TestCase): class ProfileSuggestionTests(unittest.TestCase):
@@ -58,6 +58,27 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("workflow_state", profile["categorical_fields"]) self.assertIn("workflow_state", profile["categorical_fields"])
self.assertIn("risk_points", profile["numeric_fields"]) self.assertIn("risk_points", profile["numeric_fields"])
def test_applies_valid_llm_advice_and_rejects_unknown_fields(self):
suggestion = suggest_stream_profiles([
parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure")
])[0]
advised = apply_profile_advice([suggestion], [{
"stream_id": "windows",
"entity_fields": ["username", "not_a_field"],
"timestamp_field": "eventtime",
"categorical_fields": ["eventid", "full_message"],
"numeric_fields": ["missing_number"],
"detectors": {"auth_failure": {"enabled": True, "minimum": 3, "z_threshold": 2.5}, "made_up": {"enabled": True}},
"reason": "Windows auth fields",
}])[0]
self.assertEqual(advised["profile_advisor"]["status"], "ok")
self.assertEqual(advised["profile"]["entity_fields"], ["username"])
self.assertIn("eventid", advised["profile"]["categorical_fields"])
self.assertNotIn("full_message", advised["profile"]["categorical_fields"])
self.assertEqual(set(advised["profile"]["detectors"]), {"auth_failure"})
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()