Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added the optional Ollama profile advisor

Browse Source
This commit is contained in:
larssand
2026-06-30 09:09:04 +02:00
parent f2385a00a9
commit d7b8c494e9
9 changed files with 178 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/fgai/config.py
View File

@@ -20,6 +20,9 @@ DEFAULT_CONFIG: dict[str, object] = {
"graylog_field_mapping": "",
"llm_enabled": False,
"llm_model": "",
"profile_advisor_enabled": False,
"profile_advisor_model": "qwen3:8b",
"profile_advisor_timeout": 120,
"threat_intel_enabled": False,
"threat_intel_provider": "auto",
"abuseipdb_api_key": "",
@@ -58,11 +61,11 @@ class ConfigStore:
continue
if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "":
continue
if key in {"llm_enabled", "threat_intel_enabled"}:
if key in {"llm_enabled", "profile_advisor_enabled", "threat_intel_enabled"}:
current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value
elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
try:
minimum = 60 if key == "graylog_range_seconds" else 1
current[key] = max(minimum, int(value))

7
src/fgai/dashboard.py
View File

@@ -89,7 +89,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main>
<script>
function esc(value) {
@@ -176,10 +176,12 @@ async function refresh() {
metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Correlated entities', correlations.length)
].join('');
window.profileSuggestions = data.profile_suggestions || [];
const advisor = ((data.capabilities || {}).profile_advisor || {});
document.getElementById('profileSuggestions').innerHTML = table(window.profileSuggestions, [
{label:'Stream', key:'stream_name'},
{label:'Events', key:'events'},
{label:'Confidence', key:'confidence'},
{label:'Advisor', render:r => esc((r.profile_advisor || {}).status || (advisor.enabled ? advisor.status : 'heuristic'))},
{label:'Profile', render:r => r.profile_exists ? 'exists' : 'new'},
{label:'Entity fields', render:r => esc((r.entity_fields || []).join(', ') || '-')},
{label:'Time', key:'timestamp_field'},
@@ -361,6 +363,9 @@ async function applySuggestedProfile(streamId) {
baseline_max_values_per_field: config.baseline_max_values_per_field || 2000,
llm_enabled: Boolean(config.llm_enabled),
llm_model: config.llm_model || '',
profile_advisor_enabled: Boolean(config.profile_advisor_enabled),
profile_advisor_model: config.profile_advisor_model || 'qwen3:8b',
profile_advisor_timeout: config.profile_advisor_timeout || 120,
threat_intel_enabled: Boolean(config.threat_intel_enabled),
threat_intel_provider: config.threat_intel_provider || 'auto',
threat_intel_daily_limit: config.threat_intel_daily_limit || 100,

40
src/fgai/llm.py
View File

@@ -82,3 +82,43 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
},
timeout=timeout,
)
def ollama_profile_advice(suggestions: list[dict[str, object]], model: str | None = None, timeout: int | None = None) -> list[dict[str, object]]:
host = os.getenv("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
selected_model = model or os.getenv("FGAI_PROFILE_ADVISOR_MODEL", "qwen3:8b")
selected_timeout = timeout or int(os.getenv("FGAI_PROFILE_ADVISOR_TIMEOUT", "120"))
compact = [
{
"stream_id": item.get("stream_id"),
"stream_name": item.get("stream_name"),
"events": item.get("events"),
"common_fields": item.get("common_fields", [])[:20],
"heuristic_profile": item.get("profile", {}),
}
for item in suggestions[:10]
]
body = json.dumps(
{
"model": selected_model,
"stream": False,
"format": "json",
"options": {"num_predict": 1200, "temperature": 0.1},
"prompt": (
"You are SignalScope's local profile advisor. Infer stream profile mappings from observed field statistics. "
"Return only valid JSON with this schema: "
"{\"profiles\":[{\"stream_id\":\"...\",\"entity_fields\":[\"...\"],\"timestamp_field\":\"...\","
"\"categorical_fields\":[\"...\"],\"numeric_fields\":[\"...\"],\"detectors\":{\"auth_failure\":{\"enabled\":true,\"minimum\":5,\"z_threshold\":3}},"
"\"reason\":\"short reason\"}]}. "
"Use only field names present in common_fields or heuristic_profile. Do not include raw message/full_message fields. "
"Allowed detectors are auth_failure, dns_query, deny_action. Prefer canonical fields such as username, hostname, eventid, srcip, dstip when present. "
f"\n\nObserved streams:\n{json.dumps(compact, sort_keys=True)}"
),
}
).encode("utf-8")
req = request.Request(f"{host}/api/generate", data=body, method="POST", headers={"Content-Type": "application/json"})
with request.urlopen(req, timeout=selected_timeout) as response:
data = json.loads(response.read().decode("utf-8"))
payload = json.loads(str(data.get("response", "{}")))
profiles = payload.get("profiles", [])
return profiles if isinstance(profiles, list) else []

18
src/fgai/monitor.py
View File

@@ -15,11 +15,11 @@ from .graylog_source import GraylogStreamSource
from .history import HistoryStore
from .incidents import IncidentStore, build_incidents
from .data_quality import assess_data_quality
from .llm import ollama_dashboard_assessment
from .llm import ollama_dashboard_assessment, ollama_profile_advice
from .logs import local_in_failures, read_events, summarize_events, top_field_values
from .mitigation import parse_allowlist, suggest_block_candidates
from .policies import audit_policies, read_policies
from .profile_suggestions import suggest_stream_profiles
from .profile_suggestions import apply_profile_advice, suggest_stream_profiles
from .recommendations import build_recommendations
from .sequences import detect_sequences
from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
@@ -203,6 +203,18 @@ def build_status(
]
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles)
profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"}
if runtime_values.get("profile_advisor_enabled") and profile_suggestions:
try:
advice = ollama_profile_advice(
profile_suggestions,
model=str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b"),
timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120),
)
profile_suggestions = apply_profile_advice(profile_suggestions, advice)
profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
except Exception as exc:
profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
intel_ips = sorted(
{
ip
@@ -249,7 +261,7 @@ def build_status(
"summary": summarize_events(events),
"anomaly_summary": anomaly_summary(anomalies),
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
"configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
"stream_coverage": stream_coverage,

66
src/fgai/profile_suggestions.py
View File

@@ -57,6 +57,7 @@ CATEGORICAL_PRIORITY = (
"context",
)
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "eventid", "event_id", "winlog_event_id", "event_code")
ALLOWED_DETECTORS = {"auth_failure", "dns_query", "deny_action"}
IGNORED_DISCOVERY_FIELDS = {
"message",
"msg",
@@ -151,6 +152,71 @@ def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str]
return output
def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
profile = suggestion.get("profile", {}) if isinstance(suggestion.get("profile"), dict) else {}
fields = {str(item.get("field", "")) for item in suggestion.get("common_fields", []) if isinstance(item, dict)}
for key in ("entity_fields", "categorical_fields", "numeric_fields"):
value = profile.get(key, [])
if isinstance(value, list):
fields.update(str(item) for item in value if item)
if profile.get("timestamp_field"):
fields.add(str(profile["timestamp_field"]))
return {field for field in fields if field and field not in IGNORED_DISCOVERY_FIELDS}
def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict[str, object]]) -> list[dict[str, object]]:
advice_by_stream = {str(item.get("stream_id", "")): item for item in advice if isinstance(item, dict)}
output = []
for suggestion in suggestions:
stream_id = str(suggestion.get("stream_id", ""))
item = dict(suggestion)
advisor = advice_by_stream.get(stream_id)
if not advisor:
item["profile_advisor"] = {"status": "not_run"}
output.append(item)
continue
allowed_fields = _allowed_fields(item)
def valid_fields(key: str, fallback: list[str]) -> list[str]:
value = advisor.get(key, [])
fields = [str(field) for field in value if str(field) in allowed_fields] if isinstance(value, list) else []
return list(dict.fromkeys(fields or fallback))
heuristic = item.get("profile", {}) if isinstance(item.get("profile"), dict) else {}
entity_fields = valid_fields("entity_fields", list(heuristic.get("entity_fields", [])) if isinstance(heuristic.get("entity_fields"), list) else [])
timestamp = str(advisor.get("timestamp_field", ""))
if timestamp not in allowed_fields:
timestamp = str(heuristic.get("timestamp_field", "timestamp"))
categorical = valid_fields("categorical_fields", list(heuristic.get("categorical_fields", [])) if isinstance(heuristic.get("categorical_fields"), list) else [])
numeric = valid_fields("numeric_fields", list(heuristic.get("numeric_fields", [])) if isinstance(heuristic.get("numeric_fields"), list) else [])
detectors_raw = advisor.get("detectors", {})
detectors = {}
if isinstance(detectors_raw, dict):
for name, settings in detectors_raw.items():
if str(name) not in ALLOWED_DETECTORS or not isinstance(settings, dict):
continue
try:
detectors[str(name)] = {"enabled": bool(settings.get("enabled", True)), "minimum": max(1, int(settings.get("minimum", 1))), "z_threshold": max(1.0, float(settings.get("z_threshold", 3.0)))}
except (TypeError, ValueError):
continue
if not detectors:
detectors = dict(heuristic.get("detectors", {})) if isinstance(heuristic.get("detectors"), dict) else {}
if entity_fields:
profile = {
**heuristic,
"entity_field": entity_fields[0],
"entity_fields": entity_fields,
"timestamp_field": timestamp,
"categorical_fields": categorical,
"numeric_fields": numeric,
"detectors": detectors,
}
item.update({"profile": profile, "entity_fields": entity_fields, "timestamp_field": timestamp, "categorical_fields": categorical, "numeric_fields": numeric, "detectors": detectors})
item["profile_advisor"] = {"status": "ok", "model": str(advisor.get("model", "")), "reason": str(advisor.get("reason", ""))}
else:
item["profile_advisor"] = {"status": "invalid", "reason": "advisor returned no valid entity fields"}
output.append(item)
return output
def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]:
existing_profiles = existing_profiles or {}
grouped: dict[str, list[LogEvent]] = defaultdict(list)