Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added the Profile Discovery report.

Browse Source
This commit is contained in:
larssand
2026-06-30 16:15:54 +02:00
parent 70191d93fb
commit 78f11af302
3 changed files with 117 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/fgai/dashboard.py
View File

@@ -298,6 +298,7 @@ async function refresh() {
document.getElementById('profileSuggestions').innerHTML = visibleProfileSuggestions.length ? table(visibleProfileSuggestions, [
{label:'Stream', key:'stream_name'},
{label:'Events', key:'events'},
{label:'Type', render:r => esc(r.detected_log_type || (r.discovery || {}).log_type || '-')},
{label:'Confidence', key:'confidence'},
{label:'Advisor', render:r => esc((r.profile_advisor || {}).status || (advisor.enabled ? advisor.status : 'heuristic'))},
{label:'Profile', render:r => r.profile_exists ? 'exists' : 'new'},
@@ -306,6 +307,7 @@ async function refresh() {
{label:'Baseline fields', render:r => esc([...(r.categorical_fields || []), ...(r.numeric_fields || [])].slice(0,8).join(', ') || '-')},
{label:'Detectors', render:r => esc(Object.keys(r.detectors || {}).join(', ') || '-')},
{label:'Common denominators', render:r => esc((r.common_fields || []).slice(0,5).map(item => `${item.field} ${(item.coverage*100).toFixed(0)}%`).join(', ') || '-')},
{label:'Discovery', render:r => { const d=r.discovery || {}; const selected=d.selected_fields || {}; const top=(d.top_fields || []).slice(0,8).map(item => `${item.field} ${(Number(item.coverage || 0)*100).toFixed(0)}%/${item.unique_values}`).join(', '); const rejected=(d.rejected_fields || []).slice(0,8).map(item => `${item.field}: ${item.reason}`).join('; '); const reasons=(d.reasons || []).join('; '); return `<details><summary>${esc(d.field_count || 0)} fields analyzed</summary><p><b>Selected</b><br>Entity: ${esc((selected.entity || []).join(', ') || '-')}<br>Time: ${esc((selected.time || []).join(', ') || '-')}<br>Categorical: ${esc((selected.categorical || []).join(', ') || '-')}<br>Numeric: ${esc((selected.numeric || []).join(', ') || '-')}</p><p><b>Top fields</b><br>${esc(top || '-')}</p><p><b>Rejected</b><br>${esc(rejected || '-')}</p><p>${esc(reasons || '')}</p></details>`; }},
{label:'Action', render:r => r.profile_exists ? '<button type="button" disabled>Applied</button>' : `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">Apply profile</button>`}
], 'profile-suggestions') : '<p class="muted">No missing stream profiles. Enable "show existing profiles" to inspect already configured profiles.</p>';
const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');

92
src/fgai/profile_suggestions.py
View File

@@ -242,6 +242,77 @@ def _compact_windows_fields(fields: list[str]) -> list[str]:
return output
def _detected_log_type(stream_id: str, stream_name: str, coverage: Counter[str]) -> str:
name = f"{stream_id} {stream_name}".lower()
if _looks_like_windows_stream(stream_id, stream_name, coverage):
return "windows"
if any(token in name for token in ("fortigate", "firewall", "pan-os", "checkpoint", "iptables", "ufw")) or (
coverage["policyid"] and (coverage["dstip"] or coverage["dstport"]) and coverage["action"]
):
return "firewall"
if any(token in name for token in ("dns", "adguard", "bind", "unbound")) or coverage["dns_query"] or coverage["query_domain"] or coverage["qh"]:
return "dns"
if any(token in name for token in ("squid", "proxy", "nginx", "apache", "iis", "web")) or coverage["url"] or coverage["request_uri"] or coverage["http_method"]:
return "web_proxy"
if any(token in name for token in ("audit", "app", "application", "serverlog", "prod")):
return "application"
return "unknown"
def _field_stats(coverage: Counter[str], unique_values: dict[str, set[str]], numeric_counts: Counter[str], total: int) -> list[dict[str, object]]:
rows = []
for field, count in coverage.most_common():
if field in IGNORED_DISCOVERY_FIELDS:
continue
numeric_ratio = numeric_counts[field] / max(1, count)
rows.append({
"field": field,
"coverage": round(count / max(1, total), 2),
"unique_values": len(unique_values[field]),
"numeric_ratio": round(numeric_ratio, 2),
})
return rows
def _rejected_fields(
coverage: Counter[str],
unique_values: dict[str, set[str]],
numeric_counts: Counter[str],
total: int,
selected_fields: set[str],
*,
limit: int = 12,
) -> list[dict[str, object]]:
rejected = []
for field, count in coverage.most_common():
if len(rejected) >= limit:
break
coverage_ratio = count / max(1, total)
cardinality = len(unique_values[field])
reason = ""
if field in selected_fields:
continue
if field in IGNORED_DISCOVERY_FIELDS:
reason = "raw/internal field"
elif coverage_ratio < 0.1:
reason = "low coverage"
elif cardinality <= 1:
reason = "constant value"
elif cardinality > min(200, max(8, int(total * 0.7))):
reason = "too high cardinality"
elif numeric_counts[field] and numeric_counts[field] / max(1, count) < 0.95:
reason = "mixed numeric/text values"
else:
reason = "lower signal than selected fields"
rejected.append({
"field": field,
"reason": reason,
"coverage": round(coverage_ratio, 2),
"unique_values": cardinality,
})
return rejected
def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[str]], total: int, selected: list[str]) -> list[str]:
output = list(selected)
for field, count in coverage.most_common():
@@ -376,6 +447,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
stream_name = names.get(stream_id, stream_id)
is_windows = _looks_like_windows_stream(stream_id, stream_name, coverage)
log_type = _detected_log_type(stream_id, stream_name, coverage)
entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
@@ -433,6 +505,24 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
{"field": field, "coverage": round(count / max(1, total), 2), "unique_values": len(unique_values[field])}
for field, count in coverage.most_common(12)
]
selected_fields = {timestamp, *entity_fields, *categorical, *numeric}
discovery = {
"log_type": log_type,
"field_count": len([field for field in coverage if field not in IGNORED_DISCOVERY_FIELDS]),
"selected_fields": {
"entity": entity_fields,
"time": [timestamp] if timestamp else [],
"categorical": categorical,
"numeric": numeric,
},
"top_fields": _field_stats(coverage, unique_values, numeric_counts, total)[:16],
"rejected_fields": _rejected_fields(coverage, unique_values, numeric_counts, total, selected_fields),
"reasons": [
f"detected {log_type} log pattern",
f"selected {len(entity_fields)} entity field(s), {len(categorical)} categorical field(s), and {len(numeric)} numeric field(s)",
"ignored raw/internal, constant, sparse, and very high-cardinality fields",
],
}
existing = existing_profiles.get(stream_id)
score = min(100, 30 + len(entity_fields) * 15 + min(20, len(categorical) * 3) + min(15, len(detectors) * 5))
suggestions.append({
@@ -448,6 +538,8 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
"numeric_fields": numeric,
"detectors": detectors,
"common_fields": high_coverage,
"discovery": discovery,
"detected_log_type": log_type,
"profile": {
"stream_id": stream_id,
"name": f"{names.get(stream_id, stream_id)} recommended profile",