Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve find fileds for default profiles

Browse Source
This commit is contained in:
larssand
2026-07-02 09:51:25 +02:00
parent 53437a3952
commit 7c0b482a22
6 changed files with 167 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/fgai/dashboard.py
View File

@@ -189,11 +189,17 @@ function profileDiscoveryDetails(row) {
}
const selected = d.selected_fields || {};
const top = (d.top_fields || []).slice(0,8).map(item => `${item.field} ${(Number(item.coverage || 0)*100).toFixed(0)}%/${item.unique_values}`).join(', ');
const shared = (d.shared_fields || row.shared_fields || []).slice(0,8).map(item => `${item.field} (${item.streams} streams)`).join(', ');
const shared = (d.shared_fields || row.shared_fields || []).slice(0,8).map(sharedFieldLabel).join(', ');
const rejected = (d.rejected_fields || []).slice(0,8).map(item => `${item.field}: ${item.reason}`).join('; ');
const reasons = (d.reasons || []).join('; ');
return `<details data-detail-id="${esc(detailId)}"><summary>${esc(d.field_count)} fields analyzed</summary><p><b>Selected</b><br>Entity: ${esc((selected.entity || []).join(', ') || '-')}<br>Time: ${esc((selected.time || []).join(', ') || '-')}<br>Categorical: ${esc((selected.categorical || []).join(', ') || '-')}<br>Numeric: ${esc((selected.numeric || []).join(', ') || '-')}</p><p><b>Shared across streams</b><br>${esc(shared || '-')}</p><p><b>Top fields</b><br>${esc(top || '-')}</p><p><b>Rejected</b><br>${esc(rejected || '-')}</p><p>${esc(reasons || '')}</p></details>`;
}
function sharedFieldLabel(item) {
const aliases = (item.aliases || []).filter(alias => alias !== item.field).slice(0,4);
const suffix = aliases.length ? ` via ${aliases.join('/')}` : '';
const kind = item.kind ? ` ${item.kind}` : '';
return `${item.field}${kind} (${item.streams} streams${suffix})`;
}
function profileAdvisorLabel(row, advisor) {
const rowAdvisor = row.profile_advisor || {};
if (rowAdvisor.status === 'heuristic' && rowAdvisor.error) return 'heuristic (advisor error)';
@@ -346,8 +352,9 @@ async function refresh() {
{label:'Baseline fields', render:r => esc([...(r.categorical_fields || []), ...(r.numeric_fields || [])].slice(0,8).join(', ') || '-')},
{label:'Detectors', render:r => esc(Object.keys(r.detectors || {}).join(', ') || '-')},
{label:'Common denominators', render:r => esc((r.common_fields || []).slice(0,5).map(item => `${item.field} ${(item.coverage*100).toFixed(0)}%`).join(', ') || '-')},
{label:'Shared fields', render:r => esc((r.shared_fields || []).slice(0,4).map(sharedFieldLabel).join(', ') || '-')},
{label:'Discovery', render:r => profileDiscoveryDetails(r)},
{label:'Action', render:r => r.profile_exists ? '<button type="button" disabled>Applied</button>' : `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">Apply profile</button>`}
{label:'Action', render:r => `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">${r.profile_exists ? 'Update profile' : 'Apply profile'}</button>`}
], 'profile-suggestions') : '<p class="muted">No missing stream profiles. Enable "show existing profiles" to inspect already configured profiles.</p>';
const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`;
@@ -552,6 +559,22 @@ function renderStreamPicker(errorText='') {
document.getElementById('enableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = true; }); renderStreamPicker(); });
document.getElementById('disableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = false; }); renderStreamPicker(); });
}
function mergeProfile(existing, recommended) {
const mergeList = (left, right) => [...new Set([...(left || []), ...(right || [])].filter(Boolean))];
const detectors = {...(recommended.detectors || {}), ...(existing.detectors || {})};
return {
...recommended,
...existing,
name: existing.name || recommended.name,
entity_field: (existing.entity_field || recommended.entity_field || (recommended.entity_fields || [])[0] || ''),
entity_fields: mergeList(existing.entity_fields || (existing.entity_field ? [existing.entity_field] : []), recommended.entity_fields || (recommended.entity_field ? [recommended.entity_field] : [])),
timestamp_field: existing.timestamp_field || recommended.timestamp_field || 'timestamp',
categorical_fields: mergeList(existing.categorical_fields, recommended.categorical_fields),
numeric_fields: mergeList(existing.numeric_fields, recommended.numeric_fields),
detectors,
field_weights: {...(recommended.field_weights || {}), ...(existing.field_weights || {})}
};
}
async function applySuggestedProfile(streamId) {
const notice = document.getElementById('profileApplyResult');
const button = document.querySelector(`.apply-suggested-profile[data-stream-id="${CSS.escape(streamId)}"]`);
@@ -565,7 +588,8 @@ async function applySuggestedProfile(streamId) {
notice.textContent = `Applying profile for ${suggestion.stream_name || streamId}...`;
const configResponse = await fetch('/api/config', {cache: 'no-store'});
const config = await configResponse.json();
const profile = suggestion.profile;
const existingProfile = (config.graylog_stream_profiles || []).find(item => String(item.stream_id) === String(suggestion.profile.stream_id));
const profile = existingProfile ? mergeProfile(existingProfile, suggestion.profile) : suggestion.profile;
const payload = {
graylog_stream_profiles: [...(config.graylog_stream_profiles || []).filter(item => item.stream_id !== profile.stream_id), profile],
graylog_streams: config.graylog_streams || [],
@@ -601,7 +625,7 @@ async function applySuggestedProfile(streamId) {
notice.textContent = `Could not apply profile: ${result.error || response.statusText || response.status}`;
return;
}
notice.textContent = `Applied recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`;
notice.textContent = `${existingProfile ? 'Updated' : 'Applied'} recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`;
document.getElementById('settingsResult').textContent = notice.textContent;
window.streamProfiles = result.graylog_stream_profiles || payload.graylog_stream_profiles;
loadSettings();

2
src/fgai/monitor.py
View File

@@ -261,7 +261,7 @@ def build_status(
timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120),
)
profile_suggestions = apply_profile_advice(profile_suggestions, advice)
profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
profile_advisor_status = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
except Exception as exc:
profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
for suggestion in profile_suggestions:

93
src/fgai/profile_suggestions.py
View File

@@ -5,6 +5,13 @@ from collections import Counter, defaultdict
from .detectors import event_detector_categories
from .entities import ENTITY_FIELDS
from .models import LogEvent
from .normalization import FIELD_ALIASES
ALIAS_CANONICAL_BY_FIELD = {
alias: canonical
for canonical, aliases in FIELD_ALIASES.items()
for alias in aliases
}
ENTITY_PRIORITY = (
"srcip",
@@ -211,6 +218,22 @@ def _looks_like_time_field(field: str) -> bool:
return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
def _semantic_field_group(field: str) -> str:
return ALIAS_CANONICAL_BY_FIELD.get(field.lower(), field.lower())
def _shared_field_kind(field: str, *, cardinality: int, total: int, numeric_ratio: float) -> str:
if _looks_like_entity_field(field):
return "entity"
if _looks_like_time_field(field):
return "time"
if numeric_ratio >= 0.95:
return "numeric"
if cardinality > min(200, max(8, int(total * 0.7))):
return "high_cardinality"
return "categorical"
def _looks_like_windows_stream(stream_id: str, stream_name: str, coverage: Counter[str]) -> bool:
name = f"{stream_id} {stream_name}".lower()
if any(token in name for token in ("windows", "winlog", "event log", "security event", "sysmon", "powershell")):
@@ -365,10 +388,12 @@ def _shared_profile_fields(
numeric_counts: Counter[str],
total: int,
stream_field_counts: Counter[str],
stream_semantic_counts: Counter[str],
stream_semantic_fields: dict[str, set[str]],
) -> tuple[list[str], list[str], list[dict[str, object]]]:
categorical: list[str] = []
numeric: list[str] = []
stats: list[dict[str, object]] = []
stats_by_key: dict[str, dict[str, object]] = {}
for field, stream_count in stream_field_counts.most_common():
if stream_count < 2 or field in IGNORED_DISCOVERY_FIELDS or not coverage[field]:
continue
@@ -376,16 +401,58 @@ def _shared_profile_fields(
cardinality = len(unique_values[field])
if coverage_ratio < 0.1 or cardinality <= 1:
continue
if cardinality > min(200, max(8, int(total * 0.7))):
continue
numeric_ratio = numeric_counts[field] / max(1, coverage[field])
row = {"field": field, "streams": stream_count, "coverage": round(coverage_ratio, 2), "unique_values": cardinality}
stats.append(row)
if numeric_ratio >= 0.95:
kind = _shared_field_kind(field, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio)
candidate = kind in {"categorical", "numeric"}
stats_by_key.setdefault(field, {
"field": field,
"streams": stream_count,
"coverage": round(coverage_ratio, 2),
"unique_values": cardinality,
"kind": kind,
"candidate": candidate,
"semantic_group": _semantic_field_group(field),
})
if kind == "numeric":
numeric.append(field)
elif not _looks_like_entity_field(field) and not _looks_like_time_field(field):
elif kind == "categorical":
categorical.append(field)
return categorical[:8], numeric[:5], stats[:12]
for group, stream_count in stream_semantic_counts.most_common():
if stream_count < 2 or group in IGNORED_DISCOVERY_FIELDS:
continue
aliases = sorted(stream_semantic_fields.get(group, set()))
present_aliases = [field for field in aliases if coverage[field]]
if not present_aliases:
continue
best = max(present_aliases, key=lambda field: coverage[field])
coverage_ratio = coverage[best] / max(1, total)
cardinality = len(unique_values[best])
numeric_ratio = numeric_counts[best] / max(1, coverage[best])
kind = _shared_field_kind(group, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio)
if group not in stats_by_key:
stats_by_key[group] = {
"field": group,
"streams": stream_count,
"coverage": round(coverage_ratio, 2),
"unique_values": cardinality,
"kind": kind,
"candidate": kind in {"categorical", "numeric"},
"semantic_group": group,
"aliases": aliases[:10],
}
else:
stats_by_key[group].setdefault("aliases", aliases[:10])
stats_by_key[group]["streams"] = max(int(stats_by_key[group].get("streams", 0) or 0), stream_count)
stats = sorted(
stats_by_key.values(),
key=lambda item: (
0 if item.get("kind") in {"entity", "time"} else 1,
-int(item.get("streams", 0) or 0),
-float(item.get("coverage", 0) or 0),
str(item.get("field", "")),
),
)
return list(dict.fromkeys(categorical))[:8], list(dict.fromkeys(numeric))[:5], stats[:16]
def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
@@ -408,7 +475,7 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
item = dict(suggestion)
advisor = advice_by_stream.get(stream_id)
if not advisor:
item["profile_advisor"] = {"status": "not_run"}
item["profile_advisor"] = {"status": "heuristic", "reason": "advisor returned no profile for this stream"}
output.append(item)
continue
allowed_fields = _allowed_fields(item)
@@ -462,6 +529,8 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
grouped[stream_id].append(event)
names.setdefault(stream_id, _stream_name(event, stream_id))
stream_field_counts: Counter[str] = Counter()
stream_semantic_counts: Counter[str] = Counter()
stream_semantic_fields: dict[str, set[str]] = defaultdict(set)
for stream_events in grouped.values():
fields = {
field
@@ -470,6 +539,10 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
if field not in IGNORED_DISCOVERY_FIELDS and _has_discovery_value(value)
}
stream_field_counts.update(fields)
semantic_groups = {_semantic_field_group(field) for field in fields}
stream_semantic_counts.update(semantic_groups)
for field in fields:
stream_semantic_fields[_semantic_field_group(field)].add(field)
suggestions: list[dict[str, object]] = []
for stream_id, stream_events in grouped.items():
@@ -495,7 +568,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts)
shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields)
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3))
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
if not timestamp: