Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve find fileds for default profiles

Browse Source
This commit is contained in:
larssand
2026-07-02 09:51:25 +02:00
parent 53437a3952
commit 7c0b482a22
6 changed files with 167 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
README.md
View File

@@ -136,6 +136,23 @@ Ollama profile advisor can refine those recommendations, but the deterministic
profile discovery remains the fallback when Ollama is disabled, missing, slow, or
returns invalid JSON.
Cross-stream discovery is semantic, not just exact-name matching. Source IP
fields such as `srcip`, `source.ip`, `source_ip`, and `client_ip` are grouped as
the same shared entity field. The same approach is used for destination IPs,
ports, timestamps, actions, severities, usernames, hosts, event IDs, DNS names,
services, URLs, and context/message fields. Recommended profiles show both the
selected per-stream fields and the shared alias groups so you can see why a field
is useful for correlation even when different products use different schemas.
If the Ollama advisor returns no usable profile for a stream, the row stays on
the deterministic profile and is labeled as a heuristic fallback instead of
pretending the whole recommendation failed.
Recommended profiles can be re-applied to existing profiles. Enable `show
existing profiles` and click `Update profile` to append newly discovered entity,
categorical, numeric, and detector fields. Existing profile names, field weights,
and detector threshold settings are preserved, so this is the fast path after
field-alias matching improves or after Graylog starts parsing additional fields.
Enabled streams are normalized through the same event model. Stream profiles
define the entity, timestamp, categorical, and numeric fields used for baselines.
The dashboard and Ollama then correlate behavior across sources, for example a

32
src/fgai/dashboard.py
View File

@@ -189,11 +189,17 @@ function profileDiscoveryDetails(row) {
}
const selected = d.selected_fields || {};
const top = (d.top_fields || []).slice(0,8).map(item => `${item.field} ${(Number(item.coverage || 0)*100).toFixed(0)}%/${item.unique_values}`).join(', ');
const shared = (d.shared_fields || row.shared_fields || []).slice(0,8).map(item => `${item.field} (${item.streams} streams)`).join(', ');
const shared = (d.shared_fields || row.shared_fields || []).slice(0,8).map(sharedFieldLabel).join(', ');
const rejected = (d.rejected_fields || []).slice(0,8).map(item => `${item.field}: ${item.reason}`).join('; ');
const reasons = (d.reasons || []).join('; ');
return `<details data-detail-id="${esc(detailId)}"><summary>${esc(d.field_count)} fields analyzed</summary><p><b>Selected</b><br>Entity: ${esc((selected.entity || []).join(', ') || '-')}<br>Time: ${esc((selected.time || []).join(', ') || '-')}<br>Categorical: ${esc((selected.categorical || []).join(', ') || '-')}<br>Numeric: ${esc((selected.numeric || []).join(', ') || '-')}</p><p><b>Shared across streams</b><br>${esc(shared || '-')}</p><p><b>Top fields</b><br>${esc(top || '-')}</p><p><b>Rejected</b><br>${esc(rejected || '-')}</p><p>${esc(reasons || '')}</p></details>`;
}
function sharedFieldLabel(item) {
const aliases = (item.aliases || []).filter(alias => alias !== item.field).slice(0,4);
const suffix = aliases.length ? ` via ${aliases.join('/')}` : '';
const kind = item.kind ? ` ${item.kind}` : '';
return `${item.field}${kind} (${item.streams} streams${suffix})`;
}
function profileAdvisorLabel(row, advisor) {
const rowAdvisor = row.profile_advisor || {};
if (rowAdvisor.status === 'heuristic' && rowAdvisor.error) return 'heuristic (advisor error)';
@@ -346,8 +352,9 @@ async function refresh() {
{label:'Baseline fields', render:r => esc([...(r.categorical_fields || []), ...(r.numeric_fields || [])].slice(0,8).join(', ') || '-')},
{label:'Detectors', render:r => esc(Object.keys(r.detectors || {}).join(', ') || '-')},
{label:'Common denominators', render:r => esc((r.common_fields || []).slice(0,5).map(item => `${item.field} ${(item.coverage*100).toFixed(0)}%`).join(', ') || '-')},
{label:'Shared fields', render:r => esc((r.shared_fields || []).slice(0,4).map(sharedFieldLabel).join(', ') || '-')},
{label:'Discovery', render:r => profileDiscoveryDetails(r)},
{label:'Action', render:r => r.profile_exists ? '<button type="button" disabled>Applied</button>' : `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">Apply profile</button>`}
{label:'Action', render:r => `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">${r.profile_exists ? 'Update profile' : 'Apply profile'}</button>`}
], 'profile-suggestions') : '<p class="muted">No missing stream profiles. Enable "show existing profiles" to inspect already configured profiles.</p>';
const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`;
@@ -552,6 +559,22 @@ function renderStreamPicker(errorText='') {
document.getElementById('enableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = true; }); renderStreamPicker(); });
document.getElementById('disableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = false; }); renderStreamPicker(); });
}
function mergeProfile(existing, recommended) {
const mergeList = (left, right) => [...new Set([...(left || []), ...(right || [])].filter(Boolean))];
const detectors = {...(recommended.detectors || {}), ...(existing.detectors || {})};
return {
...recommended,
...existing,
name: existing.name || recommended.name,
entity_field: (existing.entity_field || recommended.entity_field || (recommended.entity_fields || [])[0] || ''),
entity_fields: mergeList(existing.entity_fields || (existing.entity_field ? [existing.entity_field] : []), recommended.entity_fields || (recommended.entity_field ? [recommended.entity_field] : [])),
timestamp_field: existing.timestamp_field || recommended.timestamp_field || 'timestamp',
categorical_fields: mergeList(existing.categorical_fields, recommended.categorical_fields),
numeric_fields: mergeList(existing.numeric_fields, recommended.numeric_fields),
detectors,
field_weights: {...(recommended.field_weights || {}), ...(existing.field_weights || {})}
};
}
async function applySuggestedProfile(streamId) {
const notice = document.getElementById('profileApplyResult');
const button = document.querySelector(`.apply-suggested-profile[data-stream-id="${CSS.escape(streamId)}"]`);
@@ -565,7 +588,8 @@ async function applySuggestedProfile(streamId) {
notice.textContent = `Applying profile for ${suggestion.stream_name || streamId}...`;
const configResponse = await fetch('/api/config', {cache: 'no-store'});
const config = await configResponse.json();
const profile = suggestion.profile;
const existingProfile = (config.graylog_stream_profiles || []).find(item => String(item.stream_id) === String(suggestion.profile.stream_id));
const profile = existingProfile ? mergeProfile(existingProfile, suggestion.profile) : suggestion.profile;
const payload = {
graylog_stream_profiles: [...(config.graylog_stream_profiles || []).filter(item => item.stream_id !== profile.stream_id), profile],
graylog_streams: config.graylog_streams || [],
@@ -601,7 +625,7 @@ async function applySuggestedProfile(streamId) {
notice.textContent = `Could not apply profile: ${result.error || response.statusText || response.status}`;
return;
}
notice.textContent = `Applied recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`;
notice.textContent = `${existingProfile ? 'Updated' : 'Applied'} recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`;
document.getElementById('settingsResult').textContent = notice.textContent;
window.streamProfiles = result.graylog_stream_profiles || payload.graylog_stream_profiles;
loadSettings();

2
src/fgai/monitor.py
View File

@@ -261,7 +261,7 @@ def build_status(
timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120),
)
profile_suggestions = apply_profile_advice(profile_suggestions, advice)
profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
profile_advisor_status = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
except Exception as exc:
profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")}
for suggestion in profile_suggestions:

93
src/fgai/profile_suggestions.py
View File

@@ -5,6 +5,13 @@ from collections import Counter, defaultdict
from .detectors import event_detector_categories
from .entities import ENTITY_FIELDS
from .models import LogEvent
from .normalization import FIELD_ALIASES
ALIAS_CANONICAL_BY_FIELD = {
alias: canonical
for canonical, aliases in FIELD_ALIASES.items()
for alias in aliases
}
ENTITY_PRIORITY = (
"srcip",
@@ -211,6 +218,22 @@ def _looks_like_time_field(field: str) -> bool:
return any(token in field for token in ("time", "timestamp", "created", "@timestamp"))
def _semantic_field_group(field: str) -> str:
return ALIAS_CANONICAL_BY_FIELD.get(field.lower(), field.lower())
def _shared_field_kind(field: str, *, cardinality: int, total: int, numeric_ratio: float) -> str:
if _looks_like_entity_field(field):
return "entity"
if _looks_like_time_field(field):
return "time"
if numeric_ratio >= 0.95:
return "numeric"
if cardinality > min(200, max(8, int(total * 0.7))):
return "high_cardinality"
return "categorical"
def _looks_like_windows_stream(stream_id: str, stream_name: str, coverage: Counter[str]) -> bool:
name = f"{stream_id} {stream_name}".lower()
if any(token in name for token in ("windows", "winlog", "event log", "security event", "sysmon", "powershell")):
@@ -365,10 +388,12 @@ def _shared_profile_fields(
numeric_counts: Counter[str],
total: int,
stream_field_counts: Counter[str],
stream_semantic_counts: Counter[str],
stream_semantic_fields: dict[str, set[str]],
) -> tuple[list[str], list[str], list[dict[str, object]]]:
categorical: list[str] = []
numeric: list[str] = []
stats: list[dict[str, object]] = []
stats_by_key: dict[str, dict[str, object]] = {}
for field, stream_count in stream_field_counts.most_common():
if stream_count < 2 or field in IGNORED_DISCOVERY_FIELDS or not coverage[field]:
continue
@@ -376,16 +401,58 @@ def _shared_profile_fields(
cardinality = len(unique_values[field])
if coverage_ratio < 0.1 or cardinality <= 1:
continue
if cardinality > min(200, max(8, int(total * 0.7))):
continue
numeric_ratio = numeric_counts[field] / max(1, coverage[field])
row = {"field": field, "streams": stream_count, "coverage": round(coverage_ratio, 2), "unique_values": cardinality}
stats.append(row)
if numeric_ratio >= 0.95:
kind = _shared_field_kind(field, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio)
candidate = kind in {"categorical", "numeric"}
stats_by_key.setdefault(field, {
"field": field,
"streams": stream_count,
"coverage": round(coverage_ratio, 2),
"unique_values": cardinality,
"kind": kind,
"candidate": candidate,
"semantic_group": _semantic_field_group(field),
})
if kind == "numeric":
numeric.append(field)
elif not _looks_like_entity_field(field) and not _looks_like_time_field(field):
elif kind == "categorical":
categorical.append(field)
return categorical[:8], numeric[:5], stats[:12]
for group, stream_count in stream_semantic_counts.most_common():
if stream_count < 2 or group in IGNORED_DISCOVERY_FIELDS:
continue
aliases = sorted(stream_semantic_fields.get(group, set()))
present_aliases = [field for field in aliases if coverage[field]]
if not present_aliases:
continue
best = max(present_aliases, key=lambda field: coverage[field])
coverage_ratio = coverage[best] / max(1, total)
cardinality = len(unique_values[best])
numeric_ratio = numeric_counts[best] / max(1, coverage[best])
kind = _shared_field_kind(group, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio)
if group not in stats_by_key:
stats_by_key[group] = {
"field": group,
"streams": stream_count,
"coverage": round(coverage_ratio, 2),
"unique_values": cardinality,
"kind": kind,
"candidate": kind in {"categorical", "numeric"},
"semantic_group": group,
"aliases": aliases[:10],
}
else:
stats_by_key[group].setdefault("aliases", aliases[:10])
stats_by_key[group]["streams"] = max(int(stats_by_key[group].get("streams", 0) or 0), stream_count)
stats = sorted(
stats_by_key.values(),
key=lambda item: (
0 if item.get("kind") in {"entity", "time"} else 1,
-int(item.get("streams", 0) or 0),
-float(item.get("coverage", 0) or 0),
str(item.get("field", "")),
),
)
return list(dict.fromkeys(categorical))[:8], list(dict.fromkeys(numeric))[:5], stats[:16]
def _allowed_fields(suggestion: dict[str, object]) -> set[str]:
@@ -408,7 +475,7 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict
item = dict(suggestion)
advisor = advice_by_stream.get(stream_id)
if not advisor:
item["profile_advisor"] = {"status": "not_run"}
item["profile_advisor"] = {"status": "heuristic", "reason": "advisor returned no profile for this stream"}
output.append(item)
continue
allowed_fields = _allowed_fields(item)
@@ -462,6 +529,8 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
grouped[stream_id].append(event)
names.setdefault(stream_id, _stream_name(event, stream_id))
stream_field_counts: Counter[str] = Counter()
stream_semantic_counts: Counter[str] = Counter()
stream_semantic_fields: dict[str, set[str]] = defaultdict(set)
for stream_events in grouped.values():
fields = {
field
@@ -470,6 +539,10 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
if field not in IGNORED_DISCOVERY_FIELDS and _has_discovery_value(value)
}
stream_field_counts.update(fields)
semantic_groups = {_semantic_field_group(field) for field in fields}
stream_semantic_counts.update(semantic_groups)
for field in fields:
stream_semantic_fields[_semantic_field_group(field)].add(field)
suggestions: list[dict[str, object]] = []
for stream_id, stream_events in grouped.items():
@@ -495,7 +568,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY
categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY
numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY
shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts)
shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields)
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3))
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
if not timestamp:

4
tests/test_dashboard.py
View File

@@ -28,6 +28,10 @@ class DashboardTests(unittest.TestCase):
self.assertIn('data-view="howto"', HTML)
self.assertIn("How To Use SignalScope", HTML)
def test_dashboard_can_update_existing_recommended_profiles(self):
self.assertIn("Update profile", HTML)
self.assertIn("function mergeProfile", HTML)
if __name__ == "__main__":
unittest.main()

34
tests/test_profile_suggestions.py
View File

@@ -178,6 +178,30 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("result_code", profile["categorical_fields"])
self.assertIn("latency_ms", profile["numeric_fields"])
def test_semantic_alias_fields_are_shared_across_streams(self):
events = []
events.extend(
parse_log_line(
f"fgai_stream_id=proxy fgai_stream=Proxy source.ip=10.0.0.{index % 7} url=/item/{index % 3} action=allow"
)
for index in range(1, 30)
)
events.extend(
parse_log_line(
f"fgai_stream_id=firewall fgai_stream=Firewall client_ip=10.0.0.{index % 7} dstport=443 action=accept"
)
for index in range(1, 30)
)
suggestions = {item["stream_id"]: item for item in suggest_stream_profiles(events)}
for suggestion in suggestions.values():
shared = {item["field"]: item for item in suggestion["shared_fields"]}
self.assertIn("srcip", shared)
self.assertEqual(shared["srcip"]["kind"], "entity")
self.assertIn("source.ip", shared["srcip"]["aliases"])
self.assertIn("client_ip", shared["srcip"]["aliases"])
def test_applies_valid_llm_advice_and_rejects_unknown_fields(self):
suggestion = suggest_stream_profiles([
parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure")
@@ -199,6 +223,16 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertNotIn("full_message", advised["profile"]["categorical_fields"])
self.assertEqual(set(advised["profile"]["detectors"]), {"auth_failure"})
def test_missing_llm_advice_keeps_heuristic_profile(self):
suggestion = suggest_stream_profiles([
parse_log_line("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login")
])[0]
advised = apply_profile_advice([suggestion], [])[0]
self.assertEqual(advised["profile_advisor"]["status"], "heuristic")
self.assertEqual(advised["profile"], suggestion["profile"])
if __name__ == "__main__":
unittest.main()