Selected
Entity: ${esc((selected.entity || []).join(', ') || '-')}
Time: ${esc((selected.time || []).join(', ') || '-')}
Categorical: ${esc((selected.categorical || []).join(', ') || '-')}
Numeric: ${esc((selected.numeric || []).join(', ') || '-')}
Shared across streams
${esc(shared || '-')}
Top fields
${esc(top || '-')}
Rejected
${esc(rejected || '-')}
${esc(reasons || '')}
No missing stream profiles. Enable "show existing profiles" to inspect already configured profiles.
'; const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '${esc(llm.status || 'unknown')}${llmText}
`; @@ -552,6 +559,22 @@ function renderStreamPicker(errorText='') { document.getElementById('enableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = true; }); renderStreamPicker(); }); document.getElementById('disableVisibleStreams').addEventListener('click', () => { streams.forEach(stream => { window.streamSelection[stream.id].enabled = false; }); renderStreamPicker(); }); } +function mergeProfile(existing, recommended) { + const mergeList = (left, right) => [...new Set([...(left || []), ...(right || [])].filter(Boolean))]; + const detectors = {...(recommended.detectors || {}), ...(existing.detectors || {})}; + return { + ...recommended, + ...existing, + name: existing.name || recommended.name, + entity_field: (existing.entity_field || recommended.entity_field || (recommended.entity_fields || [])[0] || ''), + entity_fields: mergeList(existing.entity_fields || (existing.entity_field ? [existing.entity_field] : []), recommended.entity_fields || (recommended.entity_field ? [recommended.entity_field] : [])), + timestamp_field: existing.timestamp_field || recommended.timestamp_field || 'timestamp', + categorical_fields: mergeList(existing.categorical_fields, recommended.categorical_fields), + numeric_fields: mergeList(existing.numeric_fields, recommended.numeric_fields), + detectors, + field_weights: {...(recommended.field_weights || {}), ...(existing.field_weights || {})} + }; +} async function applySuggestedProfile(streamId) { const notice = document.getElementById('profileApplyResult'); const button = document.querySelector(`.apply-suggested-profile[data-stream-id="${CSS.escape(streamId)}"]`); @@ -565,7 +588,8 @@ async function applySuggestedProfile(streamId) { notice.textContent = `Applying profile for ${suggestion.stream_name || streamId}...`; const configResponse = await fetch('/api/config', {cache: 'no-store'}); const config = await configResponse.json(); - const profile = suggestion.profile; + const existingProfile = (config.graylog_stream_profiles || []).find(item => String(item.stream_id) === String(suggestion.profile.stream_id)); + const profile = existingProfile ? mergeProfile(existingProfile, suggestion.profile) : suggestion.profile; const payload = { graylog_stream_profiles: [...(config.graylog_stream_profiles || []).filter(item => item.stream_id !== profile.stream_id), profile], graylog_streams: config.graylog_streams || [], @@ -601,7 +625,7 @@ async function applySuggestedProfile(streamId) { notice.textContent = `Could not apply profile: ${result.error || response.statusText || response.status}`; return; } - notice.textContent = `Applied recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`; + notice.textContent = `${existingProfile ? 'Updated' : 'Applied'} recommended profile for ${suggestion.stream_name || streamId}. Monitor will use it on the next cycle.`; document.getElementById('settingsResult').textContent = notice.textContent; window.streamProfiles = result.graylog_stream_profiles || payload.graylog_stream_profiles; loadSettings(); diff --git a/src/fgai/monitor.py b/src/fgai/monitor.py index 22de8b6..8120e75 100644 --- a/src/fgai/monitor.py +++ b/src/fgai/monitor.py @@ -261,7 +261,7 @@ def build_status( timeout=int(runtime_values.get("profile_advisor_timeout", 120) or 120), ) profile_suggestions = apply_profile_advice(profile_suggestions, advice) - profile_advisor_status = {"enabled": True, "status": "ok", "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")} + profile_advisor_status = {"enabled": True, "status": "ok" if advice else "empty", "profiles_returned": len(advice), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")} except Exception as exc: profile_advisor_status = {"enabled": True, "status": "error", "error": str(exc), "model": str(runtime_values.get("profile_advisor_model", "") or "qwen3:8b")} for suggestion in profile_suggestions: diff --git a/src/fgai/profile_suggestions.py b/src/fgai/profile_suggestions.py index 89198b9..9da2eaf 100644 --- a/src/fgai/profile_suggestions.py +++ b/src/fgai/profile_suggestions.py @@ -5,6 +5,13 @@ from collections import Counter, defaultdict from .detectors import event_detector_categories from .entities import ENTITY_FIELDS from .models import LogEvent +from .normalization import FIELD_ALIASES + +ALIAS_CANONICAL_BY_FIELD = { + alias: canonical + for canonical, aliases in FIELD_ALIASES.items() + for alias in aliases +} ENTITY_PRIORITY = ( "srcip", @@ -211,6 +218,22 @@ def _looks_like_time_field(field: str) -> bool: return any(token in field for token in ("time", "timestamp", "created", "@timestamp")) +def _semantic_field_group(field: str) -> str: + return ALIAS_CANONICAL_BY_FIELD.get(field.lower(), field.lower()) + + +def _shared_field_kind(field: str, *, cardinality: int, total: int, numeric_ratio: float) -> str: + if _looks_like_entity_field(field): + return "entity" + if _looks_like_time_field(field): + return "time" + if numeric_ratio >= 0.95: + return "numeric" + if cardinality > min(200, max(8, int(total * 0.7))): + return "high_cardinality" + return "categorical" + + def _looks_like_windows_stream(stream_id: str, stream_name: str, coverage: Counter[str]) -> bool: name = f"{stream_id} {stream_name}".lower() if any(token in name for token in ("windows", "winlog", "event log", "security event", "sysmon", "powershell")): @@ -365,10 +388,12 @@ def _shared_profile_fields( numeric_counts: Counter[str], total: int, stream_field_counts: Counter[str], + stream_semantic_counts: Counter[str], + stream_semantic_fields: dict[str, set[str]], ) -> tuple[list[str], list[str], list[dict[str, object]]]: categorical: list[str] = [] numeric: list[str] = [] - stats: list[dict[str, object]] = [] + stats_by_key: dict[str, dict[str, object]] = {} for field, stream_count in stream_field_counts.most_common(): if stream_count < 2 or field in IGNORED_DISCOVERY_FIELDS or not coverage[field]: continue @@ -376,16 +401,58 @@ def _shared_profile_fields( cardinality = len(unique_values[field]) if coverage_ratio < 0.1 or cardinality <= 1: continue - if cardinality > min(200, max(8, int(total * 0.7))): - continue numeric_ratio = numeric_counts[field] / max(1, coverage[field]) - row = {"field": field, "streams": stream_count, "coverage": round(coverage_ratio, 2), "unique_values": cardinality} - stats.append(row) - if numeric_ratio >= 0.95: + kind = _shared_field_kind(field, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio) + candidate = kind in {"categorical", "numeric"} + stats_by_key.setdefault(field, { + "field": field, + "streams": stream_count, + "coverage": round(coverage_ratio, 2), + "unique_values": cardinality, + "kind": kind, + "candidate": candidate, + "semantic_group": _semantic_field_group(field), + }) + if kind == "numeric": numeric.append(field) - elif not _looks_like_entity_field(field) and not _looks_like_time_field(field): + elif kind == "categorical": categorical.append(field) - return categorical[:8], numeric[:5], stats[:12] + for group, stream_count in stream_semantic_counts.most_common(): + if stream_count < 2 or group in IGNORED_DISCOVERY_FIELDS: + continue + aliases = sorted(stream_semantic_fields.get(group, set())) + present_aliases = [field for field in aliases if coverage[field]] + if not present_aliases: + continue + best = max(present_aliases, key=lambda field: coverage[field]) + coverage_ratio = coverage[best] / max(1, total) + cardinality = len(unique_values[best]) + numeric_ratio = numeric_counts[best] / max(1, coverage[best]) + kind = _shared_field_kind(group, cardinality=cardinality, total=total, numeric_ratio=numeric_ratio) + if group not in stats_by_key: + stats_by_key[group] = { + "field": group, + "streams": stream_count, + "coverage": round(coverage_ratio, 2), + "unique_values": cardinality, + "kind": kind, + "candidate": kind in {"categorical", "numeric"}, + "semantic_group": group, + "aliases": aliases[:10], + } + else: + stats_by_key[group].setdefault("aliases", aliases[:10]) + stats_by_key[group]["streams"] = max(int(stats_by_key[group].get("streams", 0) or 0), stream_count) + stats = sorted( + stats_by_key.values(), + key=lambda item: ( + 0 if item.get("kind") in {"entity", "time"} else 1, + -int(item.get("streams", 0) or 0), + -float(item.get("coverage", 0) or 0), + str(item.get("field", "")), + ), + ) + return list(dict.fromkeys(categorical))[:8], list(dict.fromkeys(numeric))[:5], stats[:16] def _allowed_fields(suggestion: dict[str, object]) -> set[str]: @@ -408,7 +475,7 @@ def apply_profile_advice(suggestions: list[dict[str, object]], advice: list[dict item = dict(suggestion) advisor = advice_by_stream.get(stream_id) if not advisor: - item["profile_advisor"] = {"status": "not_run"} + item["profile_advisor"] = {"status": "heuristic", "reason": "advisor returned no profile for this stream"} output.append(item) continue allowed_fields = _allowed_fields(item) @@ -462,6 +529,8 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s grouped[stream_id].append(event) names.setdefault(stream_id, _stream_name(event, stream_id)) stream_field_counts: Counter[str] = Counter() + stream_semantic_counts: Counter[str] = Counter() + stream_semantic_fields: dict[str, set[str]] = defaultdict(set) for stream_events in grouped.values(): fields = { field @@ -470,6 +539,10 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s if field not in IGNORED_DISCOVERY_FIELDS and _has_discovery_value(value) } stream_field_counts.update(fields) + semantic_groups = {_semantic_field_group(field) for field in fields} + stream_semantic_counts.update(semantic_groups) + for field in fields: + stream_semantic_fields[_semantic_field_group(field)].add(field) suggestions: list[dict[str, object]] = [] for stream_id, stream_events in grouped.items(): @@ -495,7 +568,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s entity_priority = WINDOWS_ENTITY_PRIORITY if is_windows else ENTITY_PRIORITY categorical_priority = WINDOWS_CATEGORICAL_PRIORITY if is_windows else CATEGORICAL_PRIORITY numeric_priority = WINDOWS_NUMERIC_PRIORITY if is_windows else NUMERIC_PRIORITY - shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts) + shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields) entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3)) timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "") if not timestamp: diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 20f9941..1eebb53 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -28,6 +28,10 @@ class DashboardTests(unittest.TestCase): self.assertIn('data-view="howto"', HTML) self.assertIn("How To Use SignalScope", HTML) + def test_dashboard_can_update_existing_recommended_profiles(self): + self.assertIn("Update profile", HTML) + self.assertIn("function mergeProfile", HTML) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_profile_suggestions.py b/tests/test_profile_suggestions.py index c176011..d556072 100644 --- a/tests/test_profile_suggestions.py +++ b/tests/test_profile_suggestions.py @@ -178,6 +178,30 @@ class ProfileSuggestionTests(unittest.TestCase): self.assertIn("result_code", profile["categorical_fields"]) self.assertIn("latency_ms", profile["numeric_fields"]) + def test_semantic_alias_fields_are_shared_across_streams(self): + events = [] + events.extend( + parse_log_line( + f"fgai_stream_id=proxy fgai_stream=Proxy source.ip=10.0.0.{index % 7} url=/item/{index % 3} action=allow" + ) + for index in range(1, 30) + ) + events.extend( + parse_log_line( + f"fgai_stream_id=firewall fgai_stream=Firewall client_ip=10.0.0.{index % 7} dstport=443 action=accept" + ) + for index in range(1, 30) + ) + + suggestions = {item["stream_id"]: item for item in suggest_stream_profiles(events)} + + for suggestion in suggestions.values(): + shared = {item["field"]: item for item in suggestion["shared_fields"]} + self.assertIn("srcip", shared) + self.assertEqual(shared["srcip"]["kind"], "entity") + self.assertIn("source.ip", shared["srcip"]["aliases"]) + self.assertIn("client_ip", shared["srcip"]["aliases"]) + def test_applies_valid_llm_advice_and_rejects_unknown_fields(self): suggestion = suggest_stream_profiles([ parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure") @@ -199,6 +223,16 @@ class ProfileSuggestionTests(unittest.TestCase): self.assertNotIn("full_message", advised["profile"]["categorical_fields"]) self.assertEqual(set(advised["profile"]["detectors"]), {"auth_failure"}) + def test_missing_llm_advice_keeps_heuristic_profile(self): + suggestion = suggest_stream_profiles([ + parse_log_line("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login") + ])[0] + + advised = apply_profile_advice([suggestion], [])[0] + + self.assertEqual(advised["profile_advisor"]["status"], "heuristic") + self.assertEqual(advised["profile"], suggestion["profile"]) + if __name__ == "__main__": unittest.main()