Looks at observed logs per stream.
Finds common denominators:
This commit is contained in:
larssand
@@ -89,7 +89,7 @@ HTML = """<!doctype html>
|
||||
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
|
||||
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
|
||||
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
|
||||
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
|
||||
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
|
||||
</main>
|
||||
<script>
|
||||
function esc(value) {
|
||||
@@ -175,6 +175,19 @@ async function refresh() {
|
||||
document.getElementById('health').innerHTML = [
|
||||
metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Correlated entities', correlations.length)
|
||||
].join('');
|
||||
window.profileSuggestions = data.profile_suggestions || [];
|
||||
document.getElementById('profileSuggestions').innerHTML = table(window.profileSuggestions, [
|
||||
{label:'Stream', key:'stream_name'},
|
||||
{label:'Events', key:'events'},
|
||||
{label:'Confidence', key:'confidence'},
|
||||
{label:'Profile', render:r => r.profile_exists ? 'exists' : 'new'},
|
||||
{label:'Entity fields', render:r => esc((r.entity_fields || []).join(', ') || '-')},
|
||||
{label:'Time', key:'timestamp_field'},
|
||||
{label:'Baseline fields', render:r => esc([...(r.categorical_fields || []), ...(r.numeric_fields || [])].slice(0,8).join(', ') || '-')},
|
||||
{label:'Detectors', render:r => esc(Object.keys(r.detectors || {}).join(', ') || '-')},
|
||||
{label:'Common denominators', render:r => esc((r.common_fields || []).slice(0,5).map(item => `${item.field} ${(item.coverage*100).toFixed(0)}%`).join(', ') || '-')},
|
||||
{label:'Action', render:r => `<button type="button" class="apply-suggested-profile" data-stream-id="${esc(r.stream_id)}">Apply profile</button>`}
|
||||
], 'profile-suggestions');
|
||||
const llmText = llm.text ? esc(llm.text).replace(/\\n/g, '<br>') : esc(llm.error || 'LLM assessment disabled or waiting for first run.');
|
||||
document.getElementById('llmAssessment').innerHTML = `<div>Status: <code>${esc(llm.status || 'unknown')}</code></div><p>${llmText}</p>`;
|
||||
document.getElementById('anomalies').innerHTML = table(data.anomalies || [], [
|
||||
@@ -302,6 +315,7 @@ async function refresh() {
|
||||
'<h3>Local-in Failures</h3>' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip'}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]);
|
||||
document.querySelectorAll('details[data-detail-id]').forEach(item => { if (openDetails.has(item.dataset.detailId)) item.open = true; });
|
||||
document.querySelectorAll('[data-sort-table]').forEach(button => button.addEventListener('click', () => { const current=tableSort[button.dataset.sortTable]; tableSort[button.dataset.sortTable]={key:button.dataset.sortKey,direction:current && current.key===button.dataset.sortKey ? -current.direction : 1}; refresh(); }));
|
||||
document.querySelectorAll('.apply-suggested-profile').forEach(button => button.addEventListener('click', () => applySuggestedProfile(button.dataset.streamId)));
|
||||
}
|
||||
async function loadSettings() {
|
||||
const config = await (await fetch('/api/config', {cache: 'no-store'})).json();
|
||||
@@ -328,6 +342,36 @@ async function loadStreams() {
|
||||
document.getElementById('streamPicker').innerHTML = (payload.streams || []).map(stream => `<div class="stream-row"><label><input type="checkbox" class="graylog-stream" data-id="${esc(stream.id)}" data-title="${esc(stream.title)}" ${selected.has(stream.id) ? 'checked' : ''}> ${esc(stream.title)}</label> <button type="button" class="edit-profile" data-id="${esc(stream.id)}" data-title="${esc(stream.title)}">Edit profile</button></div>`).join('') || esc(payload.error || 'No streams found.');
|
||||
}
|
||||
document.getElementById('loadStreams').addEventListener('click', loadStreams);
|
||||
async function applySuggestedProfile(streamId) {
|
||||
const suggestion = (window.profileSuggestions || []).find(item => item.stream_id === streamId);
|
||||
if (!suggestion || !suggestion.profile) return;
|
||||
const config = await (await fetch('/api/config', {cache: 'no-store'})).json();
|
||||
const profile = suggestion.profile;
|
||||
const payload = {
|
||||
graylog_stream_profiles: [...(config.graylog_stream_profiles || []).filter(item => item.stream_id !== profile.stream_id), profile],
|
||||
graylog_streams: config.graylog_streams || [],
|
||||
log_source: config.log_source || 'graylog_mcp',
|
||||
graylog_mcp_url: config.graylog_mcp_url || '',
|
||||
graylog_query: config.graylog_query || '*',
|
||||
graylog_range_seconds: config.graylog_range_seconds || 3600,
|
||||
graylog_field_mapping: config.graylog_field_mapping || '',
|
||||
baseline_training_days: config.baseline_training_days || 7,
|
||||
baseline_retention_days: config.baseline_retention_days || 14,
|
||||
baseline_value_retention_days: config.baseline_value_retention_days || 7,
|
||||
baseline_max_values_per_field: config.baseline_max_values_per_field || 2000,
|
||||
llm_enabled: Boolean(config.llm_enabled),
|
||||
llm_model: config.llm_model || '',
|
||||
threat_intel_enabled: Boolean(config.threat_intel_enabled),
|
||||
threat_intel_provider: config.threat_intel_provider || 'auto',
|
||||
threat_intel_daily_limit: config.threat_intel_daily_limit || 100,
|
||||
threat_intel_ttl_seconds: config.threat_intel_ttl_seconds || 604800,
|
||||
threat_intel_error_ttl_seconds: config.threat_intel_error_ttl_seconds || 3600,
|
||||
abuseipdb_max_age_days: config.abuseipdb_max_age_days || 90
|
||||
};
|
||||
const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload)});
|
||||
document.getElementById('settingsResult').textContent = response.ok ? `Applied recommended profile for ${suggestion.stream_name || streamId}.` : 'Could not apply recommended profile.';
|
||||
if (response.ok) loadSettings();
|
||||
}
|
||||
async function editStreamProfile(streamId, title) {
|
||||
document.getElementById('profileEditorStatus').innerHTML = `Editing profile for <code>${esc(title || streamId)}</code>`;
|
||||
document.getElementById('fieldPicker').textContent = 'Loading stream fields...';
|
||||
|
||||
@@ -64,6 +64,7 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None =
|
||||
"capabilities": analysis.get("capabilities", {}),
|
||||
"cross_source_correlations": analysis.get("cross_source_correlations", [])[:20],
|
||||
"incidents": analysis.get("incidents", [])[:10],
|
||||
"profile_suggestions": analysis.get("profile_suggestions", [])[:10],
|
||||
"field_deviations": analysis.get("field_deviations", {}),
|
||||
"feedback": analysis.get("feedback", []),
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ from .llm import ollama_dashboard_assessment
|
||||
from .logs import local_in_failures, read_events, summarize_events, top_field_values
|
||||
from .mitigation import parse_allowlist, suggest_block_candidates
|
||||
from .policies import audit_policies, read_policies
|
||||
from .profile_suggestions import suggest_stream_profiles
|
||||
from .recommendations import build_recommendations
|
||||
from .sequences import detect_sequences
|
||||
from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
|
||||
@@ -201,6 +202,7 @@ def build_status(
|
||||
for item in profile_readiness
|
||||
]
|
||||
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
|
||||
profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles)
|
||||
intel_ips = sorted(
|
||||
{
|
||||
ip
|
||||
@@ -251,6 +253,7 @@ def build_status(
|
||||
"configuration": runtime_config,
|
||||
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
|
||||
"stream_coverage": stream_coverage,
|
||||
"profile_suggestions": profile_suggestions,
|
||||
"profile_readiness": profile_readiness,
|
||||
"diagnostics": {
|
||||
"top_source_ips": top_field_values(events, "srcip", limit=10),
|
||||
|
||||
137
src/fgai/profile_suggestions.py
Normal file
137
src/fgai/profile_suggestions.py
Normal file
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
from .detectors import event_detector_categories
|
||||
from .models import LogEvent
|
||||
|
||||
ENTITY_PRIORITY = (
|
||||
"srcip",
|
||||
"username",
|
||||
"user",
|
||||
"hostname",
|
||||
"host",
|
||||
"source",
|
||||
"client_ip",
|
||||
"ip",
|
||||
)
|
||||
TIME_PRIORITY = ("eventtime", "timestamp", "time", "created_at")
|
||||
CATEGORICAL_PRIORITY = (
|
||||
"action",
|
||||
"severity",
|
||||
"type",
|
||||
"subtype",
|
||||
"service",
|
||||
"dstport",
|
||||
"srcport",
|
||||
"policyid",
|
||||
"dns_query",
|
||||
"query_domain",
|
||||
"url",
|
||||
"context",
|
||||
)
|
||||
NUMERIC_PRIORITY = ("hitcount", "sentbyte", "rcvdbyte", "duration", "elapsed", "proto", "event_id")
|
||||
|
||||
|
||||
def _stream_id(event: LogEvent) -> str:
|
||||
return event.fields.get("fgai_stream_id") or event.fields.get("fgai_stream") or "local"
|
||||
|
||||
|
||||
def _stream_name(event: LogEvent, stream_id: str) -> str:
|
||||
return event.fields.get("fgai_stream") or event.fields.get("fgai_stream_name") or stream_id
|
||||
|
||||
|
||||
def _is_number(value: str) -> bool:
|
||||
try:
|
||||
float(value)
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _pick_present(priority: tuple[str, ...], coverage: Counter[str], total: int, *, min_ratio: float = 0.05, limit: int = 6) -> list[str]:
|
||||
selected = []
|
||||
for field in priority:
|
||||
if coverage[field] and coverage[field] / max(1, total) >= min_ratio:
|
||||
selected.append(field)
|
||||
if len(selected) >= limit:
|
||||
break
|
||||
return selected
|
||||
|
||||
|
||||
def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[str, object] | None = None) -> list[dict[str, object]]:
|
||||
existing_profiles = existing_profiles or {}
|
||||
grouped: dict[str, list[LogEvent]] = defaultdict(list)
|
||||
names: dict[str, str] = {}
|
||||
for event in events:
|
||||
stream_id = _stream_id(event)
|
||||
grouped[stream_id].append(event)
|
||||
names.setdefault(stream_id, _stream_name(event, stream_id))
|
||||
|
||||
suggestions: list[dict[str, object]] = []
|
||||
for stream_id, stream_events in grouped.items():
|
||||
total = len(stream_events)
|
||||
coverage: Counter[str] = Counter()
|
||||
unique_values: dict[str, set[str]] = defaultdict(set)
|
||||
numeric_counts: Counter[str] = Counter()
|
||||
detector_counts: Counter[str] = Counter()
|
||||
for event in stream_events:
|
||||
for field, value in event.fields.items():
|
||||
if not value:
|
||||
continue
|
||||
coverage[field] += 1
|
||||
if len(unique_values[field]) < 200:
|
||||
unique_values[field].add(value)
|
||||
if _is_number(value):
|
||||
numeric_counts[field] += 1
|
||||
detector_counts.update(event_detector_categories(event))
|
||||
|
||||
entity_fields = _pick_present(ENTITY_PRIORITY, coverage, total, min_ratio=0.02, limit=3)
|
||||
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "timestamp")
|
||||
categorical = _pick_present(CATEGORICAL_PRIORITY, coverage, total, min_ratio=0.02, limit=8)
|
||||
numeric = [
|
||||
field for field in NUMERIC_PRIORITY
|
||||
if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
|
||||
][:5]
|
||||
if not entity_fields:
|
||||
entity_fields = [field for field, _count in coverage.most_common(1)]
|
||||
detectors = {
|
||||
name: {"enabled": True, "minimum": 5 if name == "auth_failure" else 10, "z_threshold": 3.0}
|
||||
for name, count in detector_counts.items()
|
||||
if count > 0
|
||||
}
|
||||
if not detectors:
|
||||
detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
|
||||
high_coverage = [
|
||||
{"field": field, "coverage": round(count / max(1, total), 2), "unique_values": len(unique_values[field])}
|
||||
for field, count in coverage.most_common(12)
|
||||
]
|
||||
existing = existing_profiles.get(stream_id)
|
||||
score = min(100, 30 + len(entity_fields) * 15 + min(20, len(categorical) * 3) + min(15, len(detectors) * 5))
|
||||
suggestions.append({
|
||||
"stream_id": stream_id,
|
||||
"stream_name": names.get(stream_id, stream_id),
|
||||
"events": total,
|
||||
"profile_exists": bool(existing),
|
||||
"confidence": "high" if score >= 75 and total >= 100 else "medium" if score >= 55 and total >= 20 else "low",
|
||||
"score": score,
|
||||
"entity_fields": entity_fields,
|
||||
"timestamp_field": timestamp,
|
||||
"categorical_fields": categorical,
|
||||
"numeric_fields": numeric,
|
||||
"detectors": detectors,
|
||||
"common_fields": high_coverage,
|
||||
"profile": {
|
||||
"stream_id": stream_id,
|
||||
"name": f"{names.get(stream_id, stream_id)} recommended profile",
|
||||
"entity_field": entity_fields[0],
|
||||
"entity_fields": entity_fields,
|
||||
"timestamp_field": timestamp,
|
||||
"categorical_fields": categorical,
|
||||
"numeric_fields": numeric,
|
||||
"detectors": detectors,
|
||||
"field_weights": {},
|
||||
},
|
||||
"reason": "selected common entity, time, categorical, and numeric fields from observed events",
|
||||
})
|
||||
return sorted(suggestions, key=lambda item: (bool(item["profile_exists"]), -int(item["score"]), str(item["stream_name"])))
|
||||
Reference in New Issue
Block a user