Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configurable per-field detector weights är klart.

Browse Source
This commit is contained in:
larssand
2026-06-25 19:36:08 +02:00
parent 899dbdfc84
commit 9da1aedf52
9 changed files with 106 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/fgai/baseline.py
View File

@@ -42,6 +42,26 @@ def _baseline_confidence(samples: int, *, temporal: bool) -> str:
return "low"
def _weighted_score(base: int, profile: object | None, field: str, detector: str) -> tuple[int, float]:
weights = getattr(profile, "field_weights", {}) if profile else {}
if not isinstance(weights, dict):
return base, 1.0
field_key = str(field).lower()
detector_key = str(detector).lower()
multiplier = 1.0
for key in (field_key, detector_key):
value = weights.get(key)
if isinstance(value, (int, float)):
multiplier *= float(value)
nested = weights.get(field_key)
if isinstance(nested, dict):
value = nested.get(detector_key)
if isinstance(value, (int, float)):
multiplier *= float(value)
multiplier = max(0.0, min(5.0, multiplier))
return min(100, max(0, int(round(base * multiplier)))), round(multiplier, 2)
class BaselineStore:
"""Persistent five-minute behavior baseline, implemented with stdlib SQLite."""
@@ -211,6 +231,7 @@ class BaselineStore:
output: dict[str, list[dict[str, object]]] = defaultdict(list)
with self._connect() as connection:
for (stream, entity, field), values in current.items():
profile = profiles.get(stream)
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
@@ -233,8 +254,9 @@ class BaselineStore:
sample_values = sorted({event.fields.get(field, "") for event in matching})[:5]
evidence_events = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": event.fields.get(field, ""), "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
score, weight = _weighted_score(base_score, profile, field, "numeric_baseline")
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
# Event-rate burst is calculated once per stream/entity, rather than once per selected field.
for (stream, entity), matching in entity_events.items():
@@ -260,9 +282,10 @@ class BaselineStore:
if z_score < 3:
continue
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
base_score = min(30, (15 if confidence == "high" else 12 if confidence == "medium" else 8) + int(z_score))
score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst")
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or "", "action": event.action, "severity": event.severity, "service": event.fields.get("service", ""), "value": "", "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
for (stream, entity, detector), current_value in detector_current.items():
profile = profiles.get(stream)
@@ -290,9 +313,10 @@ class BaselineStore:
if z_score < z_threshold:
continue
confidence = _baseline_confidence(len(rows), temporal=temporal)
score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score))
base_score = min(35, (18 if detector == "auth_failure" else 15 if detector == "deny_action" else 12) + int(z_score))
score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst")
samples = [{"timestamp": event.fields.get("eventtime", event.fields.get("timestamp", "")), "source": event.src_ip or event.fields.get("source", ""), "destination": event.dst_ip or event.fields.get("query_domain", ""), "action": event.action, "severity": event.severity, "service": event.fields.get("service", event.fields.get("query_type", "")), "value": detector, "message": event.fields.get("message", event.fields.get("msg", ""))[:240]} for event in matching[:5]]
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
# Detect selected categorical values that have not appeared for this entity in prior data.
for event in events:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
@@ -311,7 +335,8 @@ class BaselineStore:
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 30:
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": 12, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
score, weight = _weighted_score(12, profile, field, "rare_value")
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [{"timestamp": item.fields.get("eventtime", item.fields.get("timestamp", "")), "source": item.src_ip or item.fields.get("source", ""), "destination": item.dst_ip or "", "action": item.action, "severity": item.severity, "service": item.fields.get("service", ""), "value": value, "message": item.fields.get("message", item.fields.get("msg", ""))[:240]} for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
return output

7
src/fgai/dashboard.py
View File

@@ -80,7 +80,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main>
<script>
function esc(value) {
@@ -187,7 +187,7 @@ async function refresh() {
const fieldRows = rawFieldRows.length ? rawFieldRows : uiCache.fieldRows;
if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Confidence', key:'confidence'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; samples ${r.baseline_samples ?? '-'}; scope ${r.baseline_scope ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Confidence', key:'confidence'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; samples ${r.baseline_samples ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
], 'field-deviations');
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Review note (optional):') || '';
@@ -263,6 +263,7 @@ document.getElementById('loadFields').addEventListener('click', async () => {
const profile = (window.streamProfiles || []).find(item => item.stream_id === selected.dataset.id) || {};
document.querySelector('[name="profile_name"]').value = profile.name || `${selected.dataset.title} profile`;
document.querySelector('[name="profile_detectors"]').value = Object.keys(profile.detectors || {}).length ? JSON.stringify(profile.detectors, null, 2) : '';
document.querySelector('[name="profile_field_weights"]').value = Object.keys(profile.field_weights || {}).length ? JSON.stringify(profile.field_weights, null, 2) : '';
const rows = (payload.fields || []).map(field => { const name=field.name||field.field, type=(field.type||{}).type||'', props=(field.type||{}).properties||[]; return `<div class="field-row"><code>${esc(name)}</code><span>${esc(type)}</span><span>${esc(props.join(', '))}</span><div class="field-controls"><label><input type="radio" name="profile_entity" value="${esc(name)}" ${profile.entity_field===name?'checked':''}> Entity</label><label><input type="radio" name="profile_timestamp" value="${esc(name)}" ${profile.timestamp_field===name?'checked':''}> Time</label>${props.includes('enumerable')?`<label><input class="profile-categorical" type="checkbox" value="${esc(name)}" ${(profile.categorical_fields||[]).includes(name)?'checked':''}> Categorical</label>`:''}${props.includes('numeric')?`<label><input class="profile-numeric" type="checkbox" value="${esc(name)}" ${(profile.numeric_fields||[]).includes(name)?'checked':''}> Numeric</label>`:''}</div></div>`; }).join('');
document.getElementById('fieldPicker').innerHTML = rows ? `<div class="field-header"><span>Field</span><span>Type</span><span>Capabilities</span><span>Use In Profile</span></div>${rows}` : esc(payload.error || 'No fields found.');
window.activeProfileStream = selected.dataset.id;
@@ -274,7 +275,7 @@ document.getElementById('settingsForm').addEventListener('submit', async event =
values.llm_enabled = form.elements.llm_enabled.checked;
values.threat_intel_enabled = form.elements.threat_intel_enabled.checked;
values.graylog_streams = [...document.querySelectorAll('.graylog-stream')].map(item => ({id:item.dataset.id, title:item.dataset.title, enabled:item.checked}));
if (window.activeProfileStream) { let detectors={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } const selectedStream=document.querySelector('.graylog-stream:checked'); const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${selectedStream?.dataset.title || window.activeProfileStream} profile`,entity_field:form.querySelector('[name="profile_entity"]:checked')?.value||'',timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const selectedStream=document.querySelector('.graylog-stream:checked'); const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${selectedStream?.dataset.title || window.activeProfileStream} profile`,entity_field:form.querySelector('[name="profile_entity"]:checked')?.value||'',timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(values)});
document.getElementById('settingsResult').textContent = response.ok ? 'Saved. Monitor applies supported settings on its next cycle.' : 'Could not save configuration.';
if (response.ok) loadSettings();

2
src/fgai/monitor.py
View File

@@ -179,7 +179,7 @@ def build_status(
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
"configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors} for item in stream_profiles.values()],
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
"profile_readiness": profile_readiness,
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),

25
src/fgai/stream_profiles.py
View File

@@ -12,6 +12,7 @@ class StreamProfile:
categorical_fields: tuple[str, ...] = ()
numeric_fields: tuple[str, ...] = ()
detectors: dict[str, dict[str, object]] = field(default_factory=dict)
field_weights: dict[str, object] = field(default_factory=dict)
def _detectors(value: object) -> dict[str, dict[str, object]]:
@@ -30,6 +31,28 @@ def _detectors(value: object) -> dict[str, dict[str, object]]:
return output
def _field_weights(value: object) -> dict[str, object]:
if not isinstance(value, dict):
return {}
output: dict[str, object] = {}
for key, setting in value.items():
if isinstance(setting, dict):
nested: dict[str, float] = {}
for nested_key, nested_value in setting.items():
try:
nested[str(nested_key).lower()] = max(0.0, min(5.0, float(nested_value)))
except (TypeError, ValueError):
continue
if nested:
output[str(key).lower()] = nested
continue
try:
output[str(key).lower()] = max(0.0, min(5.0, float(setting)))
except (TypeError, ValueError):
continue
return output
def parse_profiles(value: object) -> dict[str, StreamProfile]:
profiles: dict[str, StreamProfile] = {}
for item in value if isinstance(value, list) else []:
@@ -41,10 +64,12 @@ def parse_profiles(value: object) -> dict[str, StreamProfile]:
if not stream_id or not entity:
continue
detectors = _detectors(item.get("detectors", {}))
field_weights = _field_weights(item.get("field_weights", {}))
profiles[stream_id] = StreamProfile(
stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp,
tuple(str(field) for field in item.get("categorical_fields", []) if field),
tuple(str(field) for field in item.get("numeric_fields", []) if field),
detectors,
field_weights,
)
return profiles