Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented cleanup/triage direction. ui, and baselinbe days

Browse Source
This commit is contained in:
larssand
2026-06-29 20:20:00 +02:00
parent d764c5a038
commit d540b8a77d
9 changed files with 298 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/fgai/baseline.py
View File

@@ -18,6 +18,7 @@ from .query_details import event_query_details
DEFAULT_RETENTION_DAYS = 14
DEFAULT_VALUE_RETENTION_DAYS = 7
DEFAULT_MAX_VALUES_PER_FIELD = 2000
DEFAULT_TRAINING_DAYS = 7
MIN_REPORTED_DEVIATION_SCORE = 15
MAX_RARE_VALUES_PER_ENTITY = 5
@@ -50,6 +51,12 @@ def _baseline_confidence(samples: int, *, temporal: bool) -> str:
return "low"
def _age_days(oldest_bucket: int | None, newest_timestamp: int) -> float:
if not oldest_bucket:
return 0.0
return max(0.0, (newest_timestamp - int(oldest_bucket)) / 86400)
def _weighted_score(base: int, profile: object | None, field: str, detector: str) -> tuple[int, float]:
weights = getattr(profile, "field_weights", {}) if profile else {}
if not isinstance(weights, dict):
@@ -254,7 +261,7 @@ class BaselineStore:
on conflict(stream_id, entity, detector, weekday, hour, bucket_start) do update set events=events+excluded.events""", (stream_id, entity, detector, weekday, hour, bucket, count))
return len(pending)
def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object]) -> dict[str, list[dict[str, object]]]:
def profile_deviations(self, events: list[LogEvent], profiles: dict[str, object], *, min_training_days: int = 0) -> dict[str, list[dict[str, object]]]:
current: dict[tuple[str, str, str], list[float]] = defaultdict(lambda: [0, 0.0])
entity_events: dict[tuple[str, str], list[LogEvent]] = defaultdict(list)
detector_current: Counter[tuple[str, str, str]] = Counter()
@@ -291,6 +298,10 @@ class BaselineStore:
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
baseline_age_days = _age_days(oldest, current_timestamp)
if baseline_age_days < min_training_days:
continue
if values[1] == 0:
continue
history = [row[1] / row[0] if row[0] else 0 for row in rows]
@@ -304,7 +315,7 @@ class BaselineStore:
base_score = 18 if confidence == "high" else 15 if confidence == "medium" else 10
score, weight = _weighted_score(base_score, profile, field, "numeric_baseline")
if score >= MIN_REPORTED_DEVIATION_SCORE:
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
output[entity].append({"detector": "numeric_baseline", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": reason, "current": round(current_value, 2), "baseline": round(mean(history), 2), "sample_values": sample_values, "sample_events": evidence_events})
# Event-rate burst is calculated once per stream/entity, rather than once per selected field.
for (stream, entity), matching in entity_events.items():
@@ -324,6 +335,10 @@ class BaselineStore:
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, reference_field)).fetchone()[0]
baseline_age_days = _age_days(oldest, current_timestamp)
if baseline_age_days < min_training_days:
continue
history = [row[0] for row in rows]
current_value = len(matching)
z_score = (current_value - mean(history)) / (pstdev(history) or 1.0)
@@ -334,7 +349,7 @@ class BaselineStore:
score, weight = _weighted_score(base_score, profile, "event_rate", "event_rate_burst")
samples = [_sample_event(event) for event in matching[:5]]
if score >= MIN_REPORTED_DEVIATION_SCORE:
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
output[entity].append({"detector": "event_rate_burst", "field": "event_rate", "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"event rate burst above its {baseline_scope} baseline (z={z_score:.1f})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [], "sample_events": samples})
for (stream, entity, detector), current_value in detector_current.items():
profile = profiles.get(stream)
@@ -357,6 +372,10 @@ class BaselineStore:
baseline_scope = "all observed periods"
if len(rows) < 12:
continue
oldest = connection.execute("select min(bucket_start) from profile_detector_buckets where stream_id=? and entity=? and detector=?", (stream, entity, detector)).fetchone()[0]
baseline_age_days = _age_days(oldest, current_timestamp)
if baseline_age_days < min_training_days:
continue
history = [row[0] for row in rows]
z_score = (current_value - mean(history)) / (pstdev(history) or 1.0)
if z_score < z_threshold:
@@ -366,7 +385,7 @@ class BaselineStore:
score, weight = _weighted_score(base_score, profile, detector, f"{detector}_burst")
samples = [_sample_event(event, detector) for event in matching[:5]]
if score >= MIN_REPORTED_DEVIATION_SCORE:
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
output[entity].append({"detector": f"{detector}_burst", "field": detector, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": confidence, "baseline_samples": len(rows), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": baseline_scope, "reason": f"{detector.replace('_', ' ')} burst above its {baseline_scope} baseline (z={z_score:.1f}, minimum={minimum})", "current": current_value, "baseline": round(mean(history), 2), "sample_values": [detector], "sample_events": samples})
# Detect selected categorical values that have not appeared for this entity in prior data.
rare_counts: Counter[tuple[str, str]] = Counter()
for event in events:
@@ -386,6 +405,10 @@ class BaselineStore:
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 30:
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
baseline_age_days = _age_days(oldest, _event_epoch(event, int(time.time())))
if baseline_age_days < min_training_days:
continue
if rare_counts[(stream, entity)] >= MAX_RARE_VALUES_PER_ENTITY:
continue
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and entity in profile_entities(item, profile) and item.fields.get(field) == value]
@@ -393,7 +416,7 @@ class BaselineStore:
score, weight = _weighted_score(base_score, profile, field, "rare_value")
if score < MIN_REPORTED_DEVIATION_SCORE:
continue
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": base_score, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_age_days": round(baseline_age_days, 2), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
rare_counts[(stream, entity)] += 1
@@ -449,14 +472,16 @@ class BaselineStore:
connection.execute("vacuum")
return {"retention_days": retention_days, "value_retention_days": value_retention_days, "max_values_per_field": max_values_per_field, "vacuum": vacuum, "deleted": deleted, "size_bytes": self.path.stat().st_size if self.path.exists() else 0}
def profile_readiness(self, profiles: dict[str, object]) -> list[dict[str, object]]:
def profile_readiness(self, profiles: dict[str, object], *, min_training_days: int = 0) -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
with self._connect() as connection:
for stream_id, profile in profiles.items():
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
for field in fields:
count = connection.execute("select count(distinct bucket_start) from profile_buckets where stream_id=? and field=?", (stream_id, str(field).lower())).fetchone()[0]
rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "ready": count >= 12})
oldest = connection.execute("select min(bucket_start) from profile_buckets where stream_id=? and field=?", (stream_id, str(field).lower())).fetchone()[0]
age_days = _age_days(oldest, int(time.time()))
rows.append({"stream_id": stream_id, "field": str(field), "buckets": count, "age_days": round(age_days, 2), "training_days": min_training_days, "ready": count >= 12 and age_days >= min_training_days})
return rows
def profiles(self, source_ips: set[str]) -> dict[str, dict[str, object]]:

16
src/fgai/config.py
View File

@@ -16,10 +16,18 @@ DEFAULT_CONFIG: dict[str, object] = {
"baseline_retention_days": 14,
"baseline_value_retention_days": 7,
"baseline_max_values_per_field": 2000,
"baseline_training_days": 7,
"graylog_field_mapping": "",
"llm_enabled": False,
"llm_model": "",
"threat_intel_enabled": False,
"threat_intel_provider": "auto",
"abuseipdb_api_key": "",
"virustotal_api_key": "",
"threat_intel_daily_limit": 100,
"threat_intel_ttl_seconds": 604800,
"threat_intel_error_ttl_seconds": 3600,
"abuseipdb_max_age_days": 90,
}
EDITABLE_FIELDS = set(DEFAULT_CONFIG) | {"graylog_mcp_token"}
@@ -39,6 +47,8 @@ class ConfigStore:
def public(self) -> dict[str, object]:
config = self.read()
config["graylog_mcp_token_configured"] = bool(config.pop("graylog_mcp_token", ""))
config["abuseipdb_api_key_configured"] = bool(config.pop("abuseipdb_api_key", ""))
config["virustotal_api_key_configured"] = bool(config.pop("virustotal_api_key", ""))
return config
def update(self, values: dict[str, object]) -> dict[str, object]:
@@ -46,18 +56,20 @@ class ConfigStore:
for key, value in values.items():
if key not in EDITABLE_FIELDS:
continue
if key == "graylog_mcp_token" and value == "":
if key in {"graylog_mcp_token", "abuseipdb_api_key", "virustotal_api_key"} and value == "":
continue
if key in {"llm_enabled", "threat_intel_enabled"}:
current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value
elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field"}:
elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
try:
minimum = 60 if key == "graylog_range_seconds" else 1
current[key] = max(minimum, int(value))
except (TypeError, ValueError):
continue
elif key == "threat_intel_provider" and value in {"auto", "abuseipdb", "virustotal"}:
current[key] = value
elif key == "graylog_streams" and isinstance(value, list):
current[key] = [
{"id": str(item.get("id", "")), "title": str(item.get("title", "")), "enabled": bool(item.get("enabled"))}

39
src/fgai/dashboard.py
View File

@@ -60,6 +60,10 @@ HTML = """<!doctype html>
.field-controls label { white-space: nowrap; }
.stream-row { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 4px 0; }
.stream-row button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 4px 8px; cursor: pointer; }
.toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 8px 0 12px; color: #91abc4; }
.toolbar label { display: inline-flex; align-items: center; gap: 6px; }
.summary-card { border: 1px solid #163b59; background: #061a2e; padding: 10px; margin-bottom: 8px; border-radius: 6px; }
.summary-card h3 { margin: 0 0 6px; font-size: 16px; }
.review-actions { display: flex; flex-wrap: wrap; gap: 6px; min-width: 250px; }
.review-actions button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 6px 8px; cursor: pointer; }
.review-actions button[data-status="false_positive"] { border-color: #b7823a; color: #ffd36e; }
@@ -83,9 +87,9 @@ HTML = """<!doctype html>
</section>
<nav class="tabs" aria-label="Dashboard views"><button class="tab active" data-tab="overview">Overview</button><button class="tab" data-tab="findings">Findings</button><button class="tab" data-tab="diagnostics">Diagnostics</button><button class="tab" data-tab="settings">Settings</button></nav>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main>
<script>
function esc(value) {
@@ -148,6 +152,7 @@ async function refresh() {
`Critical anomalies: ${esc((a.critical || 0))}`,
`High anomalies: ${esc((a.high || 0))}`,
`Baseline sources ready: ${esc((data.baseline || {}).sources_ready || 0)}`,
`Baseline training days: ${esc((data.baseline || {}).training_days || 0)}`,
`Baseline DB size: ${esc(bytes((data.baseline || {}).size_bytes || 0))}`
].join('<br>');
document.getElementById('health').innerHTML = [
@@ -200,12 +205,31 @@ async function refresh() {
const reputationRows = Object.entries(data.reputation || {}).map(([ip, intel]) => ({ip, ...intel}));
const relatedRows = correlations.flatMap(correlation => (correlation.samples || []).map(sample => ({entity: correlation.entity || correlation.source_ip, source_ip: correlation.source_ip, ...sample})));
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
document.getElementById('triageQueue').innerHTML = table(data.triage_queue || [], [
{label:'Entity', key:'entity'},
{label:'Score', key:'score'},
{label:'Severity', render:r => `<span class="sev-${esc(r.severity)}">${esc(r.severity)}</span>`},
{label:'State', key:'state'},
{label:'Streams', render:r => esc((r.streams || []).join(', ') || '-')},
{label:'Detectors', render:r => esc((r.detectors || []).join(', ') || '-')},
{label:'Open signals', key:'active_deviation_count'},
{label:'Why', render:r => esc((r.evidence || []).join('; ') || '-')},
{label:'Next', key:'next_action'}
], 'triage-queue');
const rawFieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, ...item, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})));
const fieldRowsCached = rawFieldRows.length === 0 && uiCache.fieldRows.length > 0;
const fieldRows = rawFieldRows.length ? rawFieldRows : uiCache.fieldRows;
const sourceFieldRows = rawFieldRows.length ? rawFieldRows : uiCache.fieldRows;
const showReviewed = document.getElementById('showReviewedFindings')?.checked;
const showLow = document.getElementById('showLowFindings')?.checked;
const fieldRows = sourceFieldRows
.filter(row => showReviewed || !['expected','false_positive'].includes(row.feedback || ''))
.filter(row => showLow || Number(row.score || 0) >= 35 || row.feedback === 'confirmed')
.sort((left, right) => Number(right.score || 0) - Number(left.score || 0))
.slice(0, 50);
document.getElementById('findingSummary').textContent = `${fieldRows.length} shown from ${sourceFieldRows.length} raw deviations. Reviewed and low-score findings are hidden by default.`;
if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows;
document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '<p class="muted">No current field deviations in this poll; showing cached findings from the last non-empty poll.</p>' : ''}` + table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Confidence', key:'confidence'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; samples ${r.baseline_samples ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Mark confirmed</button></div>`}
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `<details data-detail-id="${esc(id)}"><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">False positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}" data-value="${esc(r.value || '')}">Confirm</button></div>`}
], 'field-deviations');
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Review note (optional):') || '';
@@ -242,7 +266,7 @@ async function refresh() {
'<h3>Stream Coverage</h3>' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Events', key:'events_fetched'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', key:'health'}], 'stream-coverage') +
'<h3>Cross-Source Correlations</h3>' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${r.entity || r.source_ip} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') +
'<h3>Entities</h3>' + table(context.source_profiles || [], [{label:'Entity', key:'entity'}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
'<h3>Profile Baseline Readiness</h3>' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}], 'profile-readiness') +
'<h3>Data Quality</h3>' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) +
'<h3>Security Event Samples</h3>' + table(context.security_event_samples || [], [{label:'Entity', key:'entity'}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) +
'<h3>Top Sources</h3>' + table(d.top_source_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) +
@@ -265,6 +289,10 @@ async function loadSettings() {
}
const token = form.elements.namedItem('graylog_mcp_token');
token.placeholder = config.graylog_mcp_token_configured ? 'Token configured; leave blank to keep it' : 'Paste a read-only token';
const abuseKey = form.elements.namedItem('abuseipdb_api_key');
abuseKey.placeholder = config.abuseipdb_api_key_configured ? 'Key configured; leave blank to keep it' : 'Paste AbuseIPDB API key';
const vtKey = form.elements.namedItem('virustotal_api_key');
vtKey.placeholder = config.virustotal_api_key_configured ? 'Key configured; leave blank to keep it' : 'Paste VirusTotal API key';
window.streamProfiles = config.graylog_stream_profiles || [];
if (config.graylog_mcp_token_configured && config.graylog_mcp_url) loadStreams();
}
@@ -318,6 +346,7 @@ document.querySelectorAll('.tab').forEach(button => button.addEventListener('cli
document.querySelectorAll('.tab').forEach(item => item.classList.toggle('active', item === button));
document.querySelectorAll('[data-view]').forEach(view => view.classList.toggle('active', view.dataset.view === button.dataset.tab));
}));
['showReviewedFindings','showLowFindings'].forEach(id => document.getElementById(id)?.addEventListener('change', refresh));
refresh();
loadSettings();
setInterval(refresh, 5000);

23
src/fgai/monitor.py
View File

@@ -22,6 +22,7 @@ from .policies import audit_policies, read_policies
from .recommendations import build_recommendations
from .sequences import detect_sequences
from .threat_intel import ThreatIntelClient, enrich_ips, is_public_ip
from .triage import build_triage_queue
from .stream_profiles import parse_profiles
@@ -147,7 +148,8 @@ def build_status(
events = []
baseline = BaselineStore(baseline_path) if baseline_path else None
profiles = baseline.profiles({event.src_ip for event in events if event.src_ip}) if baseline else {}
field_deviations = baseline.profile_deviations(events, stream_profiles) if baseline else {}
baseline_training_days = int(runtime_values.get("baseline_training_days", 7) or 7)
field_deviations = baseline.profile_deviations(events, stream_profiles, min_training_days=baseline_training_days) if baseline else {}
sequence_findings = detect_sequences(events)
for entity, findings in sequence_findings.items():
field_deviations.setdefault(entity, []).extend(findings)
@@ -189,7 +191,7 @@ def build_status(
if baseline
else {}
)
profile_readiness = baseline.profile_readiness(stream_profiles) if baseline else []
profile_readiness = baseline.profile_readiness(stream_profiles, min_training_days=baseline_training_days) if baseline else []
profile_readiness = [
{
**item,
@@ -208,11 +210,21 @@ def build_status(
}
)
threat_enabled = bool(runtime_values.get("threat_intel_enabled")) if runtime_values else None
reputation = enrich_ips(intel_ips, limit=25, enabled=threat_enabled)
threat_intel_status = ThreatIntelClient(enabled=threat_enabled).status()
reputation = enrich_ips(intel_ips, limit=25, enabled=threat_enabled, config=runtime_values)
threat_intel_status = ThreatIntelClient(
enabled=threat_enabled,
provider=str(runtime_values.get("threat_intel_provider", "auto")),
abuseipdb_key=str(runtime_values.get("abuseipdb_api_key", "") or "") or None,
virustotal_key=str(runtime_values.get("virustotal_api_key", "") or "") or None,
daily_limit=int(runtime_values.get("threat_intel_daily_limit", 100) or 100),
ttl_seconds=int(runtime_values.get("threat_intel_ttl_seconds", 604800) or 604800),
error_ttl_seconds=int(runtime_values.get("threat_intel_error_ttl_seconds", 3600) or 3600),
abuseipdb_max_age_days=int(runtime_values.get("abuseipdb_max_age_days", 90) or 90),
).status()
recommendations = build_recommendations(events, anomalies, reputation)
correlations = correlate_source_ips(events)
incidents = IncidentStore(incident_path or "state/signalscope-incidents.json").apply(build_incidents(anomalies, field_deviations, correlations))
triage_queue = build_triage_queue(incidents, field_deviations, correlations, recommendations)
block_candidates = suggest_block_candidates(
events,
min_events=min_block_events,
@@ -234,7 +246,7 @@ def build_status(
"policy_path": policy_path,
"summary": summarize_events(events),
"anomaly_summary": anomaly_summary(anomalies),
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
"configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
@@ -253,6 +265,7 @@ def build_status(
},
"event_context": build_event_context(events),
"field_deviations": field_deviations,
"triage_queue": triage_queue,
"sequence_findings": sequence_findings,
"feedback": feedback,
"cross_source_correlations": correlations,

41
src/fgai/threat_intel.py
View File

@@ -19,16 +19,28 @@ def is_public_ip(value: str | None) -> bool:
class ThreatIntelClient:
def __init__(self, *, cache_file: str = "state/threat-intel-cache.json", ttl_seconds: int | None = None, enabled: bool | None = None) -> None:
def __init__(
self,
*,
cache_file: str = "state/threat-intel-cache.json",
ttl_seconds: int | None = None,
enabled: bool | None = None,
provider: str | None = None,
abuseipdb_key: str | None = None,
virustotal_key: str | None = None,
daily_limit: int | None = None,
error_ttl_seconds: int | None = None,
abuseipdb_max_age_days: int | None = None,
) -> None:
self.enabled = os.getenv("FGAI_THREAT_INTEL", "").lower() in {"1", "true", "yes", "on"} if enabled is None else enabled
self.abuseipdb_key = os.getenv("ABUSEIPDB_API_KEY")
self.virustotal_key = os.getenv("VIRUSTOTAL_API_KEY")
self.provider = os.getenv("FGAI_THREAT_INTEL_PROVIDER", "auto").lower()
self.max_age_days = int(os.getenv("ABUSEIPDB_MAX_AGE_DAYS", "90"))
self.abuseipdb_key = abuseipdb_key if abuseipdb_key is not None else os.getenv("ABUSEIPDB_API_KEY")
self.virustotal_key = virustotal_key if virustotal_key is not None else os.getenv("VIRUSTOTAL_API_KEY")
self.provider = (provider if provider is not None else os.getenv("FGAI_THREAT_INTEL_PROVIDER", "auto")).lower()
self.max_age_days = abuseipdb_max_age_days if abuseipdb_max_age_days is not None else int(os.getenv("ABUSEIPDB_MAX_AGE_DAYS", "90"))
self.cache_path = Path(cache_file)
self.ttl_seconds = ttl_seconds if ttl_seconds is not None else int(os.getenv("FGAI_THREAT_INTEL_TTL_SECONDS", "604800"))
self.error_ttl_seconds = int(os.getenv("FGAI_THREAT_INTEL_ERROR_TTL_SECONDS", "3600"))
self.daily_limit = int(os.getenv("FGAI_THREAT_INTEL_DAILY_LIMIT", "100"))
self.error_ttl_seconds = error_ttl_seconds if error_ttl_seconds is not None else int(os.getenv("FGAI_THREAT_INTEL_ERROR_TTL_SECONDS", "3600"))
self.daily_limit = daily_limit if daily_limit is not None else int(os.getenv("FGAI_THREAT_INTEL_DAILY_LIMIT", "100"))
self.cache = self._read_cache()
def status(self) -> dict[str, object]:
@@ -196,9 +208,20 @@ class ThreatIntelClient:
def enrich_ips(
ips: list[str], *, cache_file: str = "state/threat-intel-cache.json", limit: int = 25, enabled: bool | None = None
ips: list[str], *, cache_file: str = "state/threat-intel-cache.json", limit: int = 25, enabled: bool | None = None, config: dict[str, object] | None = None
) -> dict[str, dict[str, object]]:
client = ThreatIntelClient(cache_file=cache_file, enabled=enabled)
config = config or {}
client = ThreatIntelClient(
cache_file=cache_file,
enabled=enabled,
provider=str(config.get("threat_intel_provider", "auto")),
abuseipdb_key=str(config.get("abuseipdb_api_key", "") or "") or None,
virustotal_key=str(config.get("virustotal_api_key", "") or "") or None,
daily_limit=int(config.get("threat_intel_daily_limit", 100) or 100),
ttl_seconds=int(config.get("threat_intel_ttl_seconds", 604800) or 604800),
error_ttl_seconds=int(config.get("threat_intel_error_ttl_seconds", 3600) or 3600),
abuseipdb_max_age_days=int(config.get("abuseipdb_max_age_days", 90) or 90),
)
enriched: dict[str, dict[str, object]] = {}
for ip in ips[:limit]:
enriched[ip] = client.lookup_ip(ip)

112
src/fgai/triage.py Normal file
View File

@@ -0,0 +1,112 @@
from __future__ import annotations
from collections import Counter, defaultdict
def _severity(score: int) -> str:
return "critical" if score >= 85 else "high" if score >= 60 else "medium" if score >= 35 else "low"
def build_triage_queue(
incidents: list[dict[str, object]],
field_deviations: dict[str, list[dict[str, object]]],
correlations: list[dict[str, object]],
recommendations: list[object],
*,
limit: int = 25,
) -> list[dict[str, object]]:
"""Summarize raw findings into analyst-sized investigation candidates."""
groups: dict[str, dict[str, object]] = defaultdict(lambda: {
"entity": "",
"score": 0,
"streams": set(),
"detectors": Counter(),
"evidence": [],
"deviations": 0,
"reviewed": Counter(),
"correlated_streams": 0,
"recommendations": [],
"incident_id": "",
"state": "open",
})
for incident in incidents:
entity = str(incident.get("entity", ""))
if not entity:
continue
group = groups[entity]
group["entity"] = entity
group["score"] = max(int(group["score"]), int(incident.get("score", 0) or 0))
group["incident_id"] = str(incident.get("id", ""))
group["state"] = str(incident.get("lifecycle_status", "open") or "open")
group["streams"].update(str(item) for item in incident.get("correlated_streams", []) if item)
group["evidence"].extend(str(item) for item in incident.get("evidence", []) if item)
for entity, deviations in field_deviations.items():
group = groups[str(entity)]
group["entity"] = str(entity)
active_scores = []
for item in deviations:
status = str(item.get("feedback", "unreviewed") or "unreviewed")
group["reviewed"][status] += 1
group["deviations"] = int(group["deviations"]) + 1
group["streams"].add(str(item.get("stream_name") or item.get("stream_title") or item.get("stream_id", "")))
group["detectors"][str(item.get("detector", "unknown"))] += 1
if status in {"expected", "false_positive"}:
continue
score = int(item.get("score", 0) or 0)
active_scores.append(score)
group["evidence"].append(str(item.get("reason", "")))
if active_scores:
group["score"] = max(int(group["score"]), max(active_scores))
for correlation in correlations:
entity = str(correlation.get("entity") or correlation.get("source_ip") or "")
if not entity:
continue
group = groups[entity]
group["entity"] = entity
streams = [str(item) for item in correlation.get("streams", []) if item]
group["streams"].update(streams)
group["correlated_streams"] = max(int(group["correlated_streams"]), len(streams))
if len(streams) > 1:
group["score"] = max(int(group["score"]), min(100, 25 + len(streams) * 8 + int(correlation.get("security_events", 0) or 0) * 2))
group["evidence"].append(f"activity observed across {len(streams)} streams")
for recommendation in recommendations:
subject = str(getattr(recommendation, "subject", ""))
if not subject:
continue
group = groups[subject]
group["entity"] = subject
group["score"] = max(int(group["score"]), int(getattr(recommendation, "score", 0) or 0))
group["recommendations"].append(str(getattr(recommendation, "title", "")))
queue = []
for group in groups.values():
score = int(group["score"])
if score <= 0:
continue
reviewed = group["reviewed"]
active = int(group["deviations"]) - int(reviewed.get("expected", 0)) - int(reviewed.get("false_positive", 0))
if active <= 0 and score < 60:
continue
detectors = group["detectors"].most_common(3)
evidence = list(dict.fromkeys(str(item) for item in group["evidence"] if item))[:4]
next_action = "confirm or suppress expected behavior"
if int(group["correlated_streams"]) > 1:
next_action = "investigate cross-source activity"
if reviewed.get("confirmed"):
next_action = "follow confirmed incident response"
queue.append({
"entity": group["entity"],
"score": score,
"severity": _severity(score),
"state": group["state"],
"incident_id": group["incident_id"],
"streams": sorted(item for item in group["streams"] if item),
"detectors": [name for name, _count in detectors],
"deviation_count": int(group["deviations"]),
"active_deviation_count": active,
"reviewed": dict(reviewed),
"evidence": evidence,
"recommendations": [item for item in group["recommendations"] if item][:3],
"next_action": next_action,
})
return sorted(queue, key=lambda item: (int(item["score"]), int(item["active_deviation_count"])), reverse=True)[:limit]