Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Field baseline deviations now show distinct Mark expected, Mark false positive, and Mark confirmed

Browse Source
This commit is contained in:
larssand
2026-06-24 18:54:37 +02:00
parent 63449c9e80
commit f6bee0438c
6 changed files with 110 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/fgai/dashboard.py
View File

@@ -55,6 +55,10 @@ HTML = """<!doctype html>
.field-row code { width: fit-content; }
.field-controls { display: flex; flex-wrap: wrap; gap: 10px; }
.field-controls label { white-space: nowrap; }
.review-actions { display: flex; flex-wrap: wrap; gap: 6px; min-width: 250px; }
.review-actions button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 6px 8px; cursor: pointer; }
.review-actions button[data-status="false_positive"] { border-color: #b7823a; color: #ffd36e; }
.review-actions button[data-status="confirmed"] { border-color: #2a9b6e; color: #7be3ae; }
.chart { width: 100%; height: 220px; background: #04182d; border: 1px solid #163b59; }
@media (max-width: 860px) { .hero, .split { grid-template-columns: 1fr; } .hero img { display: none; } }
</style>
@@ -71,7 +75,7 @@ HTML = """<!doctype html>
</section>
<nav class="tabs" aria-label="Dashboard views"><button class="tab active" data-tab="overview">Overview</button><button class="tab" data-tab="findings">Findings</button><button class="tab" data-tab="diagnostics">Diagnostics</button><button class="tab" data-tab="settings">Settings</button></nav>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="findings"><section class="panel"><h2>Field Baseline Deviations</h2><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Graylog streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Discover fields<br><button type="button" id="loadFields">Load selected stream fields</button><div id="fieldPicker" class="muted">Select a stream first.</div></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main>
@@ -157,12 +161,13 @@ async function refresh() {
const streamTitles = Object.fromEntries((configuration.graylog_streams || []).map(item => [item.id, item.title || item.id]));
const fieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, stream_title: streamTitles[item.stream_id] || item.stream_id, ...item})));
document.getElementById('fieldDeviations').innerHTML = table(fieldRows, [
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Action', render:r => `<button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Expected</button> <button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">False positive</button> <button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Confirm</button>`}
{label:'Entity', key:'entity'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)).join('<br>'); return events ? `<details><summary>${summary}</summary><p>${events}</p></details>` : summary; }}, {label:'Review action', render:r => `<div class="review-actions"><button class="feedback" data-status="expected" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark expected</button><button class="feedback" data-status="false_positive" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark false positive</button><button class="feedback" data-status="confirmed" data-entity="${esc(r.entity)}" data-stream="${esc(r.stream_id)}" data-field="${esc(r.field)}">Mark confirmed</button></div>`}
]);
document.querySelectorAll('.feedback').forEach(button => button.addEventListener('click', async () => {
const note = prompt('Review note (optional):') || '';
const days = prompt('Expiry in days (0 = no expiry):', '0') || '0';
await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
const response = await fetch('/api/feedback', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({status:button.dataset.status, entity:button.dataset.entity, stream_id:button.dataset.stream, field:button.dataset.field, note, expires_at: Number(days) > 0 ? Math.floor(Date.now()/1000) + Number(days) * 86400 : 0})});
document.getElementById('feedbackNotice').textContent = response.ok ? 'Review saved. The matching pattern will be labeled on the next refresh.' : 'Could not save review.';
refresh();
}));
document.getElementById('relatedActivity').innerHTML = table(relatedRows, [

59
src/fgai/threat_intel.py
View File

@@ -19,20 +19,29 @@ def is_public_ip(value: str | None) -> bool:
class ThreatIntelClient:
def __init__(self, *, cache_file: str = "state/threat-intel-cache.json", ttl_seconds: int = 86400, enabled: bool | None = None) -> None:
def __init__(self, *, cache_file: str = "state/threat-intel-cache.json", ttl_seconds: int | None = None, enabled: bool | None = None) -> None:
self.enabled = os.getenv("FGAI_THREAT_INTEL", "").lower() in {"1", "true", "yes", "on"} if enabled is None else enabled
self.abuseipdb_key = os.getenv("ABUSEIPDB_API_KEY")
self.virustotal_key = os.getenv("VIRUSTOTAL_API_KEY")
self.provider = os.getenv("FGAI_THREAT_INTEL_PROVIDER", "auto").lower()
self.max_age_days = int(os.getenv("ABUSEIPDB_MAX_AGE_DAYS", "90"))
self.cache_path = Path(cache_file)
self.ttl_seconds = ttl_seconds
self.ttl_seconds = ttl_seconds if ttl_seconds is not None else int(os.getenv("FGAI_THREAT_INTEL_TTL_SECONDS", "604800"))
self.error_ttl_seconds = int(os.getenv("FGAI_THREAT_INTEL_ERROR_TTL_SECONDS", "3600"))
self.daily_limit = int(os.getenv("FGAI_THREAT_INTEL_DAILY_LIMIT", "100"))
self.cache = self._read_cache()
def status(self) -> dict[str, object]:
provider = self._select_provider()
has_key = bool(self.abuseipdb_key if provider == "abuseipdb" else self.virustotal_key)
return {"enabled": self.enabled, "provider": provider, "configured": has_key}
return {
"enabled": self.enabled,
"provider": provider,
"configured": has_key,
"cache_ttl_seconds": self.ttl_seconds,
"daily_limit": self.daily_limit,
"requests_today": self._requests_today(provider),
}
def _read_cache(self) -> dict[str, dict[str, object]]:
if not self.cache_path.exists():
@@ -46,6 +55,39 @@ class ThreatIntelClient:
self.cache_path.parent.mkdir(parents=True, exist_ok=True)
self.cache_path.write_text(json.dumps(self.cache, indent=2, sort_keys=True), encoding="utf-8")
def _cache_fresh(self, cached: dict[str, object], now: int) -> bool:
ttl = self.ttl_seconds if cached.get("status") == "ok" else self.error_ttl_seconds
return now - int(cached.get("cached_at", 0)) < ttl
def _today(self) -> str:
return time.strftime("%Y-%m-%d", time.gmtime())
def _requests_today(self, provider: str) -> int:
meta = self.cache.get("_meta", {})
if not isinstance(meta, dict):
return 0
requests_by_day = meta.get("requests_by_day", {})
if not isinstance(requests_by_day, dict):
return 0
provider_counts = requests_by_day.get(provider, {})
return int(provider_counts.get(self._today(), 0)) if isinstance(provider_counts, dict) else 0
def _record_request(self, provider: str) -> None:
meta = self.cache.setdefault("_meta", {})
if not isinstance(meta, dict):
meta = {}
self.cache["_meta"] = meta
requests_by_day = meta.setdefault("requests_by_day", {})
if not isinstance(requests_by_day, dict):
requests_by_day = {}
meta["requests_by_day"] = requests_by_day
provider_counts = requests_by_day.setdefault(provider, {})
if not isinstance(provider_counts, dict):
provider_counts = {}
requests_by_day[provider] = provider_counts
today = self._today()
provider_counts[today] = int(provider_counts.get(today, 0)) + 1
def lookup_ip(self, ip: str) -> dict[str, object]:
if not is_public_ip(ip):
return {"ip": ip, "provider": "local", "status": "skipped", "reason": "not_public_ip", "score": 0}
@@ -53,10 +95,18 @@ class ThreatIntelClient:
cache_key = f"{provider}:{ip}"
cached = self.cache.get(cache_key)
now = int(time.time())
if cached and now - int(cached.get("cached_at", 0)) < self.ttl_seconds:
if isinstance(cached, dict) and self._cache_fresh(cached, now):
return cached
if not self.enabled:
return {"ip": ip, "provider": "none", "status": "disabled", "score": 0}
if self.daily_limit > 0 and self._requests_today(provider) >= self.daily_limit:
return {
"ip": ip,
"provider": provider,
"status": "daily_limit_reached",
"score": 0,
"reason": "external lookup budget reached; cached results remain available",
}
if provider == "abuseipdb":
if not self.abuseipdb_key:
return {"ip": ip, "provider": "abuseipdb", "status": "missing_api_key", "score": 0}
@@ -68,6 +118,7 @@ class ThreatIntelClient:
else:
return {"ip": ip, "provider": provider, "status": "unsupported_provider", "score": 0}
self._record_request(provider)
result["cached_at"] = now
self.cache[cache_key] = result
self._write_cache()