Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved the overview graphs. and MCP fetching poll

Browse Source
This commit is contained in:
larssand
2026-06-30 09:40:20 +02:00
parent 9c6c08b5ec
commit 31ac3c25d2
7 changed files with 91 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -202,8 +202,12 @@ Graylog under `System -> Configurations -> MCP` and use stream IDs internally;
the fgAI stream picker resolves titles in the UI. the fgAI stream picker resolves titles in the UI.
For live monitoring, `graylog_range_seconds` controls how far back each MCP poll For live monitoring, `graylog_range_seconds` controls how far back each MCP poll
searches. The default is one hour, which keeps findings and correlations more searches. The default is 300 seconds, so each poll re-checks the last five
stable than a very short window while still limiting MCP query cost. minutes. `graylog_max_events_per_stream` caps raw events fetched from each
stream during that window. If a stream hits the cap, the dashboard marks the
window as truncated because high EPS means SignalScope sampled only part of the
Graylog result set. For very high-volume streams, prefer aggregate baselines and
targeted drill-down queries over trying to pull every raw event through MCP.
## Monitoring Export ## Monitoring Export

5
src/fgai/config.py
View File

@@ -12,7 +12,8 @@ DEFAULT_CONFIG: dict[str, object] = {
"graylog_streams": [], "graylog_streams": [],
"graylog_stream_profiles": [], "graylog_stream_profiles": [],
"graylog_query": "*", "graylog_query": "*",
"graylog_range_seconds": 3600, "graylog_range_seconds": 300,
"graylog_max_events_per_stream": 5000,
"baseline_retention_days": 14, "baseline_retention_days": 14,
"baseline_value_retention_days": 7, "baseline_value_retention_days": 7,
"baseline_max_values_per_field": 2000, "baseline_max_values_per_field": 2000,
@@ -65,7 +66,7 @@ class ConfigStore:
current[key] = bool(value) current[key] = bool(value)
elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}: elif key == "log_source" and value in {"local_syslog", "graylog_mcp"}:
current[key] = value current[key] = value
elif key in {"graylog_range_seconds", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}: elif key in {"graylog_range_seconds", "graylog_max_events_per_stream", "baseline_retention_days", "baseline_value_retention_days", "baseline_max_values_per_field", "baseline_training_days", "profile_advisor_timeout", "threat_intel_daily_limit", "threat_intel_ttl_seconds", "threat_intel_error_ttl_seconds", "abuseipdb_max_age_days"}:
try: try:
minimum = 60 if key == "graylog_range_seconds" else 1 minimum = 60 if key == "graylog_range_seconds" else 1
current[key] = max(minimum, int(value)) current[key] = max(minimum, int(value))

56
src/fgai/dashboard.py
View File

@@ -93,8 +93,8 @@ HTML = """<!doctype html>
.review-actions button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 6px 8px; cursor: pointer; } .review-actions button { border: 1px solid #39709a; background: #0b2944; color: #d9e8f7; padding: 6px 8px; cursor: pointer; }
.review-actions button[data-status="false_positive"] { border-color: #b7823a; color: #ffd36e; } .review-actions button[data-status="false_positive"] { border-color: #b7823a; color: #ffd36e; }
.review-actions button[data-status="confirmed"] { border-color: #2a9b6e; color: #7be3ae; } .review-actions button[data-status="confirmed"] { border-color: #2a9b6e; color: #7be3ae; }
.chart { width: 100%; height: 220px; background: #04182d; border: 1px solid #163b59; } .chart { width: 100%; height: 280px; background: #04182d; border: 1px solid #163b59; }
.graph { width: 100%; height: 300px; background: #04182d; border: 1px solid #163b59; } .graph { width: 100%; height: 360px; background: #04182d; border: 1px solid #163b59; }
.sort-button { border: 0; background: transparent; color: #83bce9; cursor: pointer; font: inherit; font-weight: 600; padding: 0; } .sort-button { border: 0; background: transparent; color: #83bce9; cursor: pointer; font: inherit; font-weight: 600; padding: 0; }
.sort-button:hover { color: #d9e8f7; } .sort-button:hover { color: #d9e8f7; }
.model-list { display: flex; flex-wrap: wrap; gap: 8px; } .model-list { display: flex; flex-wrap: wrap; gap: 8px; }
@@ -116,7 +116,7 @@ HTML = """<!doctype html>
<div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div> <div data-view="overview" class="active"><section class="split"><div class="panel"><h2>Events and Anomalies</h2><canvas id="trendChart" class="chart"></canvas></div><div class="panel"><h2>Baseline and Stream Health</h2><div id="health"></div></div></section><section class="split"><div class="panel"><h2>Correlation Map</h2><canvas id="correlationGraph" class="graph"></canvas><div id="correlationGraphInfo" class="muted"></div></div><div class="panel"><h2>AI Assessment</h2><div id="llmAssessment" class="muted">LLM assessment disabled.</div></div></section><section class="panel"><h2>Investigation Incidents</h2><div id="incidents"></div></section><section class="split"><div class="panel"><h2>Anomalies</h2><div id="anomalies"></div></div><div class="panel"><h2>Recommendations</h2><div id="recommendations"></div></div></section></div>
<div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div> <div data-view="findings"><section class="panel"><h2>Triage Queue</h2><div id="triageQueue"></div></section><section class="panel"><h2>Field Baseline Deviations</h2><div class="toolbar"><label><input id="showReviewedFindings" type="checkbox"> show reviewed</label><label><input id="showLowFindings" type="checkbox"> show low score</label><span id="findingSummary"></span></div><div id="feedbackNotice" class="muted" role="status"></div><div id="fieldDeviations"></div></section><section class="panel"><h2>Related Activity Across Sources</h2><div id="relatedActivity"></div></section><section class="split"><div class="panel"><h2>Block Candidates</h2><div id="blocks"></div></div><div class="panel"><h2>Threat Intelligence</h2><div id="reputation"></div></div></section><section class="panel"><h2>Policy Findings</h2><div id="policies"></div></section></div>
<div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div> <div data-view="diagnostics"><section class="panel"><h2>Diagnostics</h2><div id="diagnostics"></div></section></div>
<div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Installed Ollama Models</h2><div id="ollamaModels" class="muted">Loading local Ollama models.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog analysis window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div> <div data-view="settings"><section class="panel"><h2>Recommended Stream Profiles</h2><div id="profileSuggestions" class="muted">Waiting for observed stream data.</div></section><section class="panel"><h2>Installed Ollama Models</h2><div id="ollamaModels" class="muted">Loading local Ollama models.</div></section><section class="panel"><h2>Runtime Configuration</h2><form id="settingsForm"><div class="grid"><label>Log source<br><select name="log_source"><option value="local_syslog">Local syslog</option><option value="graylog_mcp">Graylog MCP</option></select></label><label>Graylog MCP URL<br><input name="graylog_mcp_url" type="url" placeholder="https://graylog.example/api/mcp"></label><label>Enabled streams<br><button type="button" id="loadStreams">Load streams</button><div id="streamPicker" class="muted">Load streams after URL and token are saved.</div></label><label>Profile editor<br><button type="button" id="loadFields">Edit first checked stream profile</button><div id="profileEditorStatus" class="muted">No profile selected for editing.</div><div id="fieldPicker" class="muted">Click Edit profile on one stream.</div></label><label>Profile name<br><input name="profile_name" placeholder="Example: Windows login behavior"></label><label>Detector thresholds (JSON)<br><textarea name="profile_detectors" placeholder='{"auth_failure":{"enabled":true,"minimum":5,"z_threshold":3}}'></textarea></label><label>Field weights (JSON)<br><textarea name="profile_field_weights" placeholder='{"url":1.5,"auth_failure_burst":2,"query_domain":{"rare_value":1.8}}'></textarea></label><label>Graylog query<br><input name="graylog_query" placeholder="*"></label><label>Graylog MCP poll window seconds<br><input name="graylog_range_seconds" type="number" min="60" step="60" placeholder="300"></label><label>Graylog max events per stream<br><input name="graylog_max_events_per_stream" type="number" min="1" step="1000" placeholder="5000"></label><label>Baseline training days<br><input name="baseline_training_days" type="number" min="1" step="1" placeholder="7"></label><label>Baseline bucket retention days<br><input name="baseline_retention_days" type="number" min="1" step="1" placeholder="14"></label><label>Baseline value retention days<br><input name="baseline_value_retention_days" type="number" min="1" step="1" placeholder="7"></label><label>Max values per entity field<br><input name="baseline_max_values_per_field" type="number" min="1" step="100" placeholder="2000"></label><label>Threat intel provider<br><select name="threat_intel_provider"><option value="auto">Auto</option><option value="abuseipdb">AbuseIPDB</option><option value="virustotal">VirusTotal</option></select></label><label>AbuseIPDB API key<br><input name="abuseipdb_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>VirusTotal API key<br><input name="virustotal_api_key" type="password" placeholder="Leave blank to keep current key"></label><label>Threat intel daily limit<br><input name="threat_intel_daily_limit" type="number" min="1" step="1" placeholder="100"></label><label>Threat intel cache TTL seconds<br><input name="threat_intel_ttl_seconds" type="number" min="60" step="60" placeholder="604800"></label><label>Threat intel error TTL seconds<br><input name="threat_intel_error_ttl_seconds" type="number" min="60" step="60" placeholder="3600"></label><label>AbuseIPDB max age days<br><input name="abuseipdb_max_age_days" type="number" min="1" step="1" placeholder="90"></label><label>Graylog field mapping (JSON)<br><textarea name="graylog_field_mapping" placeholder='{"srcip":"client_ip","dstip":"server_ip","action":"event_action"}'></textarea></label><label>Graylog MCP token<br><input name="graylog_mcp_token" type="password" placeholder="Leave blank to keep current token"></label><label>Ollama model<br><input name="llm_model" placeholder="llama3.1"></label><label>Profile advisor model<br><input name="profile_advisor_model" placeholder="qwen3:8b"></label><label>Profile advisor timeout seconds<br><input name="profile_advisor_timeout" type="number" min="1" step="1" placeholder="120"></label><label><input name="llm_enabled" type="checkbox"> Enable Ollama analysis</label><label><input name="profile_advisor_enabled" type="checkbox"> Enable Ollama profile advisor</label><label><input name="threat_intel_enabled" type="checkbox"> Enable threat intelligence</label></div><p><button type="submit">Save configuration</button> <span id="settingsResult" class="muted"></span></p></form></section></div>
</main> </main>
<script> <script>
function esc(value) { function esc(value) {
@@ -154,8 +154,39 @@ function compactRelatedActivity(rows) {
} }
return [...groups.values()].sort((left,right) => (right.security-left.security) || (right.count-left.count)).slice(0,25); return [...groups.values()].sort((left,right) => (right.security-left.security) || (right.count-left.count)).slice(0,25);
} }
function drawTrend(history) { const canvas=document.getElementById('trendChart'), ctx=canvas.getContext('2d'), ratio=window.devicePixelRatio||1, cw=canvas.clientWidth,ch=canvas.clientHeight; canvas.width=cw*ratio; canvas.height=ch*ratio; ctx.scale(ratio,ratio); ctx.clearRect(0,0,cw,ch); if (!history.length) { ctx.fillStyle='#91abc4'; ctx.font='14px Arial'; ctx.fillText('Waiting for monitor history.', 16, 28); return; } const eventsMax=Math.max(1,...history.map(item=>Number(item.events)||0)), anomaliesMax=Math.max(1,...history.map(item=>Number(item.anomalies)||0)), left=40,right=40,top=28,bottom=24; ctx.strokeStyle='#163b59'; ctx.lineWidth=1; for(let index=0;index<4;index++){const y=top+index*(ch-top-bottom)/3;ctx.beginPath();ctx.moveTo(left,y);ctx.lineTo(cw-right,y);ctx.stroke();} const line=(key,max,color)=>{ctx.strokeStyle=color;ctx.lineWidth=2;ctx.beginPath();history.forEach((item,index)=>{const x=left+index*(cw-left-right)/Math.max(1,history.length-1);const y=ch-bottom-((Number(item[key])||0)/max)*(ch-top-bottom);index?ctx.lineTo(x,y):ctx.moveTo(x,y)});ctx.stroke();}; line('events',eventsMax,'#1ea9ff');line('anomalies',anomaliesMax,'#ff5656'); ctx.font='11px Arial';ctx.fillStyle='#1ea9ff';ctx.fillText(`Events max ${eventsMax}`,4,14);ctx.fillStyle='#ff5656';ctx.textAlign='right';ctx.fillText(`Anomalies max ${anomaliesMax}`,cw-4,14);ctx.textAlign='left'; } function drawTrend(history) {
function drawCorrelationGraph(correlations) { const canvas=document.getElementById('correlationGraph'), ctx=canvas.getContext('2d'), ratio=window.devicePixelRatio||1, cw=canvas.clientWidth,ch=canvas.clientHeight; canvas.width=cw*ratio; canvas.height=ch*ratio; ctx.scale(ratio,ratio); ctx.clearRect(0,0,cw,ch); const items=(correlations||[]).slice(0,8); if (!items.length) { ctx.fillStyle='#91abc4'; ctx.font='14px Arial'; ctx.fillText('No multi-stream entities in the current analysis window.', 16, 28); return; } const streams=[...new Set(items.flatMap(item=>item.streams||[]))].slice(0,8); const entityPoint=(index,total)=>({x:Math.max(86,cw*.25),y:42+(index+0.5)*Math.max(36,(ch-84)/total)}); const streamPoint=(index,total)=>({x:Math.min(cw-96,cw*.75),y:42+(index+0.5)*Math.max(36,(ch-84)/total)}); const streamPositions=Object.fromEntries(streams.map((name,index)=>[name,streamPoint(index,streams.length)])); const short=value=>String(value).length>22?`${String(value).slice(0,19)}...`:String(value); items.forEach((item,index)=>{const point=entityPoint(index,items.length); (item.streams||[]).filter(name=>streamPositions[name]).forEach(name=>{const target=streamPositions[name]; ctx.strokeStyle=item.security_events?'#e66b6b':'#347fae'; ctx.lineWidth=Math.min(5,1+Number(item.security_events||0)); ctx.beginPath(); ctx.moveTo(point.x,point.y); ctx.lineTo(target.x,target.y); ctx.stroke();});}); items.forEach((item,index)=>{const point=entityPoint(index,items.length); ctx.fillStyle='#1388cc'; ctx.beginPath(); ctx.arc(point.x,point.y,12,0,Math.PI*2); ctx.fill(); ctx.fillStyle='#d9e8f7'; ctx.font='12px Arial'; ctx.textAlign='right'; ctx.fillText(short(item.entity||item.source_ip),point.x-18,point.y+4);}); streams.forEach((name,index)=>{const point=streamPositions[name]; ctx.fillStyle='#218957'; ctx.fillRect(point.x-10,point.y-10,20,20); ctx.fillStyle='#d9e8f7'; ctx.font='12px Arial'; ctx.textAlign='left'; ctx.fillText(short(name),point.x+16,point.y+4);}); ctx.textAlign='left'; } const canvas=document.getElementById('trendChart'), ctx=canvas.getContext('2d'), ratio=window.devicePixelRatio||1, cw=canvas.clientWidth, ch=canvas.clientHeight;
canvas.width=cw*ratio; canvas.height=ch*ratio; ctx.scale(ratio,ratio); ctx.clearRect(0,0,cw,ch);
const rows=(history||[]).slice(-72);
if (!rows.length) { ctx.fillStyle='#91abc4'; ctx.font='14px Arial'; ctx.fillText('Waiting for monitor history.', 16, 28); return; }
const left=54,right=54,top=34,bottom=34,w=cw-left-right,h=ch-top-bottom;
const eventsMax=Math.max(1,...rows.map(item=>Number(item.events)||0));
const anomalyMax=Math.max(1,...rows.map(item=>Number(item.anomalies)||0),...rows.map(item=>Number(item.high||0)+Number(item.critical||0)));
ctx.strokeStyle='#163b59'; ctx.lineWidth=1; ctx.font='11px Arial'; ctx.textAlign='right'; ctx.fillStyle='#91abc4';
for(let tick=0; tick<=4; tick++){ const y=top+h-(tick/4)*h; ctx.beginPath(); ctx.moveTo(left,y); ctx.lineTo(cw-right,y); ctx.stroke(); ctx.fillText(Math.round(eventsMax*tick/4).toLocaleString(), left-8, y+4); }
ctx.textAlign='left'; ctx.fillStyle='#1ea9ff'; ctx.fillText('events', left, 18); ctx.fillStyle='#ffcf5a'; ctx.fillText('high', left+64, 18); ctx.fillStyle='#ff6666'; ctx.fillText('critical/anomalies', left+112, 18);
const barGap=2, barW=Math.max(2, w/rows.length-barGap);
rows.forEach((item,index)=>{ const x=left+index*w/rows.length; const eventH=((Number(item.events)||0)/eventsMax)*h; ctx.fillStyle='rgba(30,169,255,.42)'; ctx.fillRect(x, top+h-eventH, barW, eventH); const high=Number(item.high)||0, critical=Number(item.critical)||0; const highH=(high/anomalyMax)*h, critH=(critical/anomalyMax)*h; ctx.fillStyle='rgba(255,207,90,.85)'; ctx.fillRect(x, top+h-highH, Math.max(1,barW*.45), highH); ctx.fillStyle='rgba(255,83,83,.9)'; ctx.fillRect(x+Math.max(1,barW*.45), top+h-critH, Math.max(1,barW*.45), critH); });
const line=(key,color,max)=>{ ctx.strokeStyle=color; ctx.lineWidth=2.5; ctx.beginPath(); rows.forEach((item,index)=>{ const x=left+(index+.5)*w/rows.length; const y=top+h-((Number(item[key])||0)/max)*h; index?ctx.lineTo(x,y):ctx.moveTo(x,y); }); ctx.stroke(); };
line('anomalies','#ff6666',anomalyMax);
ctx.strokeStyle='#285071'; ctx.strokeRect(left,top,w,h); ctx.textAlign='right'; ctx.fillStyle='#ff9a9a'; ctx.fillText(`anomaly max ${anomalyMax}`, cw-8, 18); ctx.fillStyle='#91abc4'; ctx.fillText(`${rows.length} samples`, cw-right, ch-10); ctx.textAlign='left';
}
function drawCorrelationGraph(correlations) {
const canvas=document.getElementById('correlationGraph'), ctx=canvas.getContext('2d'), ratio=window.devicePixelRatio||1, cw=canvas.clientWidth, ch=canvas.clientHeight;
canvas.width=cw*ratio; canvas.height=ch*ratio; ctx.scale(ratio,ratio); ctx.clearRect(0,0,cw,ch);
const short=(value,max=22)=>String(value||'-').length>max?`${String(value).slice(0,max-3)}...`:String(value||'-');
const items=[...(correlations||[])].sort((a,b)=>(Number(b.security_events)||0)-(Number(a.security_events)||0) || (Number(b.events)||0)-(Number(a.events)||0) || (b.streams||[]).length-(a.streams||[]).length).slice(0,10);
if (!items.length) { ctx.fillStyle='#91abc4'; ctx.font='14px Arial'; ctx.fillText('No multi-stream entities in the current analysis window.', 16, 28); return; }
const streamScores=new Map();
items.forEach(item => (item.streams||[]).forEach(stream => streamScores.set(stream, (streamScores.get(stream)||0)+Number(item.security_events||0)+1)));
const streams=[...streamScores.entries()].sort((a,b)=>b[1]-a[1]).map(([name])=>name).slice(0,8);
const left=Math.min(210, Math.max(130, cw*.24)), top=54, right=20, bottom=34, rowH=Math.max(24, Math.min(34, (ch-top-bottom)/items.length)), cellW=(cw-left-right)/Math.max(1,streams.length);
ctx.font='12px Arial'; ctx.textAlign='center'; ctx.fillStyle='#83bce9';
streams.forEach((stream,index)=>{ const x=left+index*cellW+cellW/2; ctx.save(); ctx.translate(x, top-8); ctx.rotate(-Math.PI/6); ctx.fillText(short(stream,18),0,0); ctx.restore(); });
ctx.textAlign='right'; ctx.fillStyle='#83bce9'; ctx.fillText('Entity', left-12, top-14);
items.forEach((item,row)=>{ const y=top+row*rowH; const severity=Number(item.security_events)||0; ctx.fillStyle=severity?'#ff9a9a':'#d9e8f7'; ctx.textAlign='right'; ctx.fillText(short(item.entity||item.source_ip,24), left-12, y+rowH*.65); ctx.fillStyle='#91abc4'; ctx.fillText(`${Number(item.events)||0}`, left-12, y+rowH*.95); streams.forEach((stream,col)=>{ const x=left+col*cellW+4; const active=(item.streams||[]).includes(stream); ctx.fillStyle=active ? (severity ? `rgba(255,86,86,${Math.min(.95,.28+severity*.08)})` : 'rgba(30,169,255,.48)') : 'rgba(12,42,68,.7)'; ctx.fillRect(x,y+4,Math.max(8,cellW-8),rowH-8); }); });
ctx.textAlign='left'; ctx.fillStyle='#91abc4'; ctx.fillText('cell = entity observed in stream; red intensity = security events; number below entity = events', 8, ch-10);
}
async function refresh() { async function refresh() {
const openDetails = new Set([...document.querySelectorAll('details[open][data-detail-id]')].map(item => item.dataset.detailId)); const openDetails = new Set([...document.querySelectorAll('details[open][data-detail-id]')].map(item => item.dataset.detailId));
const res = await fetch('/api/status', {cache: 'no-store'}); const res = await fetch('/api/status', {cache: 'no-store'});
@@ -175,7 +206,7 @@ async function refresh() {
if (rawCorrelations.length) uiCache.correlations = rawCorrelations; if (rawCorrelations.length) uiCache.correlations = rawCorrelations;
drawTrend(data.history || []); drawTrend(data.history || []);
drawCorrelationGraph(correlations); drawCorrelationGraph(correlations);
document.getElementById('correlationGraphInfo').textContent = `${correlations.length} entities correlated across enabled streams${correlationsCached ? ' (cached from previous non-empty poll)' : ''}. Blue entities connect to green stream nodes; red edges carry security-event activity.`; document.getElementById('correlationGraphInfo').textContent = `${correlations.length} entities correlated across enabled streams${correlationsCached ? ' (cached from previous non-empty poll)' : ''}. Matrix shows top entities by security activity; colored cells mean the entity was observed in that stream, red intensity marks security-event activity.`;
document.getElementById('stamp').textContent = data.generated_at ? `Updated ${new Date(data.generated_at * 1000).toLocaleString()}` : 'Waiting for monitor data'; document.getElementById('stamp').textContent = data.generated_at ? `Updated ${new Date(data.generated_at * 1000).toLocaleString()}` : 'Waiting for monitor data';
document.getElementById('metrics').innerHTML = [ document.getElementById('metrics').innerHTML = [
metric('Total events', s.total || 0), metric('Total events', s.total || 0),
@@ -197,10 +228,14 @@ async function refresh() {
`High anomalies: ${esc((a.high || 0))}`, `High anomalies: ${esc((a.high || 0))}`,
`Baseline sources ready: ${esc((data.baseline || {}).sources_ready || 0)}`, `Baseline sources ready: ${esc((data.baseline || {}).sources_ready || 0)}`,
`Baseline training days: ${esc((data.baseline || {}).training_days || 0)}`, `Baseline training days: ${esc((data.baseline || {}).training_days || 0)}`,
`Baseline DB size: ${esc(bytes((data.baseline || {}).size_bytes || 0))}` `Baseline DB size: ${esc(bytes((data.baseline || {}).size_bytes || 0))}`,
].join('<br>'); `MCP poll window: ${esc(mcp.range_seconds || configuration.graylog_range_seconds || 0)}s`,
`MCP max events/stream: ${esc(mcp.max_events_per_stream || configuration.graylog_max_events_per_stream || 0)}`,
`MCP coverage: ${esc(mcp.coverage_status || 'unknown')}${mcp.truncated_streams ? ` (${esc(mcp.truncated_streams)} truncated)` : ''}`,
mcp.coverage_warning ? `<span class="sev-high">${esc(mcp.coverage_warning)}</span>` : ''
].filter(Boolean).join('<br>');
document.getElementById('health').innerHTML = [ document.getElementById('health').innerHTML = [
metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Correlated entities', correlations.length) metric('Enabled streams', enabledStreams.length), metric('Streams missing profile', streamsMissingProfile), metric('MCP events fetched', mcp.events_fetched || 0), metric('Truncated streams', mcp.truncated_streams || 0), metric('Correlated entities', correlations.length)
].join(''); ].join('');
window.profileSuggestions = data.profile_suggestions || []; window.profileSuggestions = data.profile_suggestions || [];
const advisor = ((data.capabilities || {}).profile_advisor || {}); const advisor = ((data.capabilities || {}).profile_advisor || {});
@@ -400,7 +435,8 @@ async function applySuggestedProfile(streamId) {
log_source: config.log_source || 'graylog_mcp', log_source: config.log_source || 'graylog_mcp',
graylog_mcp_url: config.graylog_mcp_url || '', graylog_mcp_url: config.graylog_mcp_url || '',
graylog_query: config.graylog_query || '*', graylog_query: config.graylog_query || '*',
graylog_range_seconds: config.graylog_range_seconds || 3600, graylog_range_seconds: config.graylog_range_seconds || 300,
graylog_max_events_per_stream: config.graylog_max_events_per_stream || 5000,
graylog_field_mapping: config.graylog_field_mapping || '', graylog_field_mapping: config.graylog_field_mapping || '',
baseline_training_days: config.baseline_training_days || 7, baseline_training_days: config.baseline_training_days || 7,
baseline_retention_days: config.baseline_retention_days || 14, baseline_retention_days: config.baseline_retention_days || 14,

5
src/fgai/graylog_source.py
View File

@@ -59,7 +59,8 @@ class GraylogStreamSource:
page_size = 1_000 page_size = 1_000
pages = 0 pages = 0
while len(events) < max_events: while len(events) < max_events:
result = self.client.call_tool("search_messages", {**arguments, "limit": page_size, "offset": len(events)}) request_limit = min(page_size, max_events - len(events))
result = self.client.call_tool("search_messages", {**arguments, "limit": request_limit, "offset": len(events)})
content = result.get("result", {}).get("content", []) if isinstance(result.get("result"), dict) else [] content = result.get("result", {}).get("content", []) if isinstance(result.get("result"), dict) else []
if isinstance(result.get("result"), dict) and result["result"].get("isError"): if isinstance(result.get("result"), dict) and result["result"].get("isError"):
detail = next((str(item.get("text")) for item in content if isinstance(item, dict) and item.get("type") == "text"), "Graylog search failed") detail = next((str(item.get("text")) for item in content if isinstance(item, dict) and item.get("type") == "text"), "Graylog search failed")
@@ -73,7 +74,7 @@ class GraylogStreamSource:
continue continue
events.extend(self._event(record) for record in records) events.extend(self._event(record) for record in records)
pages += 1 pages += 1
if len(records) < page_size: if len(records) < request_limit:
break break
latest = max((event.fields.get("eventtime", "") for event in events), default="") latest = max((event.fields.get("eventtime", "") for event in events), default="")
status.update({"source": "graylog_mcp", "events_fetched": len(events), "pages": pages, "truncated": len(events) >= max_events, "latest_event_time": latest}) status.update({"source": "graylog_mcp", "events_fetched": len(events), "pages": pages, "truncated": len(events) >= max_events, "latest_event_time": latest})

22
src/fgai/monitor.py
View File

@@ -50,7 +50,7 @@ def _range_seconds(value: object) -> int:
try: try:
return max(60, int(value)) return max(60, int(value))
except (TypeError, ValueError): except (TypeError, ValueError):
return 3600 return 300
def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[str, object], stream_status: dict[str, object], profile_readiness: list[dict[str, object]], stream_titles: dict[str, str]) -> list[dict[str, object]]: def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[str, object], stream_status: dict[str, object], profile_readiness: list[dict[str, object]], stream_titles: dict[str, str]) -> list[dict[str, object]]:
@@ -128,7 +128,8 @@ def build_status(
stream_configs = [{"id": str(runtime_values.get("graylog_stream", "")), "title": "Graylog"}] stream_configs = [{"id": str(runtime_values.get("graylog_stream", "")), "title": "Graylog"}]
stream_statuses = [] stream_statuses = []
events = [] events = []
range_seconds = _range_seconds(runtime_values.get("graylog_range_seconds", 3600)) range_seconds = _range_seconds(runtime_values.get("graylog_range_seconds", 300))
max_events_per_stream = max(1, int(runtime_values.get("graylog_max_events_per_stream", 5000) or 5000))
for stream_config in stream_configs: for stream_config in stream_configs:
stream_id = str(stream_config["id"]) stream_id = str(stream_config["id"])
profile = stream_profiles.get(stream_id) profile = stream_profiles.get(stream_id)
@@ -140,10 +141,23 @@ def build_status(
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())), *tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
) if profile else () ) if profile else ()
stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id) stream_name = str(stream_config.get("title", "") or stream_titles.get(stream_id) or stream_id)
stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(range_seconds=range_seconds) stream_events, stream_status = GraylogStreamSource(GraylogMcpClient(url, token), stream_id, str(runtime_values.get("graylog_query", "*")), str(runtime_values.get("graylog_field_mapping", "")), stream_name, profile_fields).fetch(max_events=max_events_per_stream, range_seconds=range_seconds)
events.extend(stream_events) events.extend(stream_events)
stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status}) stream_statuses.append({"stream_id": stream_id, "stream_name": stream_name, **stream_status})
mcp_status = {"status": "connected", "streams": stream_statuses, "events_fetched": len(events), "range_seconds": range_seconds} truncated_streams = [item for item in stream_statuses if item.get("truncated")]
mcp_status = {
"status": "connected",
"streams": stream_statuses,
"events_fetched": len(events),
"range_seconds": range_seconds,
"max_events_per_stream": max_events_per_stream,
"truncated_streams": len(truncated_streams),
"coverage_status": "truncated" if truncated_streams else "complete_window",
"coverage_warning": (
f"{len(truncated_streams)} stream(s) hit max_events_per_stream; high EPS means the analysis window is only partially sampled."
if truncated_streams else ""
),
}
except RuntimeError as exc: except RuntimeError as exc:
mcp_status = {"status": "error", "error": str(exc)} mcp_status = {"status": "error", "error": str(exc)}
events = [] events = []

8
tests/test_config.py
View File

@@ -23,6 +23,14 @@ class ConfigTests(unittest.TestCase):
public = store.update({"graylog_range_seconds": "7200"}) public = store.update({"graylog_range_seconds": "7200"})
self.assertEqual(public["graylog_range_seconds"], 7200) self.assertEqual(public["graylog_range_seconds"], 7200)
def test_graylog_max_events_per_stream_is_numeric_and_bounded(self):
with tempfile.TemporaryDirectory() as directory:
store = ConfigStore(str(Path(directory) / "config.json"))
public = store.update({"graylog_max_events_per_stream": "0"})
self.assertEqual(public["graylog_max_events_per_stream"], 1)
public = store.update({"graylog_max_events_per_stream": "25000"})
self.assertEqual(public["graylog_max_events_per_stream"], 25000)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

7
tests/test_graylog_source.py
View File

@@ -35,6 +35,13 @@ class GraylogSourceTests(unittest.TestCase):
GraylogStreamSource(client, "vpn").fetch(range_seconds=86_400) GraylogStreamSource(client, "vpn").fetch(range_seconds=86_400)
self.assertEqual(client.arguments["range_seconds"], 86_400) self.assertEqual(client.arguments["range_seconds"], 86_400)
def test_marks_stream_truncated_when_max_events_reached(self):
client = _Client()
events, status = GraylogStreamSource(client, "vpn").fetch(max_events=2)
self.assertEqual(len(events), 2)
self.assertEqual(client.arguments["limit"], 2)
self.assertTrue(status["truncated"])
def test_requests_selected_profile_fields(self): def test_requests_selected_profile_fields(self):
client = _Client() client = _Client()
GraylogStreamSource(client, "windows", profile_fields=("TargetUserName", "EventID")).fetch() GraylogStreamSource(client, "windows", profile_fields=("TargetUserName", "EventID")).fetch()