diff --git a/src/fgai/dashboard.py b/src/fgai/dashboard.py index 98db181..62bc22a 100644 --- a/src/fgai/dashboard.py +++ b/src/fgai/dashboard.py @@ -141,6 +141,12 @@ HTML = """ .ai-box { max-height: 260px; overflow: auto; } .ai-box details { border: 1px solid #163b59; background: #061a2e; border-radius: 6px; padding: 8px; } .ai-box summary { cursor: pointer; color: #83bce9; font-weight: 700; } + .collapsible-section { border: 1px solid #163b59; border-radius: 6px; padding: 0 10px; margin-bottom: 8px; background: #061a2e; } + .collapsible-section summary { cursor: pointer; color: #83bce9; font-weight: 700; font-size: 1.02em; padding: 10px 0; list-style: none; } + .collapsible-section summary::-webkit-details-marker { display: none; } + .collapsible-section summary::before { content: '▸ '; } + .collapsible-section[open] summary::before { content: '▾ '; } + .collapsible-section > div { padding-bottom: 10px; } .sort-button { border: 0; background: transparent; color: #83bce9; cursor: pointer; font: inherit; font-weight: 600; padding: 0; } .sort-button:hover { color: #d9e8f7; } .model-list { display: flex; flex-wrap: wrap; gap: 8px; } @@ -185,7 +191,7 @@ HTML = """

Events and Anomalies

Baseline and Stream Health

Operator Guidance

Waiting for monitor data.

Correlation Map

Waiting for correlated entities.

Investigation Incidents

Anomalies

Recommendations

AI Assessment

LLM assessment disabled.
-

Triage Queue

Field Baseline Deviations

Active Feedback Rules

Related Activity Across Sources

Block Candidates

Threat Intelligence

Policy Findings

+

Triage Queue

Field Baseline Deviations

Active Feedback Rules

Related Activity Across Sources
Block Candidates
Threat Intelligence
Policy Findings

Diagnostics

Recommended Stream Profiles

Waiting for observed stream data.

Installed Ollama Models

Loading local Ollama models.

Runtime Configuration

How To Use SignalScope

1. Normal workflow

  1. Settings: connect Graylog MCP and enable streams.
  2. Settings: apply recommended profiles for missing streams.
  3. Diagnostics: confirm raw samples, aggregate counts, and profile readiness.
  4. Overview: use Operator Guidance, incidents, trends, and correlation map.
  5. Findings: review only high-signal deviations first, then mark decisions.

2. Stream health

  • ready: profile exists, events are arriving, and baseline fields are ready.
  • learning: profile exists and events arrive, but baseline age or buckets are still too low.
  • missing_profile: stream is enabled but no profile exists. Apply or edit one.
  • no_events: stream is enabled but the current poll has no raw sample events.
  • partial_fetch: Graylog returned only part of the requested raw sample.

3. Ready fields

0/8 means 8 profile fields are tracked but none are mature enough yet. A field needs at least 12 baseline buckets and the configured Baseline training days before it is ready.

During learning, treat findings as signals to tune profiles, not as final alerts.

4. Profiles

  • Entity fields define who or what behavior is tracked, such as user, host, source IP, or application actor.
  • Baseline fields define the changing behavior to learn, such as action, event ID, service, URL, status, or counters.
  • Relationships learn pairs such as username to srcip or host to process.
  • Detectors add burst checks for auth failures, DNS queries, and deny actions.

5. Findings

  • Start with Triage Queue and incidents, not raw long tables.
  • Open evidence details before confirming a finding.
  • Use Expected for known behavior, False positive for bad signal, Confirmed for real investigation items.
  • Use expiry when a behavior is expected only temporarily.

6. High EPS / MCP

  • Use aggregate or auto fetch mode for high EPS streams.
  • Keep raw samples small enough for context; aggregate counts represent the full window.
  • Sample capped is normal in aggregate mode. Truncated raw mode means you may miss context.
  • If MCP is stale, the UI shows cached status so you can still inspect previous findings.

7. Ollama

  • Dashboard assessment summarizes current evidence.
  • Profile advisor maps unknown/custom fields and suggests relationships.
  • Ollama advice is constrained to fields discovered from Graylog; unknown fields are rejected.

8. Baseline DB size

  • Baseline DB size is the local SQLite behavior baseline.
  • Retention deletes old rows, but SQLite only returns disk space after manual VACUUM.
  • If the DB grows fast, lower bucket retention, value retention, or max values per entity field in Settings.
  • Stop the service before running signalscope baseline-maintenance --vacuum.

9. What to fix first

  1. No streams enabled.
  2. Enabled streams with missing profiles.
  3. Enabled streams with zero raw events.
  4. Profiles stuck at 0 ready fields after the training window.
  5. Too many repeated findings without review feedback.
  6. Baseline DB growing without maintenance.
@@ -220,8 +226,14 @@ let pendingReview = null; let lastReviewAuthor = ''; let selectedDeviationKeys = new Set(); function deviationKey(r) { return `${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; } -let openDetailIds = new Set(); +let openDetailIds = new Set(['diag-stream-coverage', 'diag-profile-readiness']); +function collapsibleSection(id, title, innerHtml) { + const detailId = `diag-${id}`; + const open = openDetailIds.has(detailId) ? ' open' : ''; + return `
${esc(title)}
${innerHtml}
`; +} let refreshInFlight = false; +let tableScrollLeft = {}; window.availableStreams = []; window.streamSelection = {}; window.configuredStreams = []; @@ -234,7 +246,7 @@ function table(rows, columns, id = '') { const sorted = sort ? [...rows].sort((left, right) => { const a=left[sort.key] ?? '', b=right[sort.key] ?? ''; const numeric=Number(a), numericB=Number(b); const compare=Number.isFinite(numeric) && Number.isFinite(numericB) && String(a).trim() !== '' && String(b).trim() !== '' ? numeric-numericB : String(a).localeCompare(String(b)); return sort.direction * compare; }) : rows; const head = columns.map(c => `${c.key && id ? `` : esc(c.label)}`).join(''); const body = sorted.map(row => `${columns.map(c => `${c.render ? c.render(row) : esc(row[c.key])}`).join('')}`).join(''); - return `
${head}${body}
`; + return `
${head}${body}
`; } function capability(label, state, detail) { return `${esc(label)}: ${esc(detail)}`; } function mcpCapability(mcp, configuration, pollRunningSeconds=0) { @@ -787,6 +799,23 @@ async function refresh() { const rawFieldRows = Object.entries(data.field_deviations || {}).flatMap(([entity, deviations]) => (deviations || []).map(item => ({entity, ...item, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id}))); const fieldRowsCached = rawFieldRows.length === 0 && uiCache.fieldRows.length > 0; const sourceFieldRows = rawFieldRows.length ? rawFieldRows : uiCache.fieldRows; + const detectorPrecision = {}; + sourceFieldRows.forEach(item => { + if (!['confirmed', 'false_positive'].includes(item.feedback || '')) return; + const key = item.detector || 'unknown'; + detectorPrecision[key] = detectorPrecision[key] || {confirmed: 0, false_positive: 0}; + detectorPrecision[key][item.feedback] += 1; + }); + const suggestReview = r => { + if (r.feedback) return '-'; + const stats = detectorPrecision[r.detector || 'unknown']; + const total = stats ? stats.confirmed + stats.false_positive : 0; + if (total < 3) return esc('not enough history'); + const fpRatio = stats.false_positive / total; + if (fpRatio >= 0.7) return `likely false positive (${Math.round(fpRatio * 100)}% of ${total} reviewed)`; + if (fpRatio <= 0.2) return `likely worth confirming (${Math.round((1 - fpRatio) * 100)}% of ${total} reviewed)`; + return esc('mixed history'); + }; const showReviewed = document.getElementById('showReviewedFindings')?.checked; const showLow = document.getElementById('showLowFindings')?.checked; const fieldRows = sourceFieldRows @@ -799,7 +828,7 @@ async function refresh() { if (rawFieldRows.length) uiCache.fieldRows = rawFieldRows; document.getElementById('fieldDeviations').innerHTML = `${fieldRowsCached ? '

No current field deviations in this poll; showing cached findings from the last non-empty poll.

' : ''}` + table(fieldRows, [ {label:'', render:r => { const key = deviationKey(r); return ``; }}, - {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `
${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('
'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} + {label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Stream', key:'stream_title'}, {label:'Detector', key:'detector'}, {label:'Field', key:'field'}, {label:'Score', key:'score'}, {label:'Baseline', render:r => esc(`${r.confidence || '-'}; ${r.baseline_age_days ?? 0}d; ${r.baseline_samples || 0} samples`)}, {label:'Review', render:r => esc(r.feedback || 'unreviewed')}, {label:'Suggested', render:suggestReview}, {label:'Evidence', render:r => { const weighted = r.weight && r.weight !== 1 ? `; weighted ${r.base_score ?? r.score} x ${r.weight}` : ''; const summary=esc(`${r.reason}; current ${r.current ?? '-'} vs baseline ${r.baseline ?? '-'}; scope ${r.baseline_scope ?? '-'}${weighted}; values: ${(r.sample_values || []).join(', ') || '-'}`); const events=(r.sample_events || []).map(item => `${esc(`${item.timestamp} | ${item.source} -> ${item.destination} | ${item.action} ${item.service} | ${item.value} | ${item.message}`)}${item.graylog_query ? `
${esc(item.graylog_query)}${graylogEvidenceLink(item.graylog_query, configuration)}` : ''}`).join('
'); const id=`deviation:${r.entity}:${r.stream_id}:${r.field}:${r.value || ''}`; return events ? `
${summary}

${events}

` : summary; }}, {label:'Review action', render:r => `
`} ], 'field-deviations'); const bulkReviewBar = document.getElementById('bulkReviewBar'); const updateBulkReviewBar = () => { @@ -950,16 +979,17 @@ async function refresh() { const profileNames = Object.fromEntries((data.stream_profiles || []).map(item => [item.stream_id, item.name || item.stream_id])); const profileReadiness = (data.profile_readiness || []).map(item => ({...item, profile_name: item.profile_name || profileNames[item.stream_id] || item.stream_id, stream_title: item.stream_name || item.stream_title || streamTitles[item.stream_id] || item.stream_id})); document.getElementById('diagnostics').innerHTML = - '

Stream Coverage

' + table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage') + - '

Cross-Source Correlations

' + table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations') + - '

Entities

' + table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities') + - '

Profile Baseline Readiness

' + table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `` : '-'}], 'profile-readiness') + - '

Data Quality

' + table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}]) + - '

Feedback Rules

' + table( + collapsibleSection('stream-coverage', 'Stream Coverage', table(streamCoverage, [{label:'Stream', key:'stream_name'}, {label:'Enabled', key:'enabled', render:r => r.enabled ? 'yes' : 'no'}, {label:'Profile', render:r => esc(r.profile || 'missing')}, {label:'Entity Field', key:'entity_field'}, {label:'Tracked Fields', key:'tracked_fields'}, {label:'Ready Fields', key:'readiness'}, {label:'Raw Events', key:'events_fetched'}, {label:'Aggregate Events', key:'aggregate_events'}, {label:'Aggregate', key:'aggregate_status'}, {label:'Aggregate Schema', key:'aggregate_schema_properties'}, {label:'Latest Event', key:'latest_event_time'}, {label:'Health', render:r => esc(`${r.health || ''}${r.health_detail ? ': ' + r.health_detail : ''}`)}, {label:'Aggregate Error', render:r => esc(r.aggregate_error || '-')}, {label:'Raw Error', render:r => esc(r.raw_error || '-')}], 'stream-coverage')) + + collapsibleSection('correlations', 'Cross-Source Correlations', table(correlations, [{label:'Entity', key:'entity', render:r => esc(`${correlationShortLabel(r)} (${r.entity_type || 'ip'})`)}, {label:'Streams', render:r => esc((r.streams || []).join(', '))}, {label:'Events', key:'events'}, {label:'Security Events', key:'security_events'}], 'correlations')) + + collapsibleSection('entities', 'Entities', table(context.source_profiles || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Events', key:'events'}, {label:'UTM', key:'utm_events'}, {label:'Deny', key:'deny_or_threat_actions'}, {label:'Destinations', key:'distinct_destinations'}, {label:'Actions', render:r => esc((r.top_actions || []).join(', '))}], 'entities')) + + collapsibleSection('profile-readiness', 'Profile Baseline Readiness', table(profileReadiness, [{label:'Profile', key:'profile_name'}, {label:'Stream', key:'stream_title'}, {label:'Field', key:'field'}, {label:'Buckets', key:'buckets'}, {label:'Age days', key:'age_days'}, {label:'Training days', key:'training_days'}, {label:'Ready', key:'ready', render:r => r.ready ? 'ready' : 'learning'}, {label:'Action', render:r => r.stream_id ? `` : '-'}], 'profile-readiness')) + + collapsibleSection('data-quality', 'Data Quality', table([quality], [{label:'Events', key:'events'}, {label:'Timestamp coverage', render:r => `${r.timestamp_coverage || 0}%`}, {label:'Source coverage', render:r => `${r.source_coverage || 0}%`}, {label:'Truncated streams', render:r => esc((r.truncated_streams || []).join(', ') || 'none')}], 'data-quality')) + + collapsibleSection('feedback-rules', 'Feedback Rules', table( ['value', 'field', 'entity', 'stream'].map(scope => ({scope, count: feedbackRules.filter(item => (item.scope || 'value') === scope).length})), - [{label:'Scope', key:'scope'}, {label:'Active rules', key:'count'}] - ) + `

${rawFieldRows.filter(item => ['expected', 'false_positive'].includes(item.feedback || '')).length} finding(s) suppressed by feedback this poll.

` + - '

Detector Precision

' + table( + [{label:'Scope', key:'scope'}, {label:'Active rules', key:'count'}], + 'feedback-rules-by-scope' + ) + `

${rawFieldRows.filter(item => ['expected', 'false_positive'].includes(item.feedback || '')).length} finding(s) suppressed by feedback this poll.

`) + + collapsibleSection('detector-precision', 'Detector Precision', table( Object.entries(rawFieldRows.reduce((acc, item) => { if (!['confirmed', 'false_positive'].includes(item.feedback || '')) return acc; const key = item.detector || 'unknown'; @@ -967,22 +997,28 @@ async function refresh() { acc[key][item.feedback] += 1; return acc; }, {})).map(([detector, counts]) => ({detector, confirmed: counts.confirmed, false_positive: counts.false_positive, precision: (counts.confirmed + counts.false_positive) ? `${(counts.confirmed / (counts.confirmed + counts.false_positive) * 100).toFixed(1)}%` : '-'})), - [{label:'Detector', key:'detector'}, {label:'Confirmed', key:'confirmed'}, {label:'False positive', key:'false_positive'}, {label:'Precision', key:'precision'}] - ) + - '

Security Event Samples

' + table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}]) + - '

Top Sources

' + table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}]) + - '

Top Destinations

' + table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + - '

Top Policy IDs

' + table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + - '

Top Destination Ports

' + table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + - '

Top Source Ports

' + table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + - '

Top Services

' + table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}]) + - '

Local-in Failures

' + table(d.local_in_failures || [], [{label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}]); + [{label:'Detector', key:'detector'}, {label:'Confirmed', key:'confirmed'}, {label:'False positive', key:'false_positive'}, {label:'Precision', key:'precision'}], + 'detector-precision' + )) + + collapsibleSection('security-event-samples', 'Security Event Samples', table(context.security_event_samples || [], [{label:'Entity', key:'entity', render:r => entityCell(r)}, {label:'Type', key:'type'}, {label:'Action', key:'action'}, {label:'Severity', key:'severity'}, {label:'Destination', key:'dst'}, {label:'Service', key:'service'}], 'security-event-samples')) + + collapsibleSection('top-sources', 'Top Sources', table(d.top_source_ips || [], [{label:'Value', key:'value', render:r => entityCell({entity:r.value})}, {label:'Count', key:'count'}], 'top-sources')) + + collapsibleSection('top-destinations', 'Top Destinations', table(d.top_destination_ips || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}], 'top-destinations')) + + collapsibleSection('top-policy-ids', 'Top Policy IDs', table(d.top_policy_ids || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}], 'top-policy-ids')) + + collapsibleSection('top-destination-ports', 'Top Destination Ports', table(d.top_destination_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}], 'top-destination-ports')) + + collapsibleSection('top-source-ports', 'Top Source Ports', table(d.top_source_ports || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}], 'top-source-ports')) + + collapsibleSection('top-services', 'Top Services', table(d.top_services || [], [{label:'Value', key:'value'}, {label:'Count', key:'count'}], 'top-services')) + + collapsibleSection('local-in-failures', 'Local-in Failures', table(d.local_in_failures || [], [{label:'Source', key:'src_ip', render:r => entityCell(r, 'src_ip')}, {label:'Service', key:'service'}, {label:'Policy', key:'policy'}, {label:'Count', key:'count'}], 'local-in-failures')); document.querySelectorAll('details[data-detail-id]').forEach(item => { if (openDetailIds.has(item.dataset.detailId)) item.open = true; item.addEventListener('toggle', () => { if (item.open) openDetailIds.add(item.dataset.detailId); else openDetailIds.delete(item.dataset.detailId); }); }); + document.querySelectorAll('.table-wrap[data-table-id]').forEach(wrap => { + const tid = wrap.dataset.tableId; + if (tableScrollLeft[tid]) wrap.scrollLeft = tableScrollLeft[tid]; + wrap.addEventListener('scroll', () => { tableScrollLeft[tid] = wrap.scrollLeft; }); + }); document.querySelectorAll('[data-sort-table]').forEach(button => button.addEventListener('click', () => { const current=tableSort[button.dataset.sortTable]; tableSort[button.dataset.sortTable]={key:button.dataset.sortKey,direction:current && current.key===button.dataset.sortKey ? -current.direction : 1}; refresh(); })); } catch (error) { showRefreshError(error); diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index f9cf23d..1dc3b0c 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -94,6 +94,30 @@ class DashboardTests(unittest.TestCase): self.assertIn("openDetailIds", HTML) self.assertIn("addEventListener('toggle'", HTML) + def test_dashboard_suggests_review_from_detector_history(self): + self.assertIn("label:'Suggested', render:suggestReview", HTML) + self.assertIn("likely false positive", HTML) + self.assertIn("likely worth confirming", HTML) + self.assertIn("not enough history", HTML) + + def test_dashboard_secondary_findings_panels_are_collapsible(self): + self.assertIn('Related Activity Across Sources', HTML) + self.assertIn('Block Candidates', HTML) + self.assertIn('Threat Intelligence', HTML) + self.assertIn('Policy Findings', HTML) + self.assertIn('id="relatedActivity"', HTML) + + def test_dashboard_diagnostics_sections_are_collapsible(self): + self.assertIn("collapsibleSection('stream-coverage'", HTML) + self.assertIn("collapsibleSection('top-services'", HTML) + self.assertIn("class=\"collapsible-section\"", HTML) + self.assertIn("diag-stream-coverage", HTML) + + def test_dashboard_table_scroll_position_survives_refresh(self): + self.assertIn("tableScrollLeft", HTML) + self.assertIn("data-table-id", HTML) + self.assertIn("addEventListener('scroll'", HTML) + def test_dashboard_has_bulk_review_bar(self): self.assertIn('id="bulkReviewBar"', HTML) self.assertIn('id="selectAllDeviations"', HTML)