Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fortsatte roadmapen med multi-entity stream profiles

Browse Source
This commit is contained in:
larssand
2026-06-29 19:08:23 +02:00
parent 20b0e0a92e
commit 68370217da
10 changed files with 102 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/fgai/baseline.py
View File

@@ -10,7 +10,7 @@ from statistics import mean, pstdev
from .logs import THREAT_ACTIONS, is_utm_event
from .models import LogEvent
from .entities import profile_entity
from .entities import profile_entities
from .detectors import DETECTOR_MINIMUMS, event_detector_categories
from .normalization import canonical_value
@@ -180,31 +180,32 @@ class BaselineStore:
fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1:
continue
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
entities = profile_entities(event, profile)
if not entities:
continue
timestamp = _event_epoch(event, observed_at)
bucket = timestamp - (timestamp % self.bucket_seconds)
moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
for detector in event_detector_categories(event):
detector_pending[(stream_id, entity, detector, bucket)] += 1
detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in fields:
key = (stream_id, entity, str(field).lower(), bucket)
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
pending[key][0] += 1
pending[key][1] += value
pending[key][2] += value * value
temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
temporal_pending[temporal][0] += 1
temporal_pending[temporal][1] += value
temporal_pending[temporal][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
pending_values[(*key[:3], raw_value)] += 1
for entity in entities:
for detector in event_detector_categories(event):
detector_pending[(stream_id, entity, detector, bucket)] += 1
detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
for field in fields:
key = (stream_id, entity, str(field).lower(), bucket)
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
pending[key][0] += 1
pending[key][1] += value
pending[key][2] += value * value
temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
temporal_pending[temporal][0] += 1
temporal_pending[temporal][1] += value
temporal_pending[temporal][2] += value * value
if key[2] not in numeric:
raw_value = event.fields.get(key[2])
if raw_value:
pending_values[(*key[:3], raw_value)] += 1
for (stream_id, entity, field, bucket), values in pending.items():
connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
@@ -230,23 +231,24 @@ class BaselineStore:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
continue
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
entities = profile_entities(event, profile)
if not entities:
continue
entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
for detector in event_detector_categories(event):
detector_current[(event.fields.get("fgai_stream_id", ""), entity, detector)] += 1
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
current[key][0] += 1
if key[2] in numeric:
current[key][1] += _number(event.fields.get(key[2]))
for entity in entities:
entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
for detector in event_detector_categories(event):
detector_current[(event.fields.get("fgai_stream_id", ""), entity, detector)] += 1
for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
current[key][0] += 1
if key[2] in numeric:
current[key][1] += _number(event.fields.get(key[2]))
output: dict[str, list[dict[str, object]]] = defaultdict(list)
with self._connect() as connection:
for (stream, entity, field), values in current.items():
profile = profiles.get(stream)
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and entity in profile_entities(event, profiles.get(stream)) and event.fields.get(field)]
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
@@ -336,23 +338,24 @@ class BaselineStore:
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
if not profile:
continue
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
if not entity:
entities = profile_entities(event, profile)
if not entities:
continue
stream = event.fields.get("fgai_stream_id", "")
for field in getattr(profile, "categorical_fields", ()):
field = str(field).lower()
value = event.fields.get(field)
if not value:
continue
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 30:
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
score, weight = _weighted_score(12, profile, field, "rare_value")
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
for entity in entities:
for field in getattr(profile, "categorical_fields", ()):
field = str(field).lower()
value = event.fields.get(field)
if not value:
continue
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
if known is None and known_total >= 30:
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and entity in profile_entities(item, profile) and item.fields.get(field) == value]
score, weight = _weighted_score(12, profile, field, "rare_value")
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
if evidence not in output[entity]:
output[entity].append(evidence)
return output
def profile_readiness(self, profiles: dict[str, object]) -> list[dict[str, object]]:

9
src/fgai/cli.py
View File

@@ -227,10 +227,11 @@ def _stream_name(config: dict[str, object], stream_id: str) -> str:
def _profile_fields(profile: object | None) -> tuple[str, ...]:
if not profile:
return ()
return tuple(
field for field in (
str(getattr(profile, "entity_field", "")),
str(getattr(profile, "timestamp_field", "")),
return tuple(
field for field in (
str(getattr(profile, "entity_field", "")),
*tuple(str(item) for item in getattr(profile, "entity_fields", ())),
str(getattr(profile, "timestamp_field", "")),
*tuple(str(item) for item in getattr(profile, "categorical_fields", ())),
*tuple(str(item) for item in getattr(profile, "numeric_fields", ())),
)

5
src/fgai/dashboard.py
View File

@@ -271,7 +271,8 @@ async function editStreamProfile(streamId, title) {
document.querySelector('[name="profile_name"]').value = profile.name || `${title || streamId} profile`;
document.querySelector('[name="profile_detectors"]').value = Object.keys(profile.detectors || {}).length ? JSON.stringify(profile.detectors, null, 2) : '';
document.querySelector('[name="profile_field_weights"]').value = Object.keys(profile.field_weights || {}).length ? JSON.stringify(profile.field_weights, null, 2) : '';
const rows = (payload.fields || []).map(field => { const name=field.name||field.field, type=(field.type||{}).type||'', props=(field.type||{}).properties||[]; return `<div class="field-row"><code>${esc(name)}</code><span>${esc(type)}</span><span>${esc(props.join(', '))}</span><div class="field-controls"><label><input type="radio" name="profile_entity" value="${esc(name)}" ${profile.entity_field===name?'checked':''}> Entity</label><label><input type="radio" name="profile_timestamp" value="${esc(name)}" ${profile.timestamp_field===name?'checked':''}> Time</label>${props.includes('enumerable')?`<label><input class="profile-categorical" type="checkbox" value="${esc(name)}" ${(profile.categorical_fields||[]).includes(name)?'checked':''}> Categorical</label>`:''}${props.includes('numeric')?`<label><input class="profile-numeric" type="checkbox" value="${esc(name)}" ${(profile.numeric_fields||[]).includes(name)?'checked':''}> Numeric</label>`:''}</div></div>`; }).join('');
const entityFields = new Set(profile.entity_fields || (profile.entity_field ? [profile.entity_field] : []));
const rows = (payload.fields || []).map(field => { const name=field.name||field.field, type=(field.type||{}).type||'', props=(field.type||{}).properties||[]; return `<div class="field-row"><code>${esc(name)}</code><span>${esc(type)}</span><span>${esc(props.join(', '))}</span><div class="field-controls"><label><input class="profile-entity" type="checkbox" value="${esc(name)}" ${entityFields.has(name)?'checked':''}> Entity</label><label><input type="radio" name="profile_timestamp" value="${esc(name)}" ${profile.timestamp_field===name?'checked':''}> Time</label>${props.includes('enumerable')?`<label><input class="profile-categorical" type="checkbox" value="${esc(name)}" ${(profile.categorical_fields||[]).includes(name)?'checked':''}> Categorical</label>`:''}${props.includes('numeric')?`<label><input class="profile-numeric" type="checkbox" value="${esc(name)}" ${(profile.numeric_fields||[]).includes(name)?'checked':''}> Numeric</label>`:''}</div></div>`; }).join('');
document.getElementById('fieldPicker').innerHTML = rows ? `<div class="field-header"><span>Field</span><span>Type</span><span>Capabilities</span><span>Use In Profile</span></div>${rows}` : esc(payload.error || 'No fields found.');
window.activeProfileStream = streamId;
window.activeProfileTitle = title || streamId;
@@ -294,7 +295,7 @@ document.getElementById('settingsForm').addEventListener('submit', async event =
values.llm_enabled = form.elements.llm_enabled.checked;
values.threat_intel_enabled = form.elements.threat_intel_enabled.checked;
values.graylog_streams = [...document.querySelectorAll('.graylog-stream')].map(item => ({id:item.dataset.id, title:item.dataset.title, enabled:item.checked}));
if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:form.querySelector('[name="profile_entity"]:checked')?.value||'',timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
if (window.activeProfileStream) { let detectors={},fieldWeights={}; try { detectors=form.elements.profile_detectors.value.trim() ? JSON.parse(form.elements.profile_detectors.value) : {}; } catch { document.getElementById('settingsResult').textContent='Detector thresholds must be valid JSON.'; return; } try { fieldWeights=form.elements.profile_field_weights.value.trim() ? JSON.parse(form.elements.profile_field_weights.value) : {}; } catch { document.getElementById('settingsResult').textContent='Field weights must be valid JSON.'; return; } const entityFields=[...form.querySelectorAll('.profile-entity:checked')].map(item=>item.value); if (!entityFields.length) { document.getElementById('settingsResult').textContent='Select at least one Entity field for the active profile.'; return; } const profileTitle=window.activeProfileTitle || window.activeProfileStream; const profile={stream_id:window.activeProfileStream,name:form.elements.profile_name.value.trim() || `${profileTitle} profile`,entity_field:entityFields[0]||'',entity_fields:entityFields,timestamp_field:form.querySelector('[name="profile_timestamp"]:checked')?.value||'timestamp',categorical_fields:[...form.querySelectorAll('.profile-categorical:checked')].map(item=>item.value),numeric_fields:[...form.querySelectorAll('.profile-numeric:checked')].map(item=>item.value),detectors,field_weights:fieldWeights}; values.graylog_stream_profiles=[...(window.streamProfiles||[]).filter(item=>item.stream_id!==profile.stream_id),profile]; }
const savedProfile = window.activeProfileStream ? ` Profile saved for ${window.activeProfileTitle || window.activeProfileStream}.` : ' No profile editor active, so profiles were not changed.';
const response = await fetch('/api/config', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(values)});
document.getElementById('settingsResult').textContent = response.ok ? `Saved enabled streams and global settings.${savedProfile} Monitor applies supported settings on its next cycle.` : 'Could not save configuration.';

10
src/fgai/entities.py
View File

@@ -42,6 +42,16 @@ def profile_entity(event: LogEvent, field: str) -> str:
return str(event.fields.get(field.lower(), "")).strip()
def profile_entities(event: LogEvent, profile: object) -> tuple[str, ...]:
fields = tuple(str(field) for field in getattr(profile, "entity_fields", ()) if field) or (str(getattr(profile, "entity_field", "")),)
values = []
for field in fields:
value = profile_entity(event, field)
if value and value not in values:
values.append(value)
return tuple(values)
def sample_timeline(events: Iterable[LogEvent], *, limit: int = 20) -> list[dict[str, str]]:
samples = [
{

5
src/fgai/monitor.py
View File

@@ -79,7 +79,7 @@ def _stream_coverage(runtime_values: dict[str, object], stream_profiles: dict[st
"enabled": enabled,
"profile": _profile_name(stream_id, stream_titles, profile) if profile else "",
"profile_ready": bool(profile),
"entity_field": str(getattr(profile, "entity_field", "")) if profile else "",
"entity_field": ", ".join(getattr(profile, "entity_fields", ()) or (str(getattr(profile, "entity_field", "")),)) if profile else "",
"tracked_fields": len(getattr(profile, "categorical_fields", ())) + len(getattr(profile, "numeric_fields", ())) if profile else 0,
"ready_fields": ready_fields,
"total_fields": total_fields,
@@ -131,6 +131,7 @@ def build_status(
profile = stream_profiles.get(stream_id)
profile_fields = (
str(getattr(profile, "entity_field", "")),
*tuple(str(field) for field in getattr(profile, "entity_fields", ())),
str(getattr(profile, "timestamp_field", "")),
*tuple(str(field) for field in getattr(profile, "categorical_fields", ())),
*tuple(str(field) for field in getattr(profile, "numeric_fields", ())),
@@ -225,7 +226,7 @@ def build_status(
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status},
"configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],
"stream_coverage": stream_coverage,
"profile_readiness": profile_readiness,
"diagnostics": {

8
src/fgai/stream_profiles.py
View File

@@ -9,6 +9,7 @@ class StreamProfile:
name: str
entity_field: str
timestamp_field: str
entity_fields: tuple[str, ...] = ()
categorical_fields: tuple[str, ...] = ()
numeric_fields: tuple[str, ...] = ()
detectors: dict[str, dict[str, object]] = field(default_factory=dict)
@@ -59,14 +60,17 @@ def parse_profiles(value: object) -> dict[str, StreamProfile]:
if not isinstance(item, dict):
continue
stream_id = str(item.get("stream_id", "")).strip()
entity = str(item.get("entity_field", "")).strip()
entity_fields = tuple(dict.fromkeys(str(field).strip() for field in item.get("entity_fields", []) if str(field).strip())) if isinstance(item.get("entity_fields", []), list) else ()
entity = str(item.get("entity_field", "")).strip() or (entity_fields[0] if entity_fields else "")
if not entity_fields and entity:
entity_fields = (entity,)
timestamp = str(item.get("timestamp_field", "timestamp")).strip()
if not stream_id or not entity:
continue
detectors = _detectors(item.get("detectors", {}))
field_weights = _field_weights(item.get("field_weights", {}))
profiles[stream_id] = StreamProfile(
stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp,
stream_id, str(item.get("name", "")).strip() or stream_id, entity, timestamp, entity_fields,
tuple(str(field) for field in item.get("categorical_fields", []) if field),
tuple(str(field) for field in item.get("numeric_fields", []) if field),
detectors,