Fortsatte roadmapen med multi-entity stream profiles
This commit is contained in:
larssand
@@ -10,7 +10,7 @@ from statistics import mean, pstdev
|
||||
|
||||
from .logs import THREAT_ACTIONS, is_utm_event
|
||||
from .models import LogEvent
|
||||
from .entities import profile_entity
|
||||
from .entities import profile_entities
|
||||
from .detectors import DETECTOR_MINIMUMS, event_detector_categories
|
||||
from .normalization import canonical_value
|
||||
|
||||
@@ -180,31 +180,32 @@ class BaselineStore:
|
||||
fingerprint = hashlib.sha256(f"{stream_id}|{event.raw}".encode("utf-8", errors="replace")).hexdigest()
|
||||
if connection.execute("insert or ignore into profile_seen_events values (?)", (fingerprint,)).rowcount != 1:
|
||||
continue
|
||||
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
|
||||
if not entity:
|
||||
entities = profile_entities(event, profile)
|
||||
if not entities:
|
||||
continue
|
||||
timestamp = _event_epoch(event, observed_at)
|
||||
bucket = timestamp - (timestamp % self.bucket_seconds)
|
||||
moment = datetime.fromtimestamp(timestamp, tz=timezone.utc)
|
||||
for detector in event_detector_categories(event):
|
||||
detector_pending[(stream_id, entity, detector, bucket)] += 1
|
||||
detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
|
||||
fields = [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]
|
||||
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
|
||||
for field in fields:
|
||||
key = (stream_id, entity, str(field).lower(), bucket)
|
||||
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
|
||||
pending[key][0] += 1
|
||||
pending[key][1] += value
|
||||
pending[key][2] += value * value
|
||||
temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
|
||||
temporal_pending[temporal][0] += 1
|
||||
temporal_pending[temporal][1] += value
|
||||
temporal_pending[temporal][2] += value * value
|
||||
if key[2] not in numeric:
|
||||
raw_value = event.fields.get(key[2])
|
||||
if raw_value:
|
||||
pending_values[(*key[:3], raw_value)] += 1
|
||||
for entity in entities:
|
||||
for detector in event_detector_categories(event):
|
||||
detector_pending[(stream_id, entity, detector, bucket)] += 1
|
||||
detector_temporal_pending[(stream_id, entity, detector, moment.weekday(), moment.hour, bucket)] += 1
|
||||
for field in fields:
|
||||
key = (stream_id, entity, str(field).lower(), bucket)
|
||||
value = _number(event.fields.get(key[2])) if key[2] in numeric else 0
|
||||
pending[key][0] += 1
|
||||
pending[key][1] += value
|
||||
pending[key][2] += value * value
|
||||
temporal = (*key[:3], moment.weekday(), moment.hour, bucket)
|
||||
temporal_pending[temporal][0] += 1
|
||||
temporal_pending[temporal][1] += value
|
||||
temporal_pending[temporal][2] += value * value
|
||||
if key[2] not in numeric:
|
||||
raw_value = event.fields.get(key[2])
|
||||
if raw_value:
|
||||
pending_values[(*key[:3], raw_value)] += 1
|
||||
for (stream_id, entity, field, bucket), values in pending.items():
|
||||
connection.execute("""insert into profile_buckets values (?, ?, ?, ?, ?, ?, ?)
|
||||
on conflict(stream_id, entity, field, bucket_start) do update set events=events+excluded.events,numeric_sum=numeric_sum+excluded.numeric_sum,numeric_sum_squares=numeric_sum_squares+excluded.numeric_sum_squares""", (stream_id, entity, field, bucket, *values))
|
||||
@@ -230,23 +231,24 @@ class BaselineStore:
|
||||
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
|
||||
if not profile:
|
||||
continue
|
||||
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
|
||||
if not entity:
|
||||
entities = profile_entities(event, profile)
|
||||
if not entities:
|
||||
continue
|
||||
entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
|
||||
for detector in event_detector_categories(event):
|
||||
detector_current[(event.fields.get("fgai_stream_id", ""), entity, detector)] += 1
|
||||
numeric = {str(field).lower() for field in getattr(profile, "numeric_fields", ())}
|
||||
for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
|
||||
key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
|
||||
current[key][0] += 1
|
||||
if key[2] in numeric:
|
||||
current[key][1] += _number(event.fields.get(key[2]))
|
||||
for entity in entities:
|
||||
entity_events[(event.fields.get("fgai_stream_id", ""), entity)].append(event)
|
||||
for detector in event_detector_categories(event):
|
||||
detector_current[(event.fields.get("fgai_stream_id", ""), entity, detector)] += 1
|
||||
for field in [*getattr(profile, "categorical_fields", ()), *getattr(profile, "numeric_fields", ())]:
|
||||
key = (event.fields.get("fgai_stream_id", ""), entity, str(field).lower())
|
||||
current[key][0] += 1
|
||||
if key[2] in numeric:
|
||||
current[key][1] += _number(event.fields.get(key[2]))
|
||||
output: dict[str, list[dict[str, object]]] = defaultdict(list)
|
||||
with self._connect() as connection:
|
||||
for (stream, entity, field), values in current.items():
|
||||
profile = profiles.get(stream)
|
||||
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and profile_entity(event, str(getattr(profiles.get(stream), "entity_field", ""))) == entity and event.fields.get(field)]
|
||||
matching = [event for event in events if event.fields.get("fgai_stream_id") == stream and entity in profile_entities(event, profiles.get(stream)) and event.fields.get(field)]
|
||||
current_timestamp = _event_epoch(matching[-1], int(time.time())) if matching else int(time.time())
|
||||
moment = datetime.fromtimestamp(current_timestamp, tz=timezone.utc)
|
||||
rows = connection.execute("select events, numeric_sum from profile_temporal_buckets where stream_id=? and entity=? and field=? and weekday=? and hour=? order by bucket_start desc limit 25", (stream, entity, field, moment.weekday(), moment.hour)).fetchall()
|
||||
@@ -336,23 +338,24 @@ class BaselineStore:
|
||||
profile = profiles.get(event.fields.get("fgai_stream_id", ""))
|
||||
if not profile:
|
||||
continue
|
||||
entity = profile_entity(event, str(getattr(profile, "entity_field", "")))
|
||||
if not entity:
|
||||
entities = profile_entities(event, profile)
|
||||
if not entities:
|
||||
continue
|
||||
stream = event.fields.get("fgai_stream_id", "")
|
||||
for field in getattr(profile, "categorical_fields", ()):
|
||||
field = str(field).lower()
|
||||
value = event.fields.get(field)
|
||||
if not value:
|
||||
continue
|
||||
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
|
||||
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
|
||||
if known is None and known_total >= 30:
|
||||
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and profile_entity(item, str(getattr(profile, "entity_field", ""))) == entity and item.fields.get(field) == value]
|
||||
score, weight = _weighted_score(12, profile, field, "rare_value")
|
||||
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
|
||||
if evidence not in output[entity]:
|
||||
output[entity].append(evidence)
|
||||
for entity in entities:
|
||||
for field in getattr(profile, "categorical_fields", ()):
|
||||
field = str(field).lower()
|
||||
value = event.fields.get(field)
|
||||
if not value:
|
||||
continue
|
||||
known = connection.execute("select seen_count from profile_values where stream_id=? and entity=? and field=? and value=?", (stream, entity, field, value)).fetchone()
|
||||
known_total = connection.execute("select coalesce(sum(seen_count), 0) from profile_values where stream_id=? and entity=? and field=?", (stream, entity, field)).fetchone()[0]
|
||||
if known is None and known_total >= 30:
|
||||
samples = [item for item in events if item.fields.get("fgai_stream_id") == stream and entity in profile_entities(item, profile) and item.fields.get(field) == value]
|
||||
score, weight = _weighted_score(12, profile, field, "rare_value")
|
||||
evidence = {"detector": "rare_value", "field": field, "stream_id": stream, "score": score, "base_score": 12, "weight": weight, "confidence": "medium", "baseline_samples": int(known_total), "baseline_scope": "known field values", "reason": f"new {field} value for this entity", "value": value, "sample_values": [value], "sample_events": [_sample_event(item, value) for item in samples[:5]]}
|
||||
if evidence not in output[entity]:
|
||||
output[entity].append(evidence)
|
||||
return output
|
||||
|
||||
def profile_readiness(self, profiles: dict[str, object]) -> list[dict[str, object]]:
|
||||
|
||||
Reference in New Issue
Block a user