Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add new structure

Browse Source
This commit is contained in:
larssand
2026-09-23 17:06:05 +02:00
parent 7cbec64b0e
commit 8c155ec9cf
8 changed files with 343 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/fgai/dashboard.py
View File

File diff suppressed because one or more lines are too long

42
src/fgai/monitor.py
View File

@@ -5,6 +5,7 @@ import json
import signal
import threading
import time
from collections import defaultdict
from pathlib import Path
from .anomaly import anomaly_summary, detect_source_anomalies
@@ -251,6 +252,40 @@ def _detect_profile_drift(
return drift
CLUSTER_SIZE_DAMPEN_THRESHOLD = 5
CLUSTER_DAMPENED_SCORE_CAP = 25
def _cluster_field_deviations(field_deviations: dict[str, list[dict[str, object]]]) -> None:
"""Recognize the same (detector, field) signature repeating across many
entities as one systemic pattern instead of N independent findings.
Mutates deviations in place: tags cluster_size, and caps score for large
clusters — a signature hitting many entities identically usually means a
detector/profile problem, not N separate critical events. Signature
deliberately excludes stream_id: the incident this guards against (a
misconfigured relationship field) recurred identically across multiple
streams, and that cross-stream repetition is itself part of the pattern.
Already-suppressed deviations don't count toward another entity's cluster
size, so a false-positive rule doesn't inflate the appearance of a pattern.
"""
signature_entities: dict[tuple[str, str], set[str]] = defaultdict(set)
for entity, deviations in field_deviations.items():
for deviation in deviations:
if deviation.get("feedback") in SUPPRESSING_STATUSES:
continue
signature = (str(deviation.get("detector", "")), str(deviation.get("field", "")))
signature_entities[signature].add(entity)
for entity, deviations in field_deviations.items():
for deviation in deviations:
signature = (str(deviation.get("detector", "")), str(deviation.get("field", "")))
cluster_size = len(signature_entities.get(signature, ()))
deviation["cluster_size"] = cluster_size
if cluster_size >= CLUSTER_SIZE_DAMPEN_THRESHOLD:
deviation["score"] = min(int(deviation.get("score", 0) or 0), CLUSTER_DAMPENED_SCORE_CAP)
deviation["cluster_dampened"] = True
def _stream_name(stream_id: str, stream_titles: dict[str, str], profile: object | None = None) -> str:
return stream_titles.get(stream_id) or getattr(profile, "name", "") or stream_id
@@ -524,6 +559,7 @@ def build_status(
deviation["feedback_rule_id"] = match.get("id", "")
if match["status"] in SUPPRESSING_STATUSES:
deviation["score"] = 0
_cluster_field_deviations(field_deviations)
generalization_min_values = int(runtime_values.get("feedback_generalization_min_values", 3) or 3)
feedback_generalization_suggestions = suggest_generalizations(feedback, min_values=generalization_min_values)
anomalies = detect_source_anomalies(events, limit=anomaly_limit, baselines=profiles, field_deviations=field_deviations, feedback=feedback)
@@ -660,6 +696,11 @@ def build_status(
summary["total"] = int(mcp_status.get("aggregate_events", 0) or 0)
summary["raw_sample_total"] = len(events)
summary["aggregate_backed"] = True
profile_shape_warnings = [
{"stream_id": suggestion.get("stream_id", ""), "stream_name": suggestion.get("stream_name", ""), **warning}
for suggestion in profile_suggestions
for warning in suggestion.get("shape_warnings", [])
]
status = {
"status_schema": 2,
"stale": False,
@@ -676,6 +717,7 @@ def build_status(
"profile_drift": profile_drift,
"stream_coverage": stream_coverage,
"profile_suggestions": profile_suggestions,
"profile_shape_warnings": profile_shape_warnings,
"profile_readiness": profile_readiness,
"diagnostics": {
"top_source_ips": top_field_values(events, "srcip", limit=10),

84
src/fgai/profile_suggestions.py
View File

@@ -11,6 +11,68 @@ from .normalization import FIELD_ALIASES
SHAPE_ENTITY_CONFIDENCE = 60
_SHAPE_VETO = {
"entity": {"free_text"},
"categorical": {"free_text"},
"numeric": {"free_text", "categorical_low_cardinality"},
"relationship": {"free_text"},
}
# classify_field_shape() gives free_text a fixed 0.4 fit ratio and no name
# bonus (field_shapes.py), so a genuine free-text verdict tops out around
# 28-43 confidence — well under the 60 that a clean ip/categorical/numeric
# fit reaches. The fallback "nothing fit" case (classify_field_shape's
# `if not scores` branch) can't exceed 15. 20 cleanly separates a real
# free-text fit from that weak fallback while staying reachable.
_SHAPE_VETO_CONFIDENCE = {
"free_text": 20,
}
_DEFAULT_VETO_CONFIDENCE = 60
def _shape_vetoes(field: str, role: str, field_shapes: dict[str, str], field_confidence: dict[str, int]) -> bool:
"""True when observed values confidently contradict a name-matched field's role.
Only vetoes on positive evidence (a confident, wrong-looking shape) —
fields with no shape info yet (insufficient samples) are never vetoed.
"""
shape = field_shapes.get(field)
if shape is None or shape not in _SHAPE_VETO.get(role, set()):
return False
threshold = _SHAPE_VETO_CONFIDENCE.get(shape, _DEFAULT_VETO_CONFIDENCE)
return field_confidence.get(field, 0) >= threshold
def detect_shape_mismatches(existing: object, field_shapes: dict[str, str], field_confidence: dict[str, int]) -> list[dict[str, object]]:
"""Flag an existing stored profile's fields whose current observed shape no
longer fits their assigned role. Flag-only — never rewrites the stored
profile; a human reviews and removes the field from the dashboard warning.
"""
if existing is None:
return []
warnings: list[dict[str, object]] = []
seen: set[tuple[str, str]] = set()
def flag(field: str, role: str) -> None:
key = (field, role)
if not field or key in seen or not _shape_vetoes(field, role, field_shapes, field_confidence):
return
seen.add(key)
warnings.append({
"field": field,
"role": role,
"shape": field_shapes.get(field),
"confidence": field_confidence.get(field, 0),
})
for field in getattr(existing, "categorical_fields", ()):
flag(field, "categorical")
for relationship in getattr(existing, "relationship_fields", ()):
flag(relationship.left, "relationship")
flag(relationship.right, "relationship")
return warnings
ALIAS_CANONICAL_BY_FIELD = {
alias: canonical
for canonical, aliases in FIELD_ALIASES.items()
@@ -392,7 +454,7 @@ def _generic_entity_fields(coverage: Counter[str], unique_values: dict[str, set[
for field, count in coverage.most_common():
if len(output) >= 4:
break
if field in output or field in IGNORED_DISCOVERY_FIELDS:
if field in output or field in IGNORED_DISCOVERY_FIELDS or _shape_vetoes(field, "entity", field_shapes, field_confidence):
continue
shape_match = field_shapes.get(field) == "ip" and field_confidence.get(field, 0) >= SHAPE_ENTITY_CONFIDENCE
if not shape_match and not _looks_like_entity_field(field):
@@ -411,7 +473,7 @@ def _generic_categorical_fields(coverage: Counter[str], unique_values: dict[str,
for field, count in coverage.most_common():
if len(output) >= 12:
break
if field in output or field in IGNORED_DISCOVERY_FIELDS:
if field in output or field in IGNORED_DISCOVERY_FIELDS or _shape_vetoes(field, "categorical", field_shapes, field_confidence):
continue
coverage_ratio = count / max(1, total)
cardinality = len(unique_values[field])
@@ -428,7 +490,7 @@ def _generic_numeric_fields(coverage: Counter[str], numeric_counts: Counter[str]
for field, count in coverage.most_common():
if len(output) >= 8:
break
if field in output or field in IGNORED_DISCOVERY_FIELDS:
if field in output or field in IGNORED_DISCOVERY_FIELDS or _shape_vetoes(field, "numeric", field_shapes, field_confidence):
continue
shape_match = field_shapes.get(field) == "numeric" and field_confidence.get(field, 0) >= SHAPE_ENTITY_CONFIDENCE
if shape_match or (count / max(1, total) >= 0.1 and numeric_counts[field] / max(1, count) >= 0.95):
@@ -479,9 +541,11 @@ def _relationship_candidates(
numeric_fields: list[str],
*,
field_shapes: dict[str, str] | None = None,
field_confidence: dict[str, int] | None = None,
limit: int = 6,
) -> list[dict[str, str]]:
field_shapes = field_shapes or {}
field_confidence = field_confidence or {}
def usable(field: str) -> bool:
return (
bool(field)
@@ -513,6 +577,10 @@ def _relationship_candidates(
if _field_matches(field, {"dstip", "*_ip"}, {"dst", "dest", "destination", "server"})
]
destination_fields.extend(field for field in categorical if field.lower() in {"dstport", "destination_port", "dest_port", "service", "application", "app", "url", "query_domain", "dns_query"})
# Defense in depth: a field can reach here via shared_categorical or a future
# code path that bypasses the name-priority seed veto applied upstream, so
# re-check shape immediately before it can become a relationship target.
destination_fields = [field for field in destination_fields if not _shape_vetoes(field, "categorical", field_shapes, field_confidence)]
# Ambiguous-name tie-break: two ip-shaped entity fields that neither name-matched
# source nor destination are split using which one leans more private (source-like)
@@ -546,6 +614,7 @@ def _relationship_candidates(
if field not in entities
and not _looks_like_time_field(field)
and not any(token in field.lower() for token in ("gl2_", "message_id", "sort_field", "accounted_message_size", "processing_timestamp"))
and not _shape_vetoes(field, "categorical", field_shapes, field_confidence)
]
for left in entities:
for right in behavior_fields[:4]:
@@ -858,17 +927,20 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
field_shapes[field] = shape.primary_shape
field_confidence[field] = shape.confidence
shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields)
entity_fields = _generic_entity_fields(coverage, unique_values, total, _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3), field_shapes=field_shapes, field_confidence=field_confidence)
entity_seed = [field for field in _pick_present(entity_priority, coverage, total, min_ratio=0.02, limit=12 if is_windows else 3) if not _shape_vetoes(field, "entity", field_shapes, field_confidence)]
entity_fields = _generic_entity_fields(coverage, unique_values, total, entity_seed, field_shapes=field_shapes, field_confidence=field_confidence)
timestamp = next(iter(_pick_present(TIME_PRIORITY, coverage, total, min_ratio=0.02, limit=1)), "")
if not timestamp:
timestamp = next((field for field, count in coverage.most_common() if _looks_like_time_field(field) and count / max(1, total) >= 0.02), "timestamp")
categorical_seed = list(dict.fromkeys([*_pick_present(categorical_priority, coverage, total, min_ratio=0.02, limit=10 if is_windows else 8), *shared_categorical]))
categorical_seed = [field for field in categorical_seed if not _shape_vetoes(field, "categorical", field_shapes, field_confidence)]
categorical = _generic_categorical_fields(coverage, unique_values, total, categorical_seed, field_shapes=field_shapes, field_confidence=field_confidence)
numeric_seed = [
field for field in numeric_priority
if coverage[field] and numeric_counts[field] / max(1, coverage[field]) >= 0.8
][:5]
numeric_seed = list(dict.fromkeys([*numeric_seed, *shared_numeric]))
numeric_seed = [field for field in numeric_seed if not _shape_vetoes(field, "numeric", field_shapes, field_confidence)]
numeric = numeric_seed if is_windows else _generic_numeric_fields(coverage, numeric_counts, total, numeric_seed, field_shapes=field_shapes, field_confidence=field_confidence)
if not entity_fields:
entity_fields = [field for field, _count in coverage.most_common() if field not in IGNORED_DISCOVERY_FIELDS][:1]
@@ -912,7 +984,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
detectors.setdefault(name, hint)
if not detectors:
detectors = {"deny_action": {"enabled": True, "minimum": 10, "z_threshold": 3.0}}
relationship_fields = _relationship_candidates(coverage, unique_values, total, entity_fields, categorical, numeric, field_shapes=field_shapes)
relationship_fields = _relationship_candidates(coverage, unique_values, total, entity_fields, categorical, numeric, field_shapes=field_shapes, field_confidence=field_confidence)
high_coverage = [
{"field": field, "coverage": round(count / max(1, total), 2), "unique_values": len(unique_values[field])}
for field, count in coverage.most_common(12)
@@ -939,12 +1011,14 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
],
}
existing = existing_profiles.get(stream_id)
shape_warnings = detect_shape_mismatches(existing, field_shapes, field_confidence)
score = min(100, 30 + len(entity_fields) * 15 + min(20, len(categorical) * 3) + min(15, len(detectors) * 5))
suggestions.append({
"stream_id": stream_id,
"stream_name": names.get(stream_id, stream_id),
"events": total,
"profile_exists": bool(existing),
"shape_warnings": shape_warnings,
"confidence": "high" if score >= 75 and total >= 100 else "medium" if score >= 55 and total >= 20 else "low",
"score": score,
"field_confidence": field_confidence,