Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix cardinality judged against sample size instead of real event volume

Browse Source 
classify_field_shape() received a deduplicated set of observed values with
no real event-volume context, so cardinality was always measured against
itself — trivially 100%, making any field with more than ~16 distinct
values look "high cardinality" no matter how large the real traffic volume
was. This misclassified genuinely low-cardinality fields (e.g. username: 82
distinct values across 1,601 events) as free text, which the new shape-veto
work then surfaced as false-positive "remove this field" warnings in
production. Fixed by passing real event coverage as an explicit
total_events parameter; omitting it keeps prior behavior for other callers.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-23 22:02:44 +02:00
parent 83ffef3570
commit caa37de98e
5 changed files with 61 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/fgai/field_shapes.py
View File

@@ -88,11 +88,21 @@ def _name_bonus(field_name: str, shape: str) -> int:
return NAME_BONUS_CAP if any(token in lower for token in tokens) else 0
def classify_field_shape(sample_values, *, field_name: str = "") -> FieldShape:
def classify_field_shape(sample_values, *, field_name: str = "", total_events: int | None = None) -> FieldShape:
"""Classify a field's semantic shape from its observed values.
Value shape is the primary signal; ``field_name`` only contributes a small,
capped bonus that can nudge an already-plausible shape, never manufacture one.
``sample_values`` is commonly a deduplicated set of distinct observed values
(callers cap how many distinct values they remember per field), so its own
length is not a usable stand-in for "how many events had this field" — a
deduplicated collection is trivially 100% cardinality against itself. Pass
the real event count as ``total_events`` when available (for example
``coverage[field]``) so cardinality is judged against real traffic volume;
otherwise this falls back to treating the sample itself as the population,
which under-detects genuinely low-cardinality fields once they have more
than a handful of distinct values.
"""
values = [str(item).strip() for item in sample_values if str(item).strip()]
if not values:
@@ -100,7 +110,8 @@ def classify_field_shape(sample_values, *, field_name: str = "") -> FieldShape:
total = len(values)
cardinality = len(set(values))
low_cardinality = cardinality <= max(10, int(total * 0.5))
population = total_events if total_events and total_events > total else total
low_cardinality = cardinality <= max(10, int(population * 0.5))
ip_ratio = sum(_is_ip(v) for v in values) / total
timestamp_ratio = sum(_is_timestamp(v) for v in values) / total
@@ -109,8 +120,8 @@ def classify_field_shape(sample_values, *, field_name: str = "") -> FieldShape:
non_numeric_ratio = 1 - numeric_ratio
avg_words = sum(len(v.split()) for v in values) / total
multiword = avg_words >= 3
high_cardinality = cardinality > max(8, int(total * 0.5))
categorical_ratio = non_numeric_ratio if (1 < cardinality <= max(8, int(total * 0.5)) and not multiword) else 0.0
high_cardinality = cardinality > max(8, int(population * 0.5))
categorical_ratio = non_numeric_ratio if (1 < cardinality <= max(8, int(population * 0.5)) and not multiword) else 0.0
free_text_ratio = 0.4 if ((high_cardinality or multiword) and numeric_ratio < 0.5) else 0.0
fit_by_shape = {

2
src/fgai/profile_suggestions.py
View File

@@ -957,7 +957,7 @@ def suggest_stream_profiles(events: list[LogEvent], *, existing_profiles: dict[s
for field in coverage:
if field in IGNORED_DISCOVERY_FIELDS:
continue
shape = classify_field_shape(unique_values[field], field_name=field)
shape = classify_field_shape(unique_values[field], field_name=field, total_events=coverage[field])
field_shapes[field] = shape.primary_shape
field_confidence[field] = shape.confidence
shared_categorical, shared_numeric, shared_stats = _shared_profile_fields(coverage, unique_values, numeric_counts, total, stream_field_counts, stream_semantic_counts, stream_semantic_fields)