Fix cardinality judged against sample size instead of real event volume
classify_field_shape() received a deduplicated set of observed values with no real event-volume context, so cardinality was always measured against itself — trivially 100%, making any field with more than ~16 distinct values look "high cardinality" no matter how large the real traffic volume was. This misclassified genuinely low-cardinality fields (e.g. username: 82 distinct values across 1,601 events) as free text, which the new shape-veto work then surfaced as false-positive "remove this field" warnings in production. Fixed by passing real event coverage as an explicit total_events parameter; omitting it keeps prior behavior for other callers. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -393,6 +393,20 @@ class ProfileSuggestionTests(unittest.TestCase):
|
||||
self.assertNotIn("context", relationship_fields)
|
||||
self.assertIn("srcip", profile["entity_fields"])
|
||||
|
||||
def test_field_shape_classification_uses_real_event_volume_not_sample_size(self):
|
||||
# 24 distinct usernames repeating across 300 events is genuinely low
|
||||
# cardinality (8%) even though 24 distinct values alone would look
|
||||
# "high cardinality" if judged only against the deduplicated sample.
|
||||
events = [
|
||||
parse_log_line(f"fgai_stream_id=app fgai_stream=App username=user{index % 24} action=login timestamp=2026-06-29T10:00:{index % 60:02d}Z")
|
||||
for index in range(1, 301)
|
||||
]
|
||||
|
||||
suggestion = suggest_stream_profiles(events)[0]
|
||||
|
||||
self.assertEqual(suggestion["field_shapes"]["username"], "categorical_low_cardinality")
|
||||
self.assertIn("username", suggestion["profile"]["entity_fields"])
|
||||
|
||||
def test_low_cardinality_name_matched_field_is_not_vetoed(self):
|
||||
events = [
|
||||
parse_log_line(
|
||||
|
||||
Reference in New Issue
Block a user