diff --git a/README.md b/README.md index 6191b9f..c1c34ac 100644 --- a/README.md +++ b/README.md @@ -252,12 +252,16 @@ even when it was not selected into the applied profile. A name match never overrides value shape, even for names on the trusted priority lists such as `context`. If the observed values for a name-matched field confidently look like free text (a raw log line or URL, not a -low-cardinality category) it is excluded from entity, categorical, and -relationship-target selection regardless of its name, both for a fresh -recommendation and as a second, independent check when a field reaches -relationship-candidate selection through cross-stream field sharing. A field -with too few samples to classify is never excluded on that basis alone; the -veto only fires on a confident, contradicting shape. +low-cardinality category) or like a timestamp, it is excluded from entity, +categorical, and relationship-target selection regardless of its name, both +for a fresh recommendation and as a second, independent check when a field +reaches relationship-candidate selection through cross-stream field sharing. +The timestamp case matters for the same reason free text does: a timestamp +is near-unique on essentially every event, so tracking it as a categorical +or relationship field is guaranteed to produce a "never seen before" finding +every single time, exactly the failure mode that made `context` a problem. +A field with too few samples to classify is never excluded on that basis +alone; the veto only fires on a confident, contradicting shape. Windows-like streams are recognized from stream names such as `Windows`, `Winlog`, `Security Event Log`, `Sysmon`, and `Powershell`, or from Windows event diff --git a/src/fgai/profile_suggestions.py b/src/fgai/profile_suggestions.py index eedb913..0a82cfe 100644 --- a/src/fgai/profile_suggestions.py +++ b/src/fgai/profile_suggestions.py @@ -12,10 +12,10 @@ from .normalization import FIELD_ALIASES SHAPE_ENTITY_CONFIDENCE = 60 _SHAPE_VETO = { - "entity": {"free_text"}, - "categorical": {"free_text"}, + "entity": {"free_text", "timestamp"}, + "categorical": {"free_text", "timestamp"}, "numeric": {"free_text", "categorical_low_cardinality"}, - "relationship": {"free_text"}, + "relationship": {"free_text", "timestamp"}, } # classify_field_shape() gives free_text a fixed 0.4 fit ratio and no name diff --git a/tests/test_profile_suggestions.py b/tests/test_profile_suggestions.py index 0732f90..24bf587 100644 --- a/tests/test_profile_suggestions.py +++ b/tests/test_profile_suggestions.py @@ -393,6 +393,28 @@ class ProfileSuggestionTests(unittest.TestCase): self.assertNotIn("context", relationship_fields) self.assertIn("srcip", profile["entity_fields"]) + def test_timestamp_shaped_field_is_vetoed_from_categorical_and_relationships(self): + events = [ + parse_log_line( + f"fgai_stream_id=squid fgai_stream=Squid srcip=10.0.0.{index % 7} " + f"gl2_processing_finished=2026-06-29T10:{index % 60:02d}:00Z " + f"action=allow timestamp=2026-06-29T10:00:{index % 60:02d}Z" + ) + for index in range(1, 40) + ] + + suggestion = suggest_stream_profiles(events)[0] + profile = suggestion["profile"] + + self.assertEqual(suggestion["field_shapes"]["gl2_processing_finished"], "timestamp") + self.assertNotIn("gl2_processing_finished", profile["entity_fields"]) + self.assertNotIn("gl2_processing_finished", profile["categorical_fields"]) + relationship_fields = {item["left"] for item in profile["relationship_fields"]} | { + item["right"] for item in profile["relationship_fields"] + } + self.assertNotIn("gl2_processing_finished", relationship_fields) + self.assertIn("srcip", profile["entity_fields"]) + def test_field_shape_classification_uses_real_event_volume_not_sample_size(self): # 24 distinct usernames repeating across 300 events is genuinely low # cardinality (8%) even though 24 distinct values alone would look