Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add new structure

Browse Source
This commit is contained in:
larssand
2026-09-23 17:06:05 +02:00
parent 7cbec64b0e
commit 8c155ec9cf
8 changed files with 343 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
tests/test_profile_suggestions.py
View File

@@ -1,7 +1,8 @@
import unittest
from fgai.logs import parse_log_line
from fgai.profile_suggestions import apply_profile_advice, suggest_stream_profiles
from fgai.profile_suggestions import apply_profile_advice, detect_shape_mismatches, suggest_stream_profiles
from fgai.stream_profiles import RelationshipField, StreamProfile
class ProfileSuggestionTests(unittest.TestCase):
@@ -364,6 +365,84 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("deny_action", detectors)
self.assertEqual(detectors["deny_action"].get("source"), "value_shape")
def test_name_matched_free_text_field_is_vetoed_from_categorical_and_relationships(self):
messages = [
"GET /checkout/cart/item-{0}?ref=promo&session=abc{0} HTTP/1.1 200",
"POST /api/v2/orders/{0}/confirm referrer=partner-{0} status=accepted",
"DELETE /users/{0}/sessions/active reason=timeout-{0} agent=curl/8.1",
"PUT /inventory/warehouse-{0}/restock qty={0} note=manual-adjustment",
]
events = [
parse_log_line(
f"fgai_stream_id=proxy fgai_stream=Proxy srcip=10.0.0.{index % 7} "
f"context='{messages[index % len(messages)].format(index)}' "
f"timestamp=2026-06-29T10:00:{index:02d}Z"
)
for index in range(1, 40)
]
suggestion = suggest_stream_profiles(events)[0]
profile = suggestion["profile"]
self.assertEqual(suggestion["field_shapes"]["context"], "free_text")
self.assertNotIn("context", profile["entity_fields"])
self.assertNotIn("context", profile["categorical_fields"])
relationship_fields = {item["left"] for item in profile["relationship_fields"]} | {
item["right"] for item in profile["relationship_fields"]
}
self.assertNotIn("context", relationship_fields)
self.assertIn("srcip", profile["entity_fields"])
def test_low_cardinality_name_matched_field_is_not_vetoed(self):
events = [
parse_log_line(
f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index % 7} "
f"action={'deny' if index % 2 else 'allow'} timestamp=2026-06-29T10:00:{index:02d}Z"
)
for index in range(1, 30)
]
suggestion = suggest_stream_profiles(events)[0]
profile = suggestion["profile"]
self.assertIn("action", profile["categorical_fields"])
def test_existing_profile_relationship_field_now_free_text_is_flagged(self):
field_shapes = {"context": "free_text", "srcip": "ip"}
field_confidence = {"context": 43, "srcip": 90}
existing = StreamProfile(
stream_id="proxy",
name="Proxy",
entity_field="srcip",
timestamp_field="timestamp",
categorical_fields=("context",),
relationship_fields=(RelationshipField(left="srcip", right="context", name="srcip -> context"),),
)
warnings = detect_shape_mismatches(existing, field_shapes, field_confidence)
flagged = {(item["field"], item["role"]) for item in warnings}
self.assertIn(("context", "categorical"), flagged)
self.assertIn(("context", "relationship"), flagged)
self.assertNotIn(("srcip", "relationship"), flagged)
def test_existing_profile_with_fitting_fields_produces_no_warnings(self):
field_shapes = {"action": "categorical_low_cardinality", "srcip": "ip"}
field_confidence = {"action": 85, "srcip": 90}
existing = StreamProfile(
stream_id="firewall",
name="Firewall",
entity_field="srcip",
timestamp_field="timestamp",
categorical_fields=("action",),
relationship_fields=(RelationshipField(left="srcip", right="action", name="srcip -> action"),),
)
self.assertEqual(detect_shape_mismatches(existing, field_shapes, field_confidence), [])
def test_missing_existing_profile_produces_no_warnings(self):
self.assertEqual(detect_shape_mismatches(None, {}, {}), [])
def test_every_suggestion_carries_field_confidence_and_shapes(self):
events = [
parse_log_line(f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index} dstip=8.8.8.8 dstport=443 action=deny timestamp=2026-06-29T10:00:{index:02d}Z hitcount={index}")