83ffef3570
Completes the last deferred item from ROADMAP §0: entities recognized as the same real-world concept across enabled streams (e.g. srcip in both a firewall and an IIS stream) now share baseline knowledge of entity presence/age and known relationship-target values, instead of each stream learning it cold. Implemented additively (synthetic shared_entity:<group> namespace in the existing baseline.py tables, no schema migration), scoped to entity presence + relationship known-values only — per-stream distribution baselines stay untouched. Also fixes a latent alias-map collision bug found while building this (ALIAS_CANONICAL_BY_FIELD let "context" silently steal "hostname"/"dns_query" from their own canonical groups). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
493 lines
24 KiB
Python
493 lines
24 KiB
Python
import unittest
|
|
|
|
from fgai.logs import parse_log_line
|
|
from fgai.profile_suggestions import ALIAS_CANONICAL_BY_FIELD, apply_profile_advice, detect_shape_mismatches, shared_entity_pools, suggest_stream_profiles
|
|
from fgai.stream_profiles import RelationshipField, StreamProfile
|
|
|
|
|
|
class ProfileSuggestionTests(unittest.TestCase):
|
|
def test_suggests_common_profile_fields_per_stream(self):
|
|
events = [
|
|
parse_log_line(f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index} dstip=8.8.8.8 dstport=443 action=deny timestamp=2026-06-29T10:00:{index:02d}Z hitcount={index}")
|
|
for index in range(1, 25)
|
|
]
|
|
|
|
suggestions = suggest_stream_profiles(events)
|
|
profile = suggestions[0]["profile"]
|
|
|
|
self.assertEqual(profile["stream_id"], "firewall")
|
|
self.assertIn("srcip", profile["entity_fields"])
|
|
self.assertEqual(profile["timestamp_field"], "eventtime")
|
|
self.assertIn("action", profile["categorical_fields"])
|
|
self.assertIn("dstport", profile["categorical_fields"])
|
|
self.assertIn("hitcount", profile["numeric_fields"])
|
|
self.assertIn("deny_action", profile["detectors"])
|
|
|
|
def test_placeholder_winlog_fields_do_not_make_firewall_windows(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=firewall fgai_stream='Firewall Logs' srcip=10.0.0.{index} dstip=8.8.8.8 "
|
|
f"dstport=443 action=deny policyid=12 eventid=- winlog.channel=- winlog.event_id=- "
|
|
f"winlog.event_data.targetusername=- hostname=- username=- timestamp=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 25)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
profile = suggestion["profile"]
|
|
|
|
self.assertEqual(suggestion["detected_log_type"], "firewall")
|
|
self.assertNotIn("username", profile["entity_fields"])
|
|
self.assertNotIn("hostname", profile["entity_fields"])
|
|
self.assertIn("srcip", profile["entity_fields"])
|
|
self.assertIn("policyid", profile["categorical_fields"])
|
|
self.assertIn("deny_action", profile["detectors"])
|
|
|
|
def test_suggests_windows_event_fields(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=windows fgai_stream='Illuminate:Windows Security Event Log Messages' winlog_event_id=4625 "
|
|
f"winlog_event_data_targetusername=alice winlog_computer_name=host{index % 3} "
|
|
f"event_provider=Microsoft-Windows-Security-Auditing channel=Security logon_type=3 "
|
|
f"winlog_event_data_ipaddress=10.0.0.{index % 5} "
|
|
f"eventtime=2026-06-29T10:00:{index:02d}Z action=failure"
|
|
)
|
|
for index in range(1, 30)
|
|
]
|
|
|
|
profile = suggest_stream_profiles(events)[0]["profile"]
|
|
|
|
self.assertIn("username", profile["entity_fields"])
|
|
self.assertIn("hostname", profile["entity_fields"])
|
|
self.assertIn("srcip", profile["entity_fields"])
|
|
self.assertNotIn("winlog_event_data_targetusername", profile["entity_fields"])
|
|
self.assertIn("eventid", profile["categorical_fields"])
|
|
self.assertIn("logon_type", profile["categorical_fields"])
|
|
self.assertIn("event_provider", profile["categorical_fields"])
|
|
self.assertNotIn("policyid", profile["categorical_fields"])
|
|
self.assertNotIn("dstport", profile["categorical_fields"])
|
|
self.assertNotIn("srcip", profile["categorical_fields"])
|
|
self.assertNotIn("eventid", profile["numeric_fields"])
|
|
self.assertIn("auth_failure", profile["detectors"])
|
|
|
|
def test_windows_named_stream_gets_auth_profile_even_before_failures(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=winsec fgai_stream='Illuminate:Windows Security Event Log Messages' "
|
|
f"eventid=4624 username=user{index % 4} hostname=host{index % 2} channel=Security "
|
|
f"eventtime=2026-06-29T10:00:{index:02d}Z action=success"
|
|
)
|
|
for index in range(1, 30)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
profile = suggestion["profile"]
|
|
|
|
self.assertEqual(suggestion["stream_name"], "Illuminate:Windows Security Event Log Messages")
|
|
self.assertIn("username", profile["entity_fields"])
|
|
self.assertIn("hostname", profile["entity_fields"])
|
|
self.assertIn("eventid", profile["categorical_fields"])
|
|
self.assertIn("auth_failure", profile["detectors"])
|
|
|
|
def test_winlogbeat_ecs_fields_build_windows_profile(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=winbeat fgai_stream='Winlogbeat Security' "
|
|
f"@timestamp=2026-06-30T10:00:{index:02d}Z event.code=4625 event.action=logon-failed "
|
|
f"event.outcome=failure event.category=authentication event.type=start "
|
|
f"user.name=user{index % 4} host.name=win{index % 3} source.ip=10.0.0.{index % 5} "
|
|
f"winlog.channel=Security winlog.provider_name=Microsoft-Windows-Security-Auditing "
|
|
f"winlog.event_data.LogonType=3 process.name=proc{index % 3}.exe"
|
|
)
|
|
for index in range(1, 30)
|
|
]
|
|
|
|
profile = suggest_stream_profiles(events)[0]["profile"]
|
|
|
|
self.assertEqual(profile["timestamp_field"], "eventtime")
|
|
self.assertEqual(profile["entity_fields"][:3], ["username", "hostname", "srcip"])
|
|
self.assertIn("eventid", profile["categorical_fields"])
|
|
self.assertIn("action", profile["categorical_fields"])
|
|
self.assertIn("event.category", profile["categorical_fields"])
|
|
self.assertIn("winlog.channel", profile["categorical_fields"])
|
|
self.assertIn("process.name", profile["categorical_fields"])
|
|
self.assertNotIn("user.name", profile["categorical_fields"])
|
|
self.assertNotIn("source.ip", profile["categorical_fields"])
|
|
self.assertIn("auth_failure", profile["detectors"])
|
|
|
|
def test_discovers_non_priority_application_fields(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=app fgai_stream=BillingApp tenant_id=t{index % 4} actor_id=user{index % 7} "
|
|
f"workflow_state={'approved' if index % 2 else 'rejected'} payment_provider=stripe risk_points={index % 9} "
|
|
f"created_at=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 40)
|
|
]
|
|
|
|
profile = suggest_stream_profiles(events)[0]["profile"]
|
|
|
|
self.assertIn("username", profile["entity_fields"])
|
|
self.assertIn("tenant_id", profile["categorical_fields"])
|
|
self.assertIn("workflow_state", profile["categorical_fields"])
|
|
self.assertIn("risk_points", profile["numeric_fields"])
|
|
|
|
def test_profile_discovery_explains_selected_and_rejected_fields(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=app fgai_stream=BillingApp tenant_id=t{index % 4} actor_id=user{index % 7} "
|
|
f"workflow_state={'approved' if index % 2 else 'rejected'} request_id=req-{index} "
|
|
f"constant_field=same raw='{{json}}' risk_points={index % 9} created_at=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 40)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
discovery = suggestion["discovery"]
|
|
|
|
self.assertEqual(suggestion["detected_log_type"], "application")
|
|
self.assertEqual(discovery["log_type"], "application")
|
|
self.assertIn("actor_id", discovery["selected_fields"]["entity"])
|
|
self.assertIn("workflow_state", discovery["selected_fields"]["categorical"])
|
|
self.assertIn("risk_points", discovery["selected_fields"]["numeric"])
|
|
rejected = {item["field"]: item["reason"] for item in discovery["rejected_fields"]}
|
|
self.assertEqual(rejected["raw"], "raw/internal field")
|
|
self.assertEqual(rejected["constant_field"], "constant value")
|
|
self.assertEqual(rejected["request_id"], "too high cardinality")
|
|
|
|
def test_fields_shared_between_streams_are_default_profile_candidates(self):
|
|
events = []
|
|
for stream_id, stream_name in (("billing", "BillingApp"), ("orders", "OrdersApp")):
|
|
events.extend(
|
|
parse_log_line(
|
|
f"fgai_stream_id={stream_id} fgai_stream={stream_name} actor_id=user{index % 6} "
|
|
f"tenant_id=t{index % 3} workflow_state={'approved' if index % 2 else 'rejected'} "
|
|
f"result_code=r{index % 4} latency_ms={index % 9} created_at=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 30)
|
|
)
|
|
|
|
suggestions = {item["stream_id"]: item for item in suggest_stream_profiles(events)}
|
|
|
|
for suggestion in suggestions.values():
|
|
profile = suggestion["profile"]
|
|
shared = {item["field"] for item in suggestion["shared_fields"]}
|
|
self.assertIn("workflow_state", shared)
|
|
self.assertIn("result_code", shared)
|
|
self.assertIn("latency_ms", shared)
|
|
self.assertIn("workflow_state", profile["categorical_fields"])
|
|
self.assertIn("result_code", profile["categorical_fields"])
|
|
self.assertIn("latency_ms", profile["numeric_fields"])
|
|
|
|
def test_semantic_alias_fields_are_shared_across_streams(self):
|
|
events = []
|
|
events.extend(
|
|
parse_log_line(
|
|
f"fgai_stream_id=proxy fgai_stream=Proxy source.ip=10.0.0.{index % 7} url=/item/{index % 3} action=allow"
|
|
)
|
|
for index in range(1, 30)
|
|
)
|
|
events.extend(
|
|
parse_log_line(
|
|
f"fgai_stream_id=firewall fgai_stream=Firewall client_ip=10.0.0.{index % 7} dstport=443 action=accept"
|
|
)
|
|
for index in range(1, 30)
|
|
)
|
|
|
|
suggestions = {item["stream_id"]: item for item in suggest_stream_profiles(events)}
|
|
|
|
for suggestion in suggestions.values():
|
|
shared = {item["field"]: item for item in suggestion["shared_fields"]}
|
|
self.assertIn("srcip", shared)
|
|
self.assertEqual(shared["srcip"]["kind"], "entity")
|
|
self.assertIn("source.ip", shared["srcip"]["aliases"])
|
|
self.assertIn("client_ip", shared["srcip"]["aliases"])
|
|
|
|
def test_custom_namespace_and_suffix_fields_are_shared_across_streams(self):
|
|
events = []
|
|
for stream_id, field in (("app1", "lcs_customer_id"), ("app2", "lcs_order_id")):
|
|
events.extend(
|
|
parse_log_line(
|
|
f"fgai_stream_id={stream_id} fgai_stream={stream_id} {field}=id{index % 4} result_state=ok actor_name=user{index % 3}"
|
|
)
|
|
for index in range(1, 20)
|
|
)
|
|
|
|
suggestions = {item["stream_id"]: item for item in suggest_stream_profiles(events)}
|
|
|
|
for suggestion in suggestions.values():
|
|
shared = {item["field"]: item for item in suggestion["shared_fields"]}
|
|
self.assertIn("lcs_*", shared)
|
|
self.assertIn("*_id", shared)
|
|
self.assertIn("*_status", shared)
|
|
self.assertIn("*_name", shared)
|
|
|
|
def test_heuristic_profile_recommends_behavior_relationships(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=windows fgai_stream=Windows username=user{index % 3} srcip=10.0.0.{index % 5} hostname=host{index % 4} action=success eventid=4624"
|
|
)
|
|
for index in range(1, 40)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
relationships = {(item["left"], item["right"]) for item in suggestion["relationship_fields"]}
|
|
|
|
self.assertIn(("username", "srcip"), relationships)
|
|
self.assertIn("username->srcip", suggestion["discovery"]["selected_fields"]["relationships"])
|
|
self.assertEqual(suggestion["profile"]["relationship_fields"], suggestion["relationship_fields"])
|
|
|
|
def test_heuristic_profile_recommends_generic_entity_behavior_relationships(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=app fgai_stream=App entry_context_clientipaddress=10.0.0.{index % 5} entry_context_type={'login' if index % 2 else 'update'} houston_category=cat{index % 3}"
|
|
)
|
|
for index in range(1, 40)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
relationships = {(item["left"], item["right"]) for item in suggestion["relationship_fields"]}
|
|
|
|
self.assertIn(("entry_context_clientipaddress", "entry_context_type"), relationships)
|
|
|
|
def test_applies_valid_llm_advice_and_rejects_unknown_fields(self):
|
|
suggestion = suggest_stream_profiles([
|
|
parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure")
|
|
])[0]
|
|
|
|
advised = apply_profile_advice([suggestion], [{
|
|
"stream_id": "windows",
|
|
"entity_fields": ["username", "not_a_field"],
|
|
"timestamp_field": "eventtime",
|
|
"categorical_fields": ["eventid", "full_message"],
|
|
"numeric_fields": ["missing_number"],
|
|
"detectors": {"auth_failure": {"enabled": True, "minimum": 3, "z_threshold": 2.5}, "made_up": {"enabled": True}},
|
|
"relationship_fields": [{"left": "username", "right": "eventid", "name": "user event"}, {"left": "username", "right": "not_a_field"}],
|
|
"field_roles": {"username": "identity", "not_a_field": "entity"},
|
|
"correlation_roles": {"identity": ["username", "not_a_field"], "asset": ["missing_host"]},
|
|
"reason": "Windows auth fields",
|
|
}])[0]
|
|
|
|
self.assertEqual(advised["profile_advisor"]["status"], "ok")
|
|
self.assertEqual(advised["profile"]["entity_fields"], ["username"])
|
|
self.assertIn("eventid", advised["profile"]["categorical_fields"])
|
|
self.assertNotIn("full_message", advised["profile"]["categorical_fields"])
|
|
self.assertEqual(set(advised["profile"]["detectors"]), {"auth_failure"})
|
|
self.assertEqual(advised["profile"]["relationship_fields"], [{"left": "username", "right": "eventid", "name": "user event"}])
|
|
self.assertEqual(advised["profile_advisor"]["field_roles"], {"username": "identity"})
|
|
self.assertEqual(advised["profile_advisor"]["correlation_roles"], {"identity": ["username"]})
|
|
|
|
def test_scoped_field_roles_patch_only_the_named_fields(self):
|
|
suggestion = suggest_stream_profiles([
|
|
parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure")
|
|
])[0]
|
|
|
|
advised = apply_profile_advice([suggestion], [{
|
|
"stream_id": "windows",
|
|
"field_roles": {"hostname": "entity", "action": "reject"},
|
|
"reason": "scoped advisor pass",
|
|
}])[0]
|
|
|
|
self.assertEqual(advised["profile_advisor"]["status"], "ok")
|
|
self.assertEqual(advised["profile_advisor"]["fields_reclassified"], 2)
|
|
self.assertIn("username", advised["profile"]["entity_fields"])
|
|
self.assertIn("hostname", advised["profile"]["entity_fields"])
|
|
self.assertNotIn("action", advised["profile"]["categorical_fields"])
|
|
|
|
def test_scoped_advisor_never_rejects_the_only_entity_field(self):
|
|
events = [
|
|
parse_log_line(f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index}")
|
|
for index in range(1, 25)
|
|
]
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
self.assertEqual(suggestion["profile"]["entity_fields"], ["x7_field_a"])
|
|
|
|
advised = apply_profile_advice([suggestion], [{
|
|
"stream_id": "custom_vendor",
|
|
"field_roles": {"x7_field_a": "reject"},
|
|
"reason": "scoped advisor pass",
|
|
}])[0]
|
|
|
|
self.assertIn("x7_field_a", advised["profile"]["entity_fields"])
|
|
|
|
def test_missing_llm_advice_keeps_heuristic_profile(self):
|
|
suggestion = suggest_stream_profiles([
|
|
parse_log_line("fgai_stream_id=app fgai_stream=App actor_id=user1 action=login")
|
|
])[0]
|
|
|
|
advised = apply_profile_advice([suggestion], [])[0]
|
|
|
|
self.assertEqual(advised["profile_advisor"]["status"], "heuristic")
|
|
self.assertEqual(advised["profile"], suggestion["profile"])
|
|
|
|
def test_unfamiliar_field_name_selected_as_entity_from_value_shape_alone(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor x7_field_a=203.0.113.{index} "
|
|
f"x7_field_b=some-{index % 5} timestamp=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 25)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
|
|
self.assertIn("x7_field_a", suggestion["profile"]["entity_fields"])
|
|
self.assertEqual(suggestion["field_shapes"]["x7_field_a"], "ip")
|
|
self.assertGreaterEqual(suggestion["field_confidence"]["x7_field_a"], 60)
|
|
|
|
def test_ambiguous_ip_fields_disambiguated_by_private_ratio(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor field_alpha=10.0.0.{index} "
|
|
f"field_beta=8.8.8.{index} timestamp=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 25)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
relationships = suggestion["profile"]["relationship_fields"]
|
|
|
|
pairs = {(item["left"], item["right"]) for item in relationships}
|
|
self.assertIn(("field_alpha", "field_beta"), pairs)
|
|
|
|
def test_deny_action_detector_suggested_from_value_pattern_alone(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=custom_vendor fgai_stream=CustomVendor whatever_field="
|
|
f"{'Denied' if index % 3 else 'Blocked'} src=10.0.0.{index} timestamp=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 25)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
detectors = suggestion["profile"]["detectors"]
|
|
|
|
self.assertIn("deny_action", detectors)
|
|
self.assertEqual(detectors["deny_action"].get("source"), "value_shape")
|
|
|
|
def test_name_matched_free_text_field_is_vetoed_from_categorical_and_relationships(self):
|
|
messages = [
|
|
"GET /checkout/cart/item-{0}?ref=promo&session=abc{0} HTTP/1.1 200",
|
|
"POST /api/v2/orders/{0}/confirm referrer=partner-{0} status=accepted",
|
|
"DELETE /users/{0}/sessions/active reason=timeout-{0} agent=curl/8.1",
|
|
"PUT /inventory/warehouse-{0}/restock qty={0} note=manual-adjustment",
|
|
]
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=proxy fgai_stream=Proxy srcip=10.0.0.{index % 7} "
|
|
f"context='{messages[index % len(messages)].format(index)}' "
|
|
f"timestamp=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 40)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
profile = suggestion["profile"]
|
|
|
|
self.assertEqual(suggestion["field_shapes"]["context"], "free_text")
|
|
self.assertNotIn("context", profile["entity_fields"])
|
|
self.assertNotIn("context", profile["categorical_fields"])
|
|
relationship_fields = {item["left"] for item in profile["relationship_fields"]} | {
|
|
item["right"] for item in profile["relationship_fields"]
|
|
}
|
|
self.assertNotIn("context", relationship_fields)
|
|
self.assertIn("srcip", profile["entity_fields"])
|
|
|
|
def test_low_cardinality_name_matched_field_is_not_vetoed(self):
|
|
events = [
|
|
parse_log_line(
|
|
f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index % 7} "
|
|
f"action={'deny' if index % 2 else 'allow'} timestamp=2026-06-29T10:00:{index:02d}Z"
|
|
)
|
|
for index in range(1, 30)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
profile = suggestion["profile"]
|
|
|
|
self.assertIn("action", profile["categorical_fields"])
|
|
|
|
def test_existing_profile_relationship_field_now_free_text_is_flagged(self):
|
|
field_shapes = {"context": "free_text", "srcip": "ip"}
|
|
field_confidence = {"context": 43, "srcip": 90}
|
|
existing = StreamProfile(
|
|
stream_id="proxy",
|
|
name="Proxy",
|
|
entity_field="srcip",
|
|
timestamp_field="timestamp",
|
|
categorical_fields=("context",),
|
|
relationship_fields=(RelationshipField(left="srcip", right="context", name="srcip -> context"),),
|
|
)
|
|
|
|
warnings = detect_shape_mismatches(existing, field_shapes, field_confidence)
|
|
|
|
flagged = {(item["field"], item["role"]) for item in warnings}
|
|
self.assertIn(("context", "categorical"), flagged)
|
|
self.assertIn(("context", "relationship"), flagged)
|
|
self.assertNotIn(("srcip", "relationship"), flagged)
|
|
|
|
def test_existing_profile_with_fitting_fields_produces_no_warnings(self):
|
|
field_shapes = {"action": "categorical_low_cardinality", "srcip": "ip"}
|
|
field_confidence = {"action": 85, "srcip": 90}
|
|
existing = StreamProfile(
|
|
stream_id="firewall",
|
|
name="Firewall",
|
|
entity_field="srcip",
|
|
timestamp_field="timestamp",
|
|
categorical_fields=("action",),
|
|
relationship_fields=(RelationshipField(left="srcip", right="action", name="srcip -> action"),),
|
|
)
|
|
|
|
self.assertEqual(detect_shape_mismatches(existing, field_shapes, field_confidence), [])
|
|
|
|
def test_missing_existing_profile_produces_no_warnings(self):
|
|
self.assertEqual(detect_shape_mismatches(None, {}, {}), [])
|
|
|
|
def test_every_suggestion_carries_field_confidence_and_shapes(self):
|
|
events = [
|
|
parse_log_line(f"fgai_stream_id=firewall fgai_stream=Firewall srcip=10.0.0.{index} dstip=8.8.8.8 dstport=443 action=deny timestamp=2026-06-29T10:00:{index:02d}Z hitcount={index}")
|
|
for index in range(1, 25)
|
|
]
|
|
|
|
suggestion = suggest_stream_profiles(events)[0]
|
|
|
|
self.assertIn("field_confidence", suggestion)
|
|
self.assertIn("field_shapes", suggestion)
|
|
self.assertIn("srcip", suggestion["field_shapes"])
|
|
self.assertEqual(suggestion["field_shapes"]["srcip"], "ip")
|
|
|
|
def test_alias_canonical_map_prefers_specific_group_over_context_fallback(self):
|
|
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["hostname"], "hostname")
|
|
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["dns_query"], "dns_query")
|
|
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["query_domain"], "dns_query")
|
|
self.assertEqual(ALIAS_CANONICAL_BY_FIELD["qh"], "dns_query")
|
|
|
|
def test_shared_entity_pools_groups_streams_with_the_same_entity_concept(self):
|
|
stream_profiles = {
|
|
"firewall": StreamProfile(stream_id="firewall", name="Firewall", entity_field="srcip", timestamp_field="timestamp", entity_fields=("srcip",)),
|
|
"iis": StreamProfile(stream_id="iis", name="IIS", entity_field="client_ip", timestamp_field="timestamp", entity_fields=("client_ip",)),
|
|
"billing": StreamProfile(stream_id="billing", name="Billing", entity_field="actor_id", timestamp_field="timestamp", entity_fields=("actor_id",)),
|
|
}
|
|
|
|
pools = shared_entity_pools(stream_profiles)
|
|
|
|
self.assertEqual(pools["firewall"]["srcip"], pools["iis"]["client_ip"])
|
|
self.assertTrue(pools["firewall"]["srcip"].startswith("shared_entity:"))
|
|
self.assertNotIn("billing", pools)
|
|
|
|
def test_shared_entity_pools_keeps_source_and_destination_ip_separate(self):
|
|
stream_profiles = {
|
|
"firewall": StreamProfile(stream_id="firewall", name="Firewall", entity_field="srcip", timestamp_field="timestamp", entity_fields=("srcip", "dstip")),
|
|
"proxy": StreamProfile(stream_id="proxy", name="Proxy", entity_field="client_ip", timestamp_field="timestamp", entity_fields=("client_ip", "server_ip")),
|
|
}
|
|
|
|
pools = shared_entity_pools(stream_profiles)
|
|
|
|
self.assertEqual(pools["firewall"]["srcip"], pools["proxy"]["client_ip"])
|
|
self.assertEqual(pools["firewall"]["dstip"], pools["proxy"]["server_ip"])
|
|
self.assertNotEqual(pools["firewall"]["srcip"], pools["firewall"]["dstip"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|