Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add relationship fields

Browse Source
This commit is contained in:
larssand
2026-07-02 15:07:15 +02:00
parent d48d34f96c
commit 7d0e66c239
13 changed files with 437 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
tests/test_baseline.py
View File

@@ -147,3 +147,24 @@ class BaselineTests(unittest.TestCase):
self.assertNotIn("alice", store.profile_deviations(burst, profiles, min_training_days=7))
self.assertIn("alice", store.profile_deviations(burst, profiles, min_training_days=0))
def test_custom_relationship_flags_new_user_source_ip(self):
with tempfile.TemporaryDirectory() as directory:
store = BaselineStore(str(Path(directory) / "baseline.sqlite3"))
profiles = parse_profiles([{
"stream_id": "windows",
"entity_field": "username",
"categorical_fields": ["action"],
"relationship_fields": [{"left": "username", "right": "srcip", "name": "user source IP"}],
}])
for index in range(12):
event = parse_log_line(f"fgai_stream_id=windows username=peter srcip=10.0.0.10 action=success baseline={index}")
store.ingest_profile_fields([event], profiles, observed_at=1_700_000_000 + index * 300)
current = [parse_log_line("fgai_stream_id=windows username=peter srcip=10.0.0.99 action=success current=1")]
deviations = store.profile_deviations(current, profiles)["peter"]
relation = next(item for item in deviations if item["detector"] == "new_relationship")
self.assertEqual(relation["field"], "relationship:username->srcip")
self.assertEqual(relation["value"], "10.0.0.99")
self.assertIn("new srcip value for username=peter", relation["reason"])

19
tests/test_monitor.py
View File

@@ -5,6 +5,7 @@ from pathlib import Path
from unittest.mock import patch
from fgai.history import StatusSnapshotStore
from fgai.history import FieldDiscoveryStore
from fgai.monitor import add_llm_assessment, build_status, cached_status_with_error, write_refreshing_status, write_status
@@ -192,6 +193,24 @@ class MonitorTests(unittest.TestCase):
self.assertIn("lcs_result", suggestion["categorical_fields"])
self.assertGreater(status["baseline"]["discovery_cache_events"], 0)
def test_field_catalog_creates_discovery_events_without_raw_values(self):
with tempfile.TemporaryDirectory() as tmp:
store = FieldDiscoveryStore(str(Path(tmp) / "history.sqlite3"))
store.ingest_catalog(
"app",
"App",
[
{"name": "lcs_customer_id", "type": {"type": "string", "properties": ["enumerable"]}},
{"name": "lcs_duration_ms", "type": {"type": "long", "properties": ["numeric"]}},
],
)
events = store.synthetic_events()
fields = {field for event in events for field in event.fields}
self.assertIn("lcs_customer_id", fields)
self.assertIn("lcs_duration_ms", fields)
def test_cached_status_with_error_keeps_last_good_dashboard_data(self):
with tempfile.TemporaryDirectory() as tmp:
cache_path = str(Path(tmp) / "status-cache.sqlite3")

25
tests/test_profile_suggestions.py
View File

@@ -202,6 +202,25 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("source.ip", shared["srcip"]["aliases"])
self.assertIn("client_ip", shared["srcip"]["aliases"])
def test_custom_namespace_and_suffix_fields_are_shared_across_streams(self):
events = []
for stream_id, field in (("app1", "lcs_customer_id"), ("app2", "lcs_order_id")):
events.extend(
parse_log_line(
f"fgai_stream_id={stream_id} fgai_stream={stream_id} {field}=id{index % 4} result_state=ok actor_name=user{index % 3}"
)
for index in range(1, 20)
)
suggestions = {item["stream_id"]: item for item in suggest_stream_profiles(events)}
for suggestion in suggestions.values():
shared = {item["field"]: item for item in suggestion["shared_fields"]}
self.assertIn("lcs_*", shared)
self.assertIn("*_id", shared)
self.assertIn("*_status", shared)
self.assertIn("*_name", shared)
def test_applies_valid_llm_advice_and_rejects_unknown_fields(self):
suggestion = suggest_stream_profiles([
parse_log_line("fgai_stream_id=windows fgai_stream=Windows username=alice hostname=host01 eventid=4625 action=failure")
@@ -214,6 +233,9 @@ class ProfileSuggestionTests(unittest.TestCase):
"categorical_fields": ["eventid", "full_message"],
"numeric_fields": ["missing_number"],
"detectors": {"auth_failure": {"enabled": True, "minimum": 3, "z_threshold": 2.5}, "made_up": {"enabled": True}},
"relationship_fields": [{"left": "username", "right": "eventid", "name": "user event"}, {"left": "username", "right": "not_a_field"}],
"field_roles": {"username": "identity", "not_a_field": "entity"},
"correlation_roles": {"identity": ["username", "not_a_field"], "asset": ["missing_host"]},
"reason": "Windows auth fields",
}])[0]
@@ -222,6 +244,9 @@ class ProfileSuggestionTests(unittest.TestCase):
self.assertIn("eventid", advised["profile"]["categorical_fields"])
self.assertNotIn("full_message", advised["profile"]["categorical_fields"])
self.assertEqual(set(advised["profile"]["detectors"]), {"auth_failure"})
self.assertEqual(advised["profile"]["relationship_fields"], [{"left": "username", "right": "eventid", "name": "user event"}])
self.assertEqual(advised["profile_advisor"]["field_roles"], {"username": "identity"})
self.assertEqual(advised["profile_advisor"]["correlation_roles"], {"identity": ["username"]})
def test_missing_llm_advice_keeps_heuristic_profile(self):
suggestion = suggest_stream_profiles([

10
tests/test_stream_profiles.py
View File

@@ -25,3 +25,13 @@ class StreamProfileTests(unittest.TestCase):
self.assertEqual(profiles["dns"].field_weights["qh"], 1.5)
self.assertEqual(profiles["dns"].field_weights["query_domain"]["rare_value"], 2.0)
self.assertNotIn("bad", profiles["dns"].field_weights)
def test_parses_relationship_fields(self):
profiles = parse_profiles([{"stream_id": "windows", "entity_field": "username", "relationship_fields": [{"left": "username", "right": "srcip", "name": "user source IP"}, {"left": "username", "right": "srcip"}]}])
relationships = profiles["windows"].relationship_fields
self.assertEqual(len(relationships), 1)
self.assertEqual(relationships[0].left, "username")
self.assertEqual(relationships[0].right, "srcip")
self.assertEqual(relationships[0].name, "user source IP")