Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add value-shape field classification and auto-apply

Browse Source 
Field roles are now inferred from observed values instead of names,
with src/dst disambiguation, value-pattern detector hints, an Ollama
advisor scoped to ambiguous fields only, confidence-gated auto-apply
with manual-field protection, and drift flagging for review
This commit is contained in:
larssand
2026-09-07 21:58:25 +02:00
parent 7b94512abc
commit e35445da4f
22 changed files with 1133 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
tests/test_llm.py
View File

@@ -1,9 +1,46 @@
import unittest
from fgai.llm import _json_object_from_text, compact_dashboard_analysis
from fgai.llm import _compact_profile_suggestion, _json_object_from_text, compact_dashboard_analysis
class LlmTests(unittest.TestCase):
def test_compact_profile_suggestion_without_fields_only_keeps_everything(self):
item = {
"stream_id": "windows",
"stream_name": "Windows",
"events": 10,
"common_fields": [{"field": "username", "coverage": 1.0}, {"field": "hostname", "coverage": 0.9}],
"discovery": {"top_fields": [{"field": "username"}, {"field": "hostname"}], "selected_fields": {}, "rejected_fields": []},
"profile": {"entity_fields": ["username"]},
"field_shapes": {"username": "categorical_low_cardinality", "hostname": "categorical_low_cardinality"},
"field_confidence": {"username": 80, "hostname": 40},
}
compact = _compact_profile_suggestion(item)
self.assertNotIn("fields_to_classify", compact)
self.assertEqual(len(compact["common_fields"]), 2)
def test_compact_profile_suggestion_with_fields_only_trims_to_subset(self):
item = {
"stream_id": "windows",
"stream_name": "Windows",
"events": 10,
"common_fields": [{"field": "username", "coverage": 1.0}, {"field": "hostname", "coverage": 0.9}],
"discovery": {"top_fields": [{"field": "username"}, {"field": "hostname"}], "selected_fields": {}, "rejected_fields": []},
"profile": {"entity_fields": ["username"]},
"field_shapes": {"username": "categorical_low_cardinality", "hostname": "categorical_low_cardinality"},
"field_confidence": {"username": 80, "hostname": 40},
}
compact = _compact_profile_suggestion(item, fields_only=["hostname"])
self.assertEqual(compact["fields_to_classify"], ["hostname"])
self.assertEqual([row["field"] for row in compact["common_fields"]], ["hostname"])
self.assertEqual([row["field"] for row in compact["top_fields"]], ["hostname"])
self.assertEqual(compact["field_shapes"], {"hostname": "categorical_low_cardinality"})
self.assertEqual(compact["field_confidence"], {"hostname": 40})
def test_json_object_from_text_accepts_markdown_wrapped_json(self):
payload = _json_object_from_text(
'```json\n{"profiles":[{"stream_id":"windows","entity_fields":["username"]}]}\n```'