Public/fgAI
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

keep discoverds fileds in separate sqlite

Browse Source
This commit is contained in:
larssand
2026-07-02 12:51:36 +02:00
parent cc6c97ec79
commit d48d34f96c
5 changed files with 134 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/fgai/monitor.py
View File

@@ -13,7 +13,7 @@ from .feedback import FeedbackStore
from .graylog_aggregate import GraylogAggregateSource
from .graylog_mcp import GraylogMcpClient
from .graylog_source import GraylogStreamSource
from .history import HistoryStore, StatusSnapshotStore
from .history import FieldDiscoveryStore, HistoryStore, StatusSnapshotStore
from .incidents import IncidentStore, build_incidents
from .data_quality import assess_data_quality
from .llm import ollama_dashboard_assessment, ollama_profile_advice
@@ -262,7 +262,13 @@ def build_status(
for item in profile_readiness
]
stream_coverage = _stream_coverage(runtime_values, stream_profiles, mcp_status, profile_readiness, stream_titles)
profile_suggestions = suggest_stream_profiles(events, existing_profiles=stream_profiles)
discovery_cache_events = []
discovered_profile_fields = 0
if history_path:
discovery_store = FieldDiscoveryStore(history_path)
discovered_profile_fields = discovery_store.ingest(events)
discovery_cache_events = discovery_store.synthetic_events()
profile_suggestions = suggest_stream_profiles([*discovery_cache_events, *events], existing_profiles=stream_profiles)
profile_advisor_status = {"enabled": bool(runtime_values.get("profile_advisor_enabled")), "status": "disabled"}
if runtime_values.get("profile_advisor_enabled") and profile_suggestions:
try:
@@ -329,7 +335,7 @@ def build_status(
"policy_path": policy_path,
"summary": summary,
"anomaly_summary": anomaly_summary(anomalies),
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
"baseline": {"enabled": bool(baseline), "sources_ready": len(profiles), "training_days": baseline_training_days, "new_events_recorded": baseline_events, "profile_fields_recorded": profile_baseline_fields, "discovery_fields_recorded": discovered_profile_fields, "discovery_cache_events": len(discovery_cache_events), "maintenance": baseline_maintenance, "size_bytes": baseline_maintenance.get("size_bytes", 0) if isinstance(baseline_maintenance, dict) else 0},
"capabilities": {"threat_intel": threat_intel_status, "graylog_mcp": mcp_status, "profile_advisor": profile_advisor_status},
"configuration": runtime_config,
"stream_profiles": [{"stream_id": item.stream_id, "name": _profile_name(item.stream_id, stream_titles, item), "stream_name": _stream_name(item.stream_id, stream_titles, item), "entity_field": item.entity_field, "entity_fields": list(item.entity_fields), "timestamp_field": item.timestamp_field, "categorical_fields": list(item.categorical_fields), "numeric_fields": list(item.numeric_fields), "detectors": item.detectors, "field_weights": item.field_weights} for item in stream_profiles.values()],