diff --git a/README.md b/README.md index 67a6554..7a1a1de 100644 --- a/README.md +++ b/README.md @@ -554,6 +554,19 @@ The monitor refreshes deterministic detections every `FGAI_MONITOR_INTERVAL` sec The dashboard also exposes `Ollama assessment timeout seconds`; raise this when Ollama is running but large multi-stream summaries still time out. +The assessment is mostly built from pre-computed statistical scores and reasons +(deviation scores, burst z-scores, relationship names), not raw log content — +deliberately compacted this way to keep the prompt small for a local, often +GPU-constrained model even when the underlying deployment has hundreds of +streams or millions of daily events. On top of that, the top-scoring field +deviations and incidents each carry a small, bounded number of real message +excerpts (already truncated short), so the note can call out content a +statistical score alone can't surface, such as a suspicious value in a URL or +command-line field. This is intentionally a qualitative addition layered on +top of the statistical findings, not a replacement for them, and it never +feeds back into scoring or suppression — it only changes the wording of the +narrative note. + The script activates `.venv` inside the script process. If you also want your current shell prompt to show the venv, run: ```bash diff --git a/ROADMAP.md b/ROADMAP.md index 8eb9cb5..b56a0fc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -98,6 +98,15 @@ Goal: make findings more accurate before adding more integrations. - [ ] Add baseline maintenance tooling in the dashboard for retention, high-cardinality fields, and database compaction status. - [x] Add observed-field profile recommendations so streams can get suggested entity, time, categorical, numeric, and detector settings. - [x] Add optional Ollama profile advisor with validated JSON output for semantic field mapping. +- [x] Let the dashboard's Ollama analyst note reason about a small, bounded + number of real log message excerpts from the top-scoring field deviations + and incidents, not only pre-computed statistical scores — so it can flag + suspicious *content* (an injection attempt in a URL, an unusual command + line) that structural/statistical detectors can't see. Kept deliberately + narrow (a handful of already-truncated excerpts from already-triaged top + items, not the raw firehose) to stay within a local, GPU-constrained + model's prompt budget, and purely qualitative — it changes the note's + wording only, never scoring or suppression. Acceptance: each finding shows its detector, confidence, baseline sample count, current value, expected value, and a bounded set of raw-event references. diff --git a/src/fgai/llm.py b/src/fgai/llm.py index 158bf43..cf7206f 100644 --- a/src/fgai/llm.py +++ b/src/fgai/llm.py @@ -98,6 +98,7 @@ def _compact_field_deviations(field_deviations: object, *, limit: int = 12) -> l "baseline": item.get("baseline"), "baseline_age_days": item.get("baseline_age_days"), "sample_values": item.get("sample_values", [])[:5] if isinstance(item.get("sample_values"), list) else [], + "sample_messages": _sample_messages(item.get("sample_events"), limit=1), "feedback": item.get("feedback", "unreviewed"), }) return sorted(rows, key=lambda item: int(item.get("score") or 0), reverse=True)[:limit] @@ -192,6 +193,23 @@ def _short_text(value: object, limit: int = 180) -> str: return text if len(text) <= limit else f"{text[:limit].rstrip()}..." +def _sample_messages(sample_events: object, *, limit: int = 2, chars: int = 140) -> list[str]: + """Pull a few short, real message excerpts from sample events for the LLM to + reason about qualitatively, on top of the statistical scores it already gets. + Bounded on purpose (a handful of already-truncated messages per already- + triaged top item, not the raw firehose) to keep the local-model prompt small. + """ + if not isinstance(sample_events, list): + return [] + messages = [] + for item in sample_events[:limit]: + if isinstance(item, dict): + message = _short_text(item.get("message"), chars) + if message: + messages.append(message) + return messages + + def _compact_anomaly(item: object) -> dict[str, object]: if not isinstance(item, dict): return {} @@ -246,6 +264,7 @@ def _compact_incident(item: object) -> dict[str, object]: "state": item.get("lifecycle_status"), "streams": item.get("correlated_streams", [])[:6] if isinstance(item.get("correlated_streams"), list) else [], "evidence": [_short_text(value, 160) for value in item.get("evidence", [])[:4]] if isinstance(item.get("evidence"), list) else [], + "sample_messages": _sample_messages(item.get("timeline"), limit=3), "first_seen": item.get("first_seen"), "last_seen": item.get("last_seen"), "field_deviations": item.get("field_deviations"), @@ -298,6 +317,12 @@ def ollama_dashboard_assessment(analysis: dict[str, object], model: str | None = "task": ( "Write a concise dashboard analyst note. Compare activity across every listed entity, " "identify the most unusual entity or behavior, and state the next investigation step. " + "Some field_deviations and incidents include sample_messages: short excerpts of the actual " + "raw log content behind the finding. Read them for content a statistical score alone can't " + "surface, such as an injection attempt in a URL, an encoded or unusual command line, or a " + "suspicious value pattern — and call that out specifically when present. Treat this as a " + "qualitative addition on top of the statistical findings, not a replacement for them, and " + "do not invent detail beyond what the excerpts actually show. " "Mention policyid=0 as implicit deny/drop, not an editable policy." ), "data": compact, diff --git a/tests/test_llm.py b/tests/test_llm.py index 06fe892..c48dc1b 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -93,6 +93,9 @@ class LlmTests(unittest.TestCase): self.assertEqual(compact["field_deviations"][0]["entity"], "alice") self.assertNotIn("sample_events", compact["field_deviations"][0]) self.assertNotIn("graylog_query", str(compact)) + # A short, bounded message excerpt is deliberately kept so the LLM can + # reason about actual content, not only the statistical score. + self.assertEqual(compact["field_deviations"][0]["sample_messages"], ["very large raw event"]) def test_dashboard_compaction_strips_incident_and_correlation_samples(self): status = { @@ -132,7 +135,10 @@ class LlmTests(unittest.TestCase): self.assertNotIn("samples", compact_text) self.assertNotIn("sample_events", compact_text) self.assertNotIn("graylog_query", compact_text) - self.assertNotIn("raw incident evidence", compact_text) + # Correlation samples and the raw anomaly evidence payload stay stripped + # entirely (no real content exists there today), but an incident's own + # timeline contributes a few bounded message excerpts. + self.assertEqual(compact["incidents"][0]["sample_messages"], ["raw incident evidence"]) self.assertEqual(compact["incidents"][0]["entity"], "10.0.0.5")