diff --git a/.harness/scripts/ci/68-validate-engine-verdict-parity.mjs b/.harness/scripts/ci/68-validate-engine-verdict-parity.mjs index 8782e1bd2..746b83370 100644 --- a/.harness/scripts/ci/68-validate-engine-verdict-parity.mjs +++ b/.harness/scripts/ci/68-validate-engine-verdict-parity.mjs @@ -54,6 +54,19 @@ * divergence) and equally when a baselined id no longer conflicts (a stale * exemption, which is how a baseline rots into permission). * + * ## The tree that is measured is the COMMITTED one (GT-716 AC5) + * + * Until AC5 this guard ran the CLI from the repository root without `--core`. On + * that run the native engine resolved the corpus to the CLI's bundled copy and + * FAILED the 138 generated ADR-conformance rules (their referenced ADRs are not in + * the copy), the exit-code handler doubled its own root (CLI-EXIT-01/03), and the + * baseline registered those artifacts as verdict conflicts. Both engines now read + * the Core from an export of the tracked files plus the compiled bundle + * (`lib/core-export.mjs`, the same export guard 73 measures) with `--core` + * pointed at it — the same corpus, on every machine, for both engines. What + * remains in the baseline is a disagreement about a rule, not about where a run + * found its corpus. + * * ## Anti-vacuous pass * * Both engine runs are asserted through `assertScannedPerSource`, so an engine that @@ -70,20 +83,19 @@ * 1 - a new conflict, a stale baseline entry, or an engine that produced nothing */ -import { existsSync, readFileSync } from 'node:fs'; +import { existsSync, readFileSync, rmSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { REPO_ROOT } from '../lib/paths.mjs'; +import { CLI_ENTRY, WASM_CANDIDATES, exportCore } from '../lib/core-export.mjs'; import { assertScannedPerSource, ZeroCoverageError } from '../lib/coverage.mjs'; import { diffDecisions } from './parity-gate.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); export const BASELINE_PATH = resolve(HERE, 'engine-verdict-parity.baseline.json'); -const CLI_ENTRY = 'src/sdk/cli/dist/main.js'; -const WASM_CANDIDATES = ['src/rulesets/opa/policy.wasm', 'src/sdk/cli/rulesets/opa/policy.wasm']; const ENGINES = ['native', 'opa']; /** Outcomes that mean "this engine reached a verdict about the rule". */ @@ -173,12 +185,12 @@ export function reconcileBaseline(conflicts, baseline) { } /** Run one engine over the whole corpus and return its parsed report. */ -function runEngine(engine, root) { +function runEngine(engine, root, core) { const started = Date.now(); const proc = spawnSync( process.execPath, - [CLI_ENTRY, 'validate', '--engine', engine, '--format', 'json'], - { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + [resolve(root, CLI_ENTRY), 'validate', '--engine', engine, '--format', 'json', '--core', core], + { cwd: core, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, ); if (proc.error) throw new Error(`could not spawn the CLI for engine '${engine}': ${proc.error.message}`); @@ -235,14 +247,17 @@ async function main() { process.exit(1); } + // GT-716 AC5 — both engines read the Core as committed, through `--core`. const runs = {}; - for (const engine of ENGINES) { - try { - runs[engine] = runEngine(engine, root); - } catch (err) { - console.error(`❌ ${err.message}`); - process.exit(1); - } + let core = null; + try { + core = exportCore(root, 'evolith-verdict-parity-core-'); + for (const engine of ENGINES) runs[engine] = runEngine(engine, root, core); + } catch (err) { + console.error(`❌ ${err.message}`); + process.exit(1); + } finally { + if (core) rmSync(core, { recursive: true, force: true }); } const outcomes = Object.fromEntries(ENGINES.map((e) => [e, deriveOutcomes(runs[e].data)])); diff --git a/.harness/scripts/ci/73-validate-engine-coverage-parity.mjs b/.harness/scripts/ci/73-validate-engine-coverage-parity.mjs index 787855cda..dd491cc20 100644 --- a/.harness/scripts/ci/73-validate-engine-coverage-parity.mjs +++ b/.harness/scripts/ci/73-validate-engine-coverage-parity.mjs @@ -65,6 +65,17 @@ * `needs-supplied-facts`, `needs-external-system`, `needs-runtime`, `documentation-only`, * `underspecified` — GT-716 AC2) need no second one. * + * ## The page says what the report says (GT-716 AC5) + * + * Besides the per-rule entries, `--write` records each engine's COVERAGE per + * scenario — in scope, decided, skipped by the class each skip states, not + * applicable — and renders it as the table inside the `engine-coverage` markers of + * `docs/known-limitations.md` / `.es.md`. The default run compares both: coverage + * that moved fails until `--write` re-measures it, and a page whose table differs + * from the render fails until `--write` rewrites it. The reporter's + * `GOV-ENGINE-COVERAGE` states the same split, in the same groups, for the run it + * describes — so the report, the page and this baseline cannot say three things. + * * ## Anti-vacuous pass * * Both engine runs of both scenarios go through `assertScannedPerSource`; a missing @@ -75,29 +86,28 @@ * node .harness/scripts/ci/73-validate-engine-coverage-parity.mjs * node .harness/scripts/ci/73-validate-engine-coverage-parity.mjs --verbose * node .harness/scripts/ci/73-validate-engine-coverage-parity.mjs --json - * node .harness/scripts/ci/73-validate-engine-coverage-parity.mjs --write # regenerate the baseline (review the diff) + * node .harness/scripts/ci/73-validate-engine-coverage-parity.mjs --write # regenerate the baseline AND the page tables (review the diff) * * Exit codes: * 0 - every coverage-only rule is registered with its reason, every debt entry carries a decision, and every entry still holds * 1 - an unregistered rule, a stale entry, a changed reason, a debt entry nobody decided, a decision the runs contradict, or an engine that produced nothing */ -import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { execFileSync, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { REPO_ROOT } from '../lib/paths.mjs'; import { assertScannedPerSource, ZeroCoverageError } from '../lib/coverage.mjs'; +import { CLI_ENTRY, WASM_CANDIDATES, exportCore as exportTrackedCore } from '../lib/core-export.mjs'; import { facetOfInputPath, readCorpusFacts, readVocabulary } from '../lib/rule-facts.mjs'; import { deriveOutcomes, outcomeOf } from './68-validate-engine-verdict-parity.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); export const BASELINE_PATH = resolve(HERE, 'engine-coverage-parity.baseline.json'); -const CLI_ENTRY = 'src/sdk/cli/dist/main.js'; -const WASM_CANDIDATES = ['src/rulesets/opa/policy.wasm', 'src/sdk/cli/rulesets/opa/policy.wasm']; const ENGINES = ['native', 'opa']; const DECIDED = new Set(['passed', 'failed']); export const SCENARIOS = ['repository', 'init-satellite']; @@ -228,6 +238,103 @@ export function unknownDecisionRules(decisions, universes) { return decisions.flatMap((d) => (d.rules ?? []).filter((id) => !seen.has(id)).map((ruleId) => ({ decision: d.id, ruleId }))); } +/** The two pages that carry the measured coverage table (GT-716 AC5). */ +export const COVERAGE_PAGES = Object.freeze({ en: 'docs/known-limitations.md', es: 'docs/known-limitations.es.md' }); +const FRAGMENT_BEGIN = ''; +const FRAGMENT_END = ''; + +/** How the page and the reporter group a skip's class — the same four groups as `describeSkipSplit` in the reporter. */ +export const SKIP_GROUPS = Object.freeze({ + supplied: ['needs-supplied-facts', 'supplied-facet-absent'], + adapter: ['needs-external-system', 'needs-runtime'], + documentation: ['documentation-only', 'underspecified'], + debt: ['unimplemented-native', 'handler-declined', 'no-policy-in-bundle'], +}); + +/** One engine's coverage in one scenario, read from its JSON report. */ +export function coverageOf(data) { + const byClass = { ...(data?.skippedByEvaluability ?? {}) }; + const sum = (keys) => keys.reduce((acc, k) => acc + (byClass[k] ?? 0), 0); + const grouped = Object.fromEntries(Object.entries(SKIP_GROUPS).map(([g, keys]) => [g, sum(keys)])); + const known = new Set(Object.values(SKIP_GROUPS).flat()); + grouped.other = Object.entries(byClass).filter(([k]) => !known.has(k)).reduce((acc, [, v]) => acc + v, 0); + return { + inScope: data?.rulesTotal ?? 0, + decided: data?.rulesChecked ?? 0, + skipped: data?.rulesSkipped ?? 0, + errored: data?.rulesErrored ?? 0, + notApplicable: data?.rulesNotApplicable ?? 0, + byClass, + grouped, + }; +} + +/** Coverage totals that moved between two measurements of the same scenario. */ +export function reconcileCoverageTotals(measured, registered) { + const out = []; + for (const engine of ENGINES) { + const m = measured?.[engine]; + const r = registered?.[engine]; + if (!m) continue; + if (!r) { out.push({ engine, field: '(all)', from: null, to: 'measured' }); continue; } + for (const field of ['inScope', 'decided', 'skipped', 'errored', 'notApplicable']) { + if ((r[field] ?? 0) !== (m[field] ?? 0)) out.push({ engine, field, from: r[field] ?? 0, to: m[field] ?? 0 }); + } + const classes = new Set([...Object.keys(m.byClass ?? {}), ...Object.keys(r.byClass ?? {})]); + for (const c of [...classes].sort()) { + if ((r.byClass?.[c] ?? 0) !== (m.byClass?.[c] ?? 0)) out.push({ engine, field: `byClass.${c}`, from: r.byClass?.[c] ?? 0, to: m.byClass?.[c] ?? 0 }); + } + } + return out; +} + +const PAGE_TEXT = { + en: { + lead: (date) => `_Measured ${date} by \`73-validate-engine-coverage-parity.mjs --write\` — one \`evolith validate --engine --format json\` per engine and scenario, on an export of the tracked tree and on a satellite fresh from \`evolith init\`. CI regenerates this table and fails when it differs from the measurement; edit the guard, not the table._`, + head: '| Scenario | Engine | In scope | Decided | Skipped | …fact not supplied | …adapter needed | …documentation | …engine debt | Not applicable |', + scenario: { repository: 'this repository', 'init-satellite': 'satellite fresh from `init`' }, + engine: { native: 'native (default)', opa: '`--engine opa`' }, + }, + es: { + lead: (date) => `_Medido el ${date} por \`73-validate-engine-coverage-parity.mjs --write\` — un \`evolith validate --engine --format json\` por motor y escenario, sobre una exportación del árbol versionado y sobre un satélite recién salido de \`evolith init\`. CI regenera esta tabla y falla cuando difiere de la medición; edita el guard, no la tabla._`, + head: '| Escenario | Motor | En alcance | Decididas | Saltadas | …hecho no suministrado | …falta adaptador | …documentación | …deuda del motor | No aplicables |', + scenario: { repository: 'este repositorio', 'init-satellite': 'satélite recién salido de `init`' }, + engine: { native: 'nativo (por defecto)', opa: '`--engine opa`' }, + }, +}; + +/** The table the page carries, rendered from the baseline's coverage block. Deterministic. */ +export function renderCoverageTable(coverage, measuredOn, lang = 'en') { + const t = PAGE_TEXT[lang] ?? PAGE_TEXT.en; + const rows = [t.lead(measuredOn), '', t.head, '|---|---|---:|---:|---:|---:|---:|---:|---:|---:|']; + for (const scenario of SCENARIOS) { + for (const engine of ENGINES) { + const c = coverage?.[scenario]?.[engine]; + if (!c) continue; + const g = c.grouped ?? {}; + const other = g.other ? ` (+${g.other})` : ''; + rows.push(`| ${t.scenario[scenario] ?? scenario} | ${t.engine[engine] ?? engine} | ${c.inScope} | ${c.decided} | ${c.skipped} | ${g.supplied ?? 0} | ${g.adapter ?? 0} | ${g.documentation ?? 0} | ${g.debt ?? 0}${other} | ${c.notApplicable} |`); + } + } + return rows.join('\n'); +} + +/** The page with its fragment replaced; null when the page carries no markers. */ +export function withCoverageFragment(pageText, rendered) { + const a = pageText.indexOf(FRAGMENT_BEGIN); + const b = pageText.indexOf(FRAGMENT_END); + if (a < 0 || b < 0 || b < a) return null; + return pageText.slice(0, a + FRAGMENT_BEGIN.length) + '\n' + rendered + '\n' + pageText.slice(b); +} + +/** The fragment a page currently carries, trimmed; null when it carries no markers. */ +export function coverageFragmentOf(pageText) { + const a = pageText.indexOf(FRAGMENT_BEGIN); + const b = pageText.indexOf(FRAGMENT_END); + if (a < 0 || b < 0 || b < a) return null; + return pageText.slice(a + FRAGMENT_BEGIN.length, b).trim(); +} + /** * Rule ids decided by exactly one engine, with the OTHER engine's outcome. * Exported for the unit tests; the precedence is 68's. @@ -372,24 +479,9 @@ function runEngine(engine, cwd, extra = []) { return parsed.data; } -/** - * The Core as committed: every tracked file (with local modifications), nothing - * untracked, plus the compiled bundle the evaluator needs. See the header. - */ +/** The Core as committed — shared with guard 68 since GT-716 AC5 (`lib/core-export.mjs`). */ function exportCore(root) { - const dir = mkdtempSync(join(tmpdir(), 'evolith-coverage-parity-core-')); - const listed = execFileSync('git', ['ls-files', '-z'], { cwd: root, maxBuffer: 256 * 1024 * 1024 }); - const archive = execFileSync('tar', ['-c', '--null', '-T', '-', '-f', '-'], { cwd: root, input: listed, maxBuffer: 1024 * 1024 * 1024 }); - execFileSync('tar', ['-x', '-f', '-', '-C', dir], { input: archive, maxBuffer: 1024 * 1024 * 1024 }); - const wasm = WASM_CANDIDATES.find((r) => existsSync(resolve(root, r))); - for (const rel of WASM_CANDIDATES) { - mkdirSync(dirname(resolve(dir, rel)), { recursive: true }); - copyFileSync(resolve(root, wasm), resolve(dir, rel)); - } - if (!existsSync(join(dir, 'src', 'rulesets', 'schema', 'facets.json'))) { - throw new Error(`the export at ${dir} has no corpus vocabulary — \`git ls-files\` produced an incomplete tree`); - } - return dir; + return exportTrackedCore(root, 'evolith-coverage-parity-core-'); } /** A satellite exactly as `evolith init` leaves it, in a temporary directory. */ @@ -429,7 +521,9 @@ function measureScenario(name, runs, manifest, snapshot, corpus, vocabulary, emi const universe = new Set([...outcomes.native.keys(), ...outcomes.opa.keys()]); const { nativeOnly, opaOnly } = coverageOnly(outcomes.native, outcomes.opa, universe); const decided = new Map([...universe].map((id) => [id, new Set(ENGINES.filter((e) => DECIDED.has(outcomeOf(outcomes[e], id))))])); + const coverage = Object.fromEntries(ENGINES.map((e) => [e, coverageOf(runs[e])])); return { + coverage, nativeOnly: nativeOnly.map((e) => ({ ...e, reason: opaReason(e.ruleId, manifest, corpus, vocabulary, runs.opa, emitted) })), opaOnly: opaOnly.map((e) => ({ ...e, reason: nativeReason(e.ruleId, runs.native, snapshot, corpus) })), decided, @@ -499,8 +593,17 @@ async function main() { measuredOn: new Date().toISOString().slice(0, 10), method: 'evolith validate --engine {native,opa} --format json, on an export of the tracked tree (git ls-files + policy.wasm) and on a satellite fresh from `evolith init` with --core pointed at that export; outcomes per 68-validate-engine-verdict-parity.mjs.', scenarios: Object.fromEntries(SCENARIOS.map((s) => [s, toBaselineScenario(measured[s])])), + coverage: Object.fromEntries(SCENARIOS.map((s) => [s, measured[s].coverage])), }; writeFileSync(BASELINE_PATH, JSON.stringify(baseline, null, 2) + '\n'); + for (const [lang, rel] of Object.entries(COVERAGE_PAGES)) { + const page = resolve(root, rel); + if (!existsSync(page)) continue; + const next = withCoverageFragment(readFileSync(page, 'utf8'), renderCoverageTable(baseline.coverage, baseline.measuredOn, lang)); + if (next === null) { console.log(` ${rel}: no ${FRAGMENT_BEGIN} … ${FRAGMENT_END} markers — the page does not carry the table.`); continue; } + writeFileSync(page, next); + console.log(` ${rel}: coverage table rewritten.`); + } for (const s of SCENARIOS) { console.log(` ${s}: native-only ${measured[s].nativeOnly.length}, opa-only ${measured[s].opaOnly.length} (${measured[s].durationMs} ms)`); } @@ -508,7 +611,7 @@ async function main() { const { undecided } = reconcileDecisions(measured[s], decisions); if (undecided.length > 0) console.log(` ${s}: ${undecided.length} debt entry(ies) carry no decision yet — record them in ${DECISIONS_PATH.replace(`${root}/`, '')}: ${undecided.map((e) => e.ruleId).join(', ')}`); } - console.log(`✓ baseline written to ${BASELINE_PATH.replace(`${root}/`, '')} — review the diff before committing it.`); + console.log(`✓ baseline and page tables written from ${BASELINE_PATH.replace(`${root}/`, '')} — review the diff before committing it.`); return; } @@ -574,6 +677,31 @@ async function main() { } } + for (const s of SCENARIOS) { + const moved = reconcileCoverageTotals(measured[s].coverage, baseline.coverage?.[s]); + report.scenarios[s].coverageMoved = moved.map((e) => `${e.engine}.${e.field}`); + if (moved.length > 0) { + failed = true; + console.error(`❌ ${s}: the measured coverage differs from the registered one in ${moved.length} place(s) — re-run with --write (it rewrites the page tables too) and review:`); + for (const e of moved) console.error(` - ${e.engine} ${e.field}: ${e.from} → ${e.to}`); + } + } + const pageDrift = []; + for (const [lang, rel] of Object.entries(COVERAGE_PAGES)) { + const page = resolve(root, rel); + if (!existsSync(page)) { pageDrift.push({ rel, why: 'the page does not exist' }); continue; } + const have = coverageFragmentOf(readFileSync(page, 'utf8')); + const want = renderCoverageTable(baseline.coverage, baseline.measuredOn, lang); + if (have === null) pageDrift.push({ rel, why: `no ${FRAGMENT_BEGIN} … ${FRAGMENT_END} markers` }); + else if (have !== want) pageDrift.push({ rel, why: 'its table differs from the render of the registered coverage' }); + } + report.pageDrift = pageDrift.map((e) => e.rel); + if (pageDrift.length > 0) { + failed = true; + console.error(`❌ ${pageDrift.length} page(s) do not say what the baseline measured (GT-716 AC5) — re-run with --write:`); + for (const e of pageDrift) console.error(` - ${e.rel}: ${e.why}`); + } + const unknown = unknownDecisionRules(decisions, SCENARIOS.map((s) => new Set(measured[s].decided.keys()))); report.unknownDecisionRules = unknown.map((e) => e.ruleId); if (unknown.length > 0) { @@ -588,7 +716,7 @@ async function main() { console.error(' A coverage difference is legitimate (ADR-0041); an unregistered one is not. Register it with its reason, or fix it.'); process.exit(1); } - console.log('✓ 73-validate-engine-coverage-parity: every coverage-only rule is registered with its reason and every debt entry with a decision, in both directions, on both scenarios.'); + console.log('✓ 73-validate-engine-coverage-parity: every coverage-only rule is registered with its reason and every debt entry with a decision, in both directions, on both scenarios; the coverage tables on the page are the measured ones.'); } if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) { diff --git a/.harness/scripts/ci/73-validate-engine-coverage-parity.test.mjs b/.harness/scripts/ci/73-validate-engine-coverage-parity.test.mjs index d3b754a80..31e193b14 100644 --- a/.harness/scripts/ci/73-validate-engine-coverage-parity.test.mjs +++ b/.harness/scripts/ci/73-validate-engine-coverage-parity.test.mjs @@ -6,21 +6,28 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { deriveOutcomes } from './68-validate-engine-verdict-parity.mjs'; +import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { + COVERAGE_PAGES, FOLLOW_UP, classFromReport, + coverageFragmentOf, + coverageOf, coverageOnly, decisionIndex, nativeReason, opaReason, readDecisions, reconcileCoverage, + reconcileCoverageTotals, reconcileDecisions, + renderCoverageTable, rulesOf, toBaselineScenario, unknownDecisionRules, validateDecisions, + withCoverageFragment, } from './73-validate-engine-coverage-parity.mjs'; import { REPO_ROOT } from '../lib/paths.mjs'; import { readCorpusFacts } from '../lib/rule-facts.mjs'; @@ -214,3 +221,57 @@ test('the committed register is well-formed and every rule it names exists in th assert.deepEqual(missing, [], 'decision rule ids must be corpus rule ids'); assert.ok(decisions.some((d) => d.kind === 'neither' && d.pattern), 'the ADR-conformance decision is a pattern over the generated id shape'); }); + +// --------------------------------------------------------------------------- +// GT-716 AC5 — the page says what the report says: the coverage block and its render +// --------------------------------------------------------------------------- + +const report = (over) => ({ rulesTotal: 159, rulesChecked: 56, rulesSkipped: 103, rulesErrored: 0, rulesNotApplicable: 30, skippedByEvaluability: { 'needs-supplied-facts': 31, 'needs-external-system': 20, 'needs-runtime': 14, 'documentation-only': 30, 'unimplemented-native': 8 }, ...over }); + +test('coverage is read from the report and grouped the way the reporter groups it', () => { + const c = coverageOf(report({})); + assert.deepEqual([c.inScope, c.decided, c.skipped, c.notApplicable], [159, 56, 103, 30]); + assert.deepEqual(c.grouped, { supplied: 31, adapter: 34, documentation: 30, debt: 8, other: 0 }); + const withOther = coverageOf(report({ skippedByEvaluability: { 'no-policy-in-bundle': 25, 'supplied-facet-absent': 34, mystery: 2 } })); + assert.deepEqual(withOther.grouped, { supplied: 34, adapter: 0, documentation: 0, debt: 25, other: 2 }); + assert.deepEqual(coverageOf(undefined).grouped, { supplied: 0, adapter: 0, documentation: 0, debt: 0, other: 0 }); +}); + +test('coverage that moved is named field by field, class by class — and an unregistered engine is a move too', () => { + const measured = { native: coverageOf(report({})), opa: coverageOf(report({ rulesChecked: 7, rulesSkipped: 152, skippedByEvaluability: { 'supplied-facet-absent': 120 } })) }; + const same = reconcileCoverageTotals(measured, JSON.parse(JSON.stringify(measured))); + assert.deepEqual(same, []); + const registered = JSON.parse(JSON.stringify(measured)); + registered.native.decided = 55; registered.native.byClass['needs-runtime'] = 15; delete registered.opa; + const moved = reconcileCoverageTotals(measured, registered); + assert.deepEqual(moved.map((e) => `${e.engine}.${e.field}:${e.from}→${e.to}`), ['native.decided:55→56', 'native.byClass.needs-runtime:15→14', 'opa.(all):null→measured']); +}); + +test('the rendered table is deterministic, bilingual, and carries the measurement date and the four groups', () => { + const coverage = { repository: { native: coverageOf(report({})), opa: coverageOf(report({ rulesChecked: 7, rulesSkipped: 152, skippedByEvaluability: { 'supplied-facet-absent': 120, 'no-policy-in-bundle': 25, mystery: 7 } })) }, 'init-satellite': {} }; + const en = renderCoverageTable(coverage, '2026-09-21', 'en'); + assert.match(en, /^_Measured 2026-09-21 by `73-validate-engine-coverage-parity\.mjs --write`/); + assert.match(en, /\| this repository \| native \(default\) \| 159 \| 56 \| 103 \| 31 \| 34 \| 30 \| 8 \| 30 \|/); + assert.match(en, /\| this repository \| `--engine opa` \| 159 \| 7 \| 152 \| 120 \| 0 \| 0 \| 25 \(\+7\) \| 30 \|/); + assert.equal(en, renderCoverageTable(coverage, '2026-09-21', 'en')); + const es = renderCoverageTable(coverage, '2026-09-21', 'es'); + assert.match(es, /^_Medido el 2026-09-21/); + assert.match(es, /\| este repositorio \| nativo \(por defecto\) \| 159 \| 56 \| 103 \|/); + assert.notEqual(en, es); +}); + +test('the fragment is replaced between its markers and read back verbatim; a page without markers is null, not silently skipped', () => { + const page = '# Page\n\nintro\n\n\nold table\n\n\noutro\n'; + const next = withCoverageFragment(page, 'NEW'); + assert.equal(next, '# Page\n\nintro\n\n\nNEW\n\n\noutro\n'); + assert.equal(coverageFragmentOf(next), 'NEW'); + assert.equal(withCoverageFragment('no markers here', 'NEW'), null); + assert.equal(coverageFragmentOf('no markers here'), null); +}); + +test('the committed pages carry the markers, in both languages', () => { + for (const rel of Object.values(COVERAGE_PAGES)) { + const text = readFileSync(resolve(REPO_ROOT, rel), 'utf8'); + assert.notEqual(coverageFragmentOf(text), null, `${rel} carries the engine-coverage markers`); + } +}); diff --git a/.harness/scripts/ci/engine-coverage-parity.baseline.json b/.harness/scripts/ci/engine-coverage-parity.baseline.json index f84fd7896..6efba7939 100644 --- a/.harness/scripts/ci/engine-coverage-parity.baseline.json +++ b/.harness/scripts/ci/engine-coverage-parity.baseline.json @@ -663,5 +663,94 @@ }, "opaOnly": {} } + }, + "coverage": { + "repository": { + "native": { + "inScope": 355, + "decided": 109, + "skipped": 246, + "errored": 0, + "notApplicable": 60, + "byClass": { + "needs-supplied-facts": 37, + "unimplemented-native": 6, + "needs-external-system": 38, + "needs-runtime": 27, + "documentation-only": 138 + }, + "grouped": { + "supplied": 37, + "adapter": 65, + "documentation": 138, + "debt": 6, + "other": 0 + } + }, + "opa": { + "inScope": 355, + "decided": 28, + "skipped": 327, + "errored": 0, + "notApplicable": 60, + "byClass": { + "supplied-facet-absent": 146, + "documentation-only": 138, + "no-policy-in-bundle": 33, + "unimplemented-native": 9, + "needs-runtime": 1 + }, + "grouped": { + "supplied": 146, + "adapter": 1, + "documentation": 138, + "debt": 42, + "other": 0 + } + } + }, + "init-satellite": { + "native": { + "inScope": 151, + "decided": 50, + "skipped": 101, + "errored": 0, + "notApplicable": 264, + "byClass": { + "needs-supplied-facts": 30, + "unimplemented-native": 6, + "needs-external-system": 39, + "needs-runtime": 22, + "documentation-only": 4 + }, + "grouped": { + "supplied": 30, + "adapter": 61, + "documentation": 4, + "debt": 6, + "other": 0 + } + }, + "opa": { + "inScope": 151, + "decided": 2, + "skipped": 149, + "errored": 0, + "notApplicable": 264, + "byClass": { + "supplied-facet-absent": 117, + "no-policy-in-bundle": 22, + "unimplemented-native": 9, + "needs-runtime": 1 + }, + "grouped": { + "supplied": 117, + "adapter": 1, + "documentation": 0, + "debt": 31, + "other": 0 + } + } + } } } diff --git a/.harness/scripts/ci/engine-verdict-parity.baseline.json b/.harness/scripts/ci/engine-verdict-parity.baseline.json index 18b6a6e99..7b5ee87e9 100644 --- a/.harness/scripts/ci/engine-verdict-parity.baseline.json +++ b/.harness/scripts/ci/engine-verdict-parity.baseline.json @@ -22,34 +22,24 @@ "run the native engine resolves the corpus to the CLI's bundled copy and FAILS the 138 generated ADR-conformance rules (their referenced", "ADRs are not in the copy — the false failures guard 73 moved to an export to escape), so it counts 0 non-executable and never emits the", "row. The OPA engine, since AC4, states `documentation-only` for a policy-less rule whose declaration says there is nothing to check, counts", - "the 138 as non-executable and emits the row (COULD, non-blocking). With `--core .` both engines count 138 and emit the identical row." + "the 138 as non-executable and emits the row (COULD, non-blocking). With `--core .` both engines count 138 and emit the identical row.", + "2026-09-21 (GT-716 AC5): the guard now runs both engines on an EXPORT of the tracked tree with `--core` pointed at it (the export guard 73", + "measures, `lib/core-export.mjs`), so what remains here is a disagreement about a rule and not about where a run found its corpus. Three", + "entries left as stale on that run: CLI-EXIT-01 and CLI-EXIT-03 (the doubled scan root does not happen when `corePath` is a resolvable root —", + "the handler reads the real CLI tree and passes, as OPA does) and GOV-RULE-NON-EXECUTABLE (both engines now count the 138 generated ADR-conformance", + "rules as non-executable and emit the identical row). One entry arrived: CLI-EXIT-02. GOV-ENGINE-COVERAGE agrees since the advisory speaks", + "for either engine that skips more than it checks." ], "measuredOn": "2026-09-21", - "method": "evolith validate --engine {native,opa} --format json over the whole corpus, both from the repository root; only rules BOTH engines decided are compared.", + "method": "evolith validate --engine {native,opa} --format json over the whole corpus, both on an export of the tracked tree (git ls-files + policy.wasm) with --core pointed at it; only rules BOTH engines decided are compared.", "conflicts": [ { - "ruleId": "CLI-EXIT-01", - "native": "failed", - "opa": "passed", - "family": "native-context-assumption", - "reason": "The most defect-shaped of the eleven. `cli-exit-taxonomy-rule.handler.ts:132` builds its scan root as `path.join(ctx.corePath, 'src', 'sdk', 'cli', 'src')`, and in a CLI run `ctx.corePath` already resolves to `.../src/sdk/cli`, so the root becomes `.../src/sdk/cli/src/sdk/cli/src` — a path that does not exist. The handler reports a missing tree as `failed` deliberately (a skip on a blocking rule would be the silent pass it exists to prevent), so the verdict is about a doubled path and not about exit codes. OPA decides the same id from the fact document `exit-code-taxonomy-facts.mjs` emits and finds no offender.", - "followUp": "Resolve the CLI source root from the CLI package rather than by re-appending its own path to `corePath`. That deletes this entry and CLI-EXIT-03 together." - }, - { - "ruleId": "CLI-EXIT-03", - "native": "failed", - "opa": "passed", - "family": "native-context-assumption", - "reason": "Same doubled root as CLI-EXIT-01, reported through the other branch of the same handler: 'exit-code taxonomy not declared: .../src/sdk/cli/src/sdk/cli/src/infrastructure/cli/exit-codes.ts not found'. Registered separately because a partial fix that repaired only the scan and not the taxonomy read is a real state, and the guard should be able to say which half moved.", - "followUp": "Same fix as CLI-EXIT-01." - }, - { - "ruleId": "GOV-RULE-NON-EXECUTABLE", + "ruleId": "CLI-EXIT-02", "native": "passed", "opa": "failed", - "family": "corpus-resolution-artifact", - "reason": "The synthetic row counts rules whose evaluability is `documentation-only` or `underspecified`. On this guard's run (repository root, no `--core`) the native engine resolves the corpus to the CLI's bundled rulesets copy and fails the 138 generated ADR-conformance rules — `CORE-0001-01: the ruleset cites 1 decision record(s) that do not exist: reference/core/architecture/adrs/core/0001-…` — so `rulesNonExecutable` is 0 and the row is absent (`passed` by 68's default). The OPA engine states the declaration's class for a policy-less rule with `facts: []` since GT-716 AC4, counts 138 non-executable and emits the row: `138 corpus rules are not executable by any engine` (COULD, non-blocking). Measured with `--core .` on 2026-09-21: both engines count 138 and emit the identical row — the disagreement is the corpus each run found, not the rule.", - "followUp": "Run this guard with `--core` at the repository root (or on the export guard 73 measures), which makes the two runs read the same corpus — and re-baseline CLI-EXIT-01/03, which pass once `corePath` is a resolvable root, plus whatever the same-corpus run disagrees on (measured 2026-09-21: GOV-ENGINE-COVERAGE and CLI-EXIT-02). GT-716 AC5 territory: the report saying the same thing on both engines." + "family": "fact-document-not-in-tree", + "reason": "`cli-exit-code-taxonomy.rego` decides CLI-EXIT-02 from a fact document — `input.core.cli.exitCodes`, emitted by `src/sdk/cli/scripts/exit-code-taxonomy-facts.mjs` — and, by its own non-vacuity clause, FAILS the rule when the document is absent: 'no exit-code taxonomy facts were supplied at input.core.cli.exitCodes — the rule cannot be evaluated, and an unevaluated blocking rule is not a pass'. On the export nothing runs that script, so the policy fails on absence. The native handler (`cli-exit-taxonomy-rule.handler.ts`) reads the CLI's exit-code source directly and passes. GT-716 AC1's absent-facet skip does not reach this: the facet `core.cli` IS present (the builder emits it); `exitCodes` is a missing key inside a present facet, and the policy chose to treat that as a verdict.", + "followUp": "Make both engines read the same source: have the OPA input builder emit `core.cli.exitCodes` from the CLI tree (or run the facts script on the way in), or declare `exitCodes` as a facet of its own so its absence is a `supplied-facet-absent` skip under AC1 rather than a failure. Either deletes this entry." } ] } diff --git a/.harness/scripts/lib/core-export.mjs b/.harness/scripts/lib/core-export.mjs new file mode 100644 index 000000000..18e5d95ac --- /dev/null +++ b/.harness/scripts/lib/core-export.mjs @@ -0,0 +1,41 @@ +/** + * The Core as COMMITTED, in a temporary directory: every tracked file (with local + * modifications), nothing untracked, plus the compiled bundle the evaluator needs. + * + * Why the engine-parity guards measure this and not the working tree (GT-716 AC3, + * AC5): the first CI run of guard 73 disagreed with the laptop that wrote its + * baseline — a local `coverage/` directory decided two rules, git history a third, + * and a run without `--core` resolved the ADR-conformance rules against the CLI's + * bundled corpus copy and failed 138 of them falsely. None of that is the corpus. + * An export of `git ls-files` is the same tree on every machine, and `--core` + * pointed at it makes both engines read the same Core. + */ +import { copyFileSync, existsSync, mkdirSync, mkdtempSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; + +export const CLI_ENTRY = 'src/sdk/cli/dist/main.js'; +export const WASM_CANDIDATES = ['src/rulesets/opa/policy.wasm', 'src/sdk/cli/rulesets/opa/policy.wasm']; + +/** + * @param {string} root the repository + * @param {string} [prefix] the temp-dir prefix, so a leftover names its owner + * @returns {string} the export's directory; the caller removes it + */ +export function exportCore(root, prefix = 'evolith-core-export-') { + const dir = mkdtempSync(join(tmpdir(), prefix)); + const listed = execFileSync('git', ['ls-files', '-z'], { cwd: root, maxBuffer: 256 * 1024 * 1024 }); + const archive = execFileSync('tar', ['-c', '--null', '-T', '-', '-f', '-'], { cwd: root, input: listed, maxBuffer: 1024 * 1024 * 1024 }); + execFileSync('tar', ['-x', '-f', '-', '-C', dir], { input: archive, maxBuffer: 1024 * 1024 * 1024 }); + const wasm = WASM_CANDIDATES.find((r) => existsSync(resolve(root, r))); + if (!wasm) throw new Error(`no compiled bundle under ${root} (build it: npm run build:policy)`); + for (const rel of WASM_CANDIDATES) { + mkdirSync(dirname(resolve(dir, rel)), { recursive: true }); + copyFileSync(resolve(root, wasm), resolve(dir, rel)); + } + if (!existsSync(join(dir, 'src', 'rulesets', 'schema', 'facets.json'))) { + throw new Error(`the export at ${dir} has no corpus vocabulary — \`git ls-files\` produced an incomplete tree`); + } + return dir; +} diff --git a/README.es.md b/README.es.md index b5f42d2f5..c551f1eb2 100644 --- a/README.es.md +++ b/README.es.md @@ -61,7 +61,7 @@ npx -y @beyondnet/evolith-cli rulesets # lista los packs npx -y @beyondnet/evolith-cli validate --engine opa --select rulesets/acl/anti-corruption-layer.rules.json ``` -`init` escribe `evolith.yaml` con el nombre, tipo y fase del producto y tu stack; `--engine opa` elige el motor con más cobertura hoy (el porqué, en [Estado real](./docs/known-limitations.es.md)). Cómo se ve una primera ejecución, fila por fila: [captura](./docs/evidence/first-run-capture.es.md). Guía completa: [Inicio rápido](./docs/guides/evolith-quickstart.es.md). +`init` escribe `evolith.yaml` con el nombre, tipo y fase del producto y tu stack; `--engine opa` elige el motor con más cobertura **en la CLI publicada (1.3.2)**; en este árbol el motor por defecto ya decide más en una ejecución a secas, y `--engine opa` es para los hechos que tú suministras — las cifras medidas, por motor y con fecha, en [Estado real](./docs/known-limitations.es.md). Cómo se ve una primera ejecución, fila por fila: [captura](./docs/evidence/first-run-capture.es.md). Guía completa: [Inicio rápido](./docs/guides/evolith-quickstart.es.md). --- diff --git a/README.md b/README.md index 699cb4af2..93f693225 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ npx -y @beyondnet/evolith-cli rulesets # lists the packs npx -y @beyondnet/evolith-cli validate --engine opa --select rulesets/acl/anti-corruption-layer.rules.json ``` -`init` writes `evolith.yaml` with the product's name, type and phase and your stack; `--engine opa` picks the engine with the most coverage today (why, in [Known limitations](./docs/known-limitations.md)). What a first run looks like, row by row: [capture](./docs/evidence/first-run-capture.md). Full guide: [Quickstart](./docs/guides/evolith-quickstart.md). +`init` writes `evolith.yaml` with the product's name, type and phase and your stack; `--engine opa` picks the engine with the most coverage **on the published CLI (1.3.2)**; in this tree the default already decides more on a bare run, and `--engine opa` is for the facts you supply — the measured numbers, per engine and dated, in [Known limitations](./docs/known-limitations.md). What a first run looks like, row by row: [capture](./docs/evidence/first-run-capture.md). Full guide: [Quickstart](./docs/guides/evolith-quickstart.md). --- diff --git a/docs/evidence/first-run-capture.es.md b/docs/evidence/first-run-capture.es.md index d7b3abdc1..ca7221248 100644 --- a/docs/evidence/first-run-capture.es.md +++ b/docs/evidence/first-run-capture.es.md @@ -148,7 +148,10 @@ el mismo corpus no cubre lo mismo: CI exige que los dos motores coincidan sobre fixtures. No exige que tengan la misma cobertura sobre un repositorio real, y hoy no la tienen. Por eso la portada -usa `--engine opa` en todas partes. +usa `--engine opa` en todas partes. (Medido en 1.3.2. Desde GT-716 — en el +árbol, aún no en una CLI publicada — el motor por defecto decide más en una ejecución +a secas, CI registra cada regla que solo un motor decide, y la tabla por motor +vigente está en la página de [estado real](../known-limitations.es.md).) ``` **Status:** failed diff --git a/docs/evidence/first-run-capture.md b/docs/evidence/first-run-capture.md index 1efe10852..06762ff45 100644 --- a/docs/evidence/first-run-capture.md +++ b/docs/evidence/first-run-capture.md @@ -148,7 +148,10 @@ version and the same corpus it does not cover the same ground: CI holds the two engines to agreement over fixtures. It does not hold them to equal coverage over a real repository, and today they do not have it. That is why -the front page uses `--engine opa` everywhere. +the front page uses `--engine opa` everywhere. (Measured on 1.3.2. Since GT-716 — +in the tree, not yet in a published CLI — the default decides more on a bare run, +CI registers every rule only one engine decides, and the current per-engine table +is on the [known-limitations](../known-limitations.md) page.) ``` **Status:** failed diff --git a/docs/known-limitations.es.md b/docs/known-limitations.es.md index 8a416072c..eb235cd3b 100644 --- a/docs/known-limitations.es.md +++ b/docs/known-limitations.es.md @@ -10,12 +10,22 @@ Auditoría completa de nuestras propias afirmaciones, con qué bloquea cada pend ## Los dos motores no cubren lo mismo -`evolith validate` corre por defecto el evaluador nativo; `--engine opa` evalúa con el bundle Rego compilado. Sobre este mismo repositorio, medido el 2026-08-21 con `@beyondnet/evolith-cli@1.3.2`: +`evolith validate` corre por defecto el evaluador nativo; `--engine opa` evalúa con el bundle Rego compilado. Deciden partes distintas del mismo corpus y ninguno es superconjunto del otro. Qué decide cada uno, y por qué salta el resto, lo mide CI en cada corrida — `73-validate-engine-coverage-parity.mjs`, sobre una exportación del árbol versionado y sobre un satélite recién salido de `init` — y la tabla de abajo la escribe ese guard, no una mano: cuando las cifras se mueven, CI falla hasta que el guard las reescribe (AC5 de GT-716). -| Motor | Evalúa | Salta | -|---|---|---| -| `--engine opa` | 133 de 159 | 26 | -| nativo (por defecto) | 41 de 159 | 118 | + +_Medido el 2026-09-21 por `73-validate-engine-coverage-parity.mjs --write` — un `evolith validate --engine --format json` por motor y escenario, sobre una exportación del árbol versionado y sobre un satélite recién salido de `evolith init`. CI regenera esta tabla y falla cuando difiere de la medición; edita el guard, no la tabla._ + +| Escenario | Motor | En alcance | Decididas | Saltadas | …hecho no suministrado | …falta adaptador | …documentación | …deuda del motor | No aplicables | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| este repositorio | nativo (por defecto) | 355 | 109 | 246 | 37 | 65 | 138 | 6 | 60 | +| este repositorio | `--engine opa` | 355 | 28 | 327 | 146 | 1 | 138 | 42 | 60 | +| satélite recién salido de `init` | nativo (por defecto) | 151 | 50 | 101 | 30 | 61 | 4 | 6 | 264 | +| satélite recién salido de `init` | `--engine opa` | 151 | 2 | 149 | 117 | 1 | 0 | 31 | 264 | + + +**Cómo leerla.** *Hecho no suministrado* es una postura que solo los dueños del repositorio pueden declarar — el filtrado por tenant, la intención de runtime, los hallazgos del sistema de CI; las políticas OPA las deciden cuando el llamador las suministra por `facts.satellite` (GT-694), y el motor nativo nunca. Esa es la única razón que queda para correr `--engine opa`, y no es cobertura: en una ejecución a secas el motor por defecto decide más, en las dos filas. *Falta adaptador* es un sistema externo o en ejecución que nadie ha cableado aún por la costura del enforcer. *Documentación* es una regla sin comprobación tal como está escrita — sobre todo los marcadores ADR-conformance generados, `documentation-only` en ambos motores por decisión. *Deuda del motor* es un handler o una política que nadie escribió; cada una lleva una decisión registrada en [`engine-coverage-decisions.json`](../.harness/scripts/ci/engine-coverage-decisions.json), y el lado solo-OPA de la [línea base de cobertura](../.harness/scripts/ci/engine-coverage-parity.baseline.json) está vacío. El informe dice lo mismo con los mismos grupos: cuando un motor salta más de lo que comprueba, su fila `GOV-ENGINE-COVERAGE` enuncia el desglose de esa corrida. + +**La CLI publicada es anterior a todo esto.** `@beyondnet/evolith-cli@1.3.2` — lo que instala hoy `npx -y @beyondnet/evolith-cli` — es anterior a GT-716. Medida el 2026-08-21 sobre este repositorio decidía 133 de 159 con `--engine opa` y 41 de 159 en nativo, y la mayor parte de esa diferencia eran veredictos sobre hechos que nadie suministró. Por eso la portada sigue diciendo `--engine opa`, y dice para qué CLI; la próxima publicación lo invierte. CI exige que coincidan sobre **hechos**, no sobre cobertura; eso es por diseño — y desde el AC3 de GT-716 cada regla que solo un motor decide queda registrada por regla, en ambas direcciones, sobre este repositorio y sobre un satélite recién salido de `init` (`73-validate-engine-coverage-parity.mjs`), de modo que una diferencia de cobertura es un diff que alguien lee y no un número que nadie lee. Desde el AC4 (2026-09-21) nada en ese registro está por omisión: cada entrada de deuda lleva una decisión registrada (`engine-coverage-decisions.json`) que el guard contrasta con las ejecuciones, y el lado solo-OPA está vacío en ambos escenarios — las últimas reglas que solo `--engine opa` decidía desde el árbol (`OBS-EVD-01..03`, `MCP-05`) tienen gemelos nativos de sus políticas. Que el comando por defecto no lo diga, no lo es ([#628](https://github.com/beyondnetcode/evolith_arch32/issues/628)). Por eso la portada usa `--engine opa` en todas partes. Medido de nuevo el 2026-09-20 con la CLI construida desde este árbol: el motor por defecto decide 56 de las mismas 159, y de las 76 reglas que solo `--engine opa` decide, 73 son veredictos sobre facetas que una ejecución a secas nunca suministra — así que la mayor parte de esa cobertura extra no es cobertura. Se sigue como GT-716 en el [Tablero de Gaps](../reference/core/control-center/gaps/gap-tracking.es.md). Desde `b2840947` (en el árbol, aún no en una CLI publicada) el motor OPA reporta esas reglas como `skipped` con la faceta que le falta: sobre el mismo satélite decide 10 de 159 a partir de lo que una ejecución a secas observa, y el resto solo cuando el llamador suministra los hechos por `facts.satellite`. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index c3b20234a..deb101791 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,12 +10,22 @@ Full audit of our own claims, with what blocks each pending item and who can unb ## The two engines do not cover the same ground -`evolith validate` runs the native evaluator by default; `--engine opa` evaluates with the compiled Rego bundle. On this very repository, measured on 2026-08-21 with `@beyondnet/evolith-cli@1.3.2`: +`evolith validate` runs the native evaluator by default; `--engine opa` evaluates with the compiled Rego bundle. They decide different parts of the same corpus and neither is a superset of the other. What each one decides, and why it skips the rest, is measured by CI on every run — `73-validate-engine-coverage-parity.mjs`, on an export of the tracked tree and on a satellite fresh from `init` — and the table below is written by that guard, not by hand: when the numbers move, CI fails until the guard rewrites them (GT-716 AC5). -| Engine | Evaluates | Skips | -|---|---|---| -| `--engine opa` | 133 of 159 | 26 | -| native (default) | 41 of 159 | 118 | + +_Measured 2026-09-21 by `73-validate-engine-coverage-parity.mjs --write` — one `evolith validate --engine --format json` per engine and scenario, on an export of the tracked tree and on a satellite fresh from `evolith init`. CI regenerates this table and fails when it differs from the measurement; edit the guard, not the table._ + +| Scenario | Engine | In scope | Decided | Skipped | …fact not supplied | …adapter needed | …documentation | …engine debt | Not applicable | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| this repository | native (default) | 355 | 109 | 246 | 37 | 65 | 138 | 6 | 60 | +| this repository | `--engine opa` | 355 | 28 | 327 | 146 | 1 | 138 | 42 | 60 | +| satellite fresh from `init` | native (default) | 151 | 50 | 101 | 30 | 61 | 4 | 6 | 264 | +| satellite fresh from `init` | `--engine opa` | 151 | 2 | 149 | 117 | 1 | 0 | 31 | 264 | + + +**How to read it.** *Fact not supplied* is a posture only the repository's owners can declare — tenant filtering, the runtime intent, the CI system's findings; the OPA policies decide those when the caller supplies them through `facts.satellite` (GT-694), and the native engine never does. That is the one reason left to run `--engine opa`, and it is not coverage: on a bare run the default decides more, on both rows. *Adapter needed* is an external or running system nobody has wired through the enforcer seam yet. *Documentation* is a rule with no check as written — mostly the generated ADR-conformance placeholders, `documentation-only` on both engines by decision. *Engine debt* is a handler or a policy nobody wrote; every one carries a recorded decision in [`engine-coverage-decisions.json`](../.harness/scripts/ci/engine-coverage-decisions.json), and the OPA-only side of the [coverage baseline](../.harness/scripts/ci/engine-coverage-parity.baseline.json) is empty. The report says the same thing in the same groups: when an engine skips more than it checks, its `GOV-ENGINE-COVERAGE` row states that run's split. + +**The published CLI is older than all of this.** `@beyondnet/evolith-cli@1.3.2` — what `npx -y @beyondnet/evolith-cli` installs today — predates GT-716. Measured on 2026-08-21 on this repository it decided 133 of 159 with `--engine opa` and 41 of 159 natively, and most of that difference was verdicts on facts nobody supplied. That is why the front page still says `--engine opa`, and says for which CLI; the next release inverts it. CI holds them to agreement over **facts**, not over coverage; that part is by design — and since GT-716 AC3 every rule only one engine decides is registered per rule, in both directions, on this repository and on a satellite fresh from `init` (`73-validate-engine-coverage-parity.mjs`), so a coverage difference is a diff somebody reads rather than a number nobody does. Since AC4 (2026-09-21) nothing in that register is there by omission: every debt entry carries a recorded decision (`engine-coverage-decisions.json`) the guard holds to the runs, and the OPA-only side is empty on both scenarios — the last rules only `--engine opa` decided from the tree (`OBS-EVD-01..03`, `MCP-05`) have native twins of their policies. That the default command never says so is not ([#628](https://github.com/beyondnetcode/evolith_arch32/issues/628)). That is why the front page uses `--engine opa` everywhere. Measured again on 2026-09-20 with the CLI built from this tree: the default decides 56 of the same 159, and of the 76 rules only `--engine opa` decides, 73 are verdicts on facets a bare run never supplies — so most of that extra coverage is not coverage. Tracked as GT-716 in the [Gap Tracking Board](../reference/core/control-center/gaps/gap-tracking.md). Since `b2840947` (in the tree, not yet in a published CLI) the OPA engine reports those rules as `skipped` with the facet it lacks: on the same satellite it decides 10 of 159 from what a bare run observes, and the rest only when the caller supplies the facts through `facts.satellite`. diff --git a/reference/core/control-center/evidence/gap-closure-evidence.json b/reference/core/control-center/evidence/gap-closure-evidence.json index dc16b4df3..bc234881c 100644 --- a/reference/core/control-center/evidence/gap-closure-evidence.json +++ b/reference/core/control-center/evidence/gap-closure-evidence.json @@ -10936,6 +10936,50 @@ "grep -n 'build-pages' .github/workflows/docs.yml && grep -n 'branches\\|permissions\\|contents:' .github/workflows/pages.yml", "EXPECTED OF THE PREVIOUS COMMAND: docs.yml runs `node .harness/scripts/pages/build-pages.mjs --check` and `node --test .harness/scripts/pages/build-pages.test.mjs` inside the `validate` job (name: Validate documentation, required); pages.yml triggers on push to main with `contents: read` at the top and `contents: write` only on the publish job" ] + }, + { + "id": "GT-716", + "closedAt": "2026-09-21", + "closureCommit": "e5b4701c", + "dependencyDisposition": "none", + "evidence": [ + "src/packages/core-domain/src/application/validators/evaluators/opa-evaluator.ts", + ".harness/scripts/lib/rego-rule-inputs.mjs", + ".harness/scripts/compile-opa-wasm.mjs", + "src/rulesets/schema/facets.json", + "src/packages/core-domain/src/domain/models/declared-facts.ts", + "src/packages/core-domain/src/application/validators/rule-evaluability.ts", + ".harness/scripts/lib/rule-facts.mjs", + ".harness/scripts/ci/73-validate-engine-coverage-parity.mjs", + ".harness/scripts/ci/engine-coverage-parity.baseline.json", + ".harness/scripts/ci/engine-coverage-decisions.json", + ".harness/scripts/ci/68-validate-engine-verdict-parity.mjs", + ".harness/scripts/ci/engine-verdict-parity.baseline.json", + ".harness/scripts/lib/core-export.mjs", + "src/packages/core-domain/src/application/validators/evaluators/handlers/telemetry-evidence-rule.handler.ts", + "src/packages/core-domain/src/application/validators/evaluators/handlers/mcp-rule.handler.ts", + "src/packages/core-domain/src/application/validators/ruleset-validator.service.ts", + "docs/known-limitations.md", + "README.md", + "src/rulesets/opa/README.md" + ], + "validationCommands": [ + "MEASURED FIRST (RED), 2026-09-20, on develop 2e2a5527's ancestor b3df7e96 with the CLI built from the tree, satellite fresh from `evolith init` (159 rules in scope): `evolith validate --engine opa --format json` decided 133 and skipped 26; the default decided 56 and skipped 103. Crossed rule by rule against the `.rego` bodies: 73 of the 76 rules only OPA decided read a facet a bare run never sends (`input.satellite.{git,runtime,testing,multiTenancy,ci,findings,protocol,scorecards,layers,contracts}`, `input.adapter`, `input.context.dod`, `input.user`, the `QT-*` metrics) — verdicts on absent input, the same `supplied-facet-absent` family GT-704's baseline already named. Guard 68 printed both counts as `coverageOnly` and gated neither.", + "RED → GREEN, AC1 (b2840947): `MTN-01` on a bare satellite came back `failed` ('tenant_id filter') about a repository nobody had looked at; after the bundle manifest `evolith/manifest/rule_input_paths` and `OpaEvaluator.skippedForAbsentFacets`, the same run is `skipped` / `supplied-facet-absent` naming `input.satellite.multiTenancy`, and with the facet supplied it is `failed` on applicationFiltering:false and `passed` on true (src/packages/core-domain/src/application/validators/evaluators/opa-supplied-facts.spec.ts, against the real policy.wasm). On the fresh satellite `--engine opa` went from 133 decided to 10.", + "RED → GREEN, AC2 (3d76f4a6): declaring `facts: []` for the twelve rules the triage table called non-executable while Rego decided them (KI-R01..07, INH-03..05, PROT-03/06) turned `npm run build:policy` red with twelve findings naming the facets each policy reads; declaring what the policies read turned it green. 44 of 415 rules changed class with nothing implemented; unimplemented-native 52 → 21.", + "node .harness/scripts/ci/73-validate-engine-coverage-parity.mjs", + "EXPECTED OF THE PREVIOUS COMMAND: 'repository: native-only 81, opa-only 0; 0 unregistered, 0 stale, 0 changed class' and 'init-satellite: native-only 48, opa-only 0; 0 unregistered, 0 stale, 0 changed class', then '✓ 73-validate-engine-coverage-parity: every coverage-only rule is registered with its reason and every debt entry with a decision, in both directions, on both scenarios; the coverage tables on the page are the measured ones.' (observed 2026-09-21 on e5b4701c, exit 0). The baseline's coverage block: this repository native 109 decided / 246 skipped (37 fact not supplied, 65 adapter, 138 documentation, 6 debt), `--engine opa` 28 / 327 (146, 1, 138, 42); satellite fresh from `init` native 50 / 101 (30, 61, 4, 6), `--engine opa` 2 / 149 (117, 1, 0, 31).", + "OBSERVED RED BEFORE TRUSTED, guard 73: (AC3, 29c8a4ba) renaming `OBS-EVD-03` in telemetry-evidence.rego → red on both scenarios naming the id, green once restored; (AC4, ea736a75) the first run of the decisions register → '❌ repository: 1 decided rule(s) are decided by BOTH engines now — retire the decision: MM-R03 (decision modular-monolith-native-only)', green once the id left the decision; (AC5, e5b4701c) one number edited in docs/known-limitations.md's table → '❌ 1 page(s) do not say what the baseline measured (GT-716 AC5) — re-run with --write: docs/known-limitations.md: its table differs from the render of the registered coverage', `--write` restored it and the guard went green.", + "node .harness/scripts/ci/68-validate-engine-verdict-parity.mjs", + "EXPECTED OF THE PREVIOUS COMMAND: 'corpus 415 rule(s); 15 decided by BOTH engines; 1 verdict conflict(s), 1 of them registered.' and '✓ 68-validate-engine-verdict-parity: 15 jointly-decided rule(s), 1 conflict(s), all registered with an individual reason.' — the one conflict is CLI-EXIT-02 (the policy fails on an absent fact document the tracked tree does not carry; the native handler reads the source), measured on the export with --core (observed 2026-09-21 on e5b4701c, exit 0).", + "node --test .harness/scripts/ci/73-validate-engine-coverage-parity.test.mjs && node --test .harness/scripts/ci/68-validate-engine-verdict-parity.test.mjs && node --test .harness/scripts/lib/rego-rule-inputs.test.mjs && node --test .harness/scripts/lib/rule-facts.test.mjs", + "EXPECTED OF THE PREVIOUS COMMAND: tests 24 / pass 24; tests 9 / pass 9; tests 10 / pass 10; the rule-facts suite green — the decisions register refuses a malformed entry, a debt entry without a decision is 'coverage-only by omission', a `neither` decision is contradicted by any engine deciding one of its rules, and the page table is rendered deterministically in both languages.", + "npm run build:policy", + "EXPECTED OF THE PREVIOUS COMMAND: 'OPA manifest: every policy read is declared by its rule (415 corpus rules checked, 84 facets in the vocabulary).' and 'OPA manifest: 238 rule id(s) declared by reachable policies; 203 of them state which input paths they read.' before 'Successfully compiled and installed policy.wasm' (observed 2026-09-21).", + "cd src/packages/core-domain && npx jest --config jest.config.js --runInBand src/application/validators/rule-corpus-triage.spec.ts src/application/validators/engine-coverage-advisory.spec.ts src/application/validators/evaluators/handlers/telemetry-evidence-rule.handler.spec.ts src/application/validators/evaluators/handlers/mcp-rule.handler.spec.ts", + "EXPECTED OF THE PREVIOUS COMMAND: all green — the corpus pins read native-handler 173 · unimplemented-native 14 · needs-external-system 33 · needs-runtime 22 · needs-supplied-facts 31 · documentation-only 138 · underspecified 4, blocking rules that do not run 71 (5 handlers, 21 adapters, 14 runtime, 27 postures, 4 authoring); the advisory fires on either engine that skips more than it checks and states the per-class split; OBS-EVD-01..03 decided from the satellite's dependencies; MCP-01..03 FAIL on absent smoke evidence with mcp.rego's words.", + "OBSERVED RED BEFORE TRUSTED, CI on #797 (Test infra-providers, GT-571 invariant): the native twins failed a freshly initialised satellite on OBS-EVD-01..03 (a6a415b8: the rules declare `appliesFromSdlcPhase: 3`, Construction — their text speaks of production paths) and, on a checkout without the gitignored `.harness/evidence/`, on MCP-01..03 (076efb60: the MCP pack declares `audience: core` — its rules judge the Core's server); both engines exclude them on a scaffold before running." + ] } ] } diff --git a/reference/core/control-center/gaps/gap-reference-catalog.es.md b/reference/core/control-center/gaps/gap-reference-catalog.es.md index eefa46f4a..7a4a4ddb9 100644 --- a/reference/core/control-center/gaps/gap-reference-catalog.es.md +++ b/reference/core/control-center/gaps/gap-reference-catalog.es.md @@ -10350,15 +10350,15 @@ Los dos se arreglaron de forma estructural y no como correcciones: el rethrow no - **Ficheros afectados:** `src/packages/core-domain/src/application/validators/evaluators/opa-evaluator.ts`, `.harness/scripts/compile-opa-wasm.mjs`, `src/packages/core-domain/src/application/validators/rule-evaluability.ts`, `src/packages/core-domain/src/application/validators/evaluators/native-evaluator.ts` y `handlers/`, `src/packages/core-domain/src/application/validators/evaluators/opa-input-coverage.spec.ts`, `src/packages/core-domain/src/application/validators/rule-corpus-triage.spec.ts`, `src/rulesets/**/*.rules.json` y el esquema de reglas, `.harness/scripts/ci/68-validate-engine-verdict-parity.mjs`, `.harness/scripts/ci/engine-verdict-parity.baseline.json`, `docs/known-limitations.es.md` - **Componente:** `Core Domain` · **Criticidad:** P1 · **Complejidad:** L - **Principal:** `L` · **Interés:** `MED` · **Base:** `estimate` -- **Procedencia:** Registrado el 2026-09-20 a partir de la pregunta del propietario sobre la tabla de la página de estado real («cómo logramos la paridad»), tras correr los dos motores sobre los dos escenarios y cruzar cada regla de un solo motor contra el cuerpo de la política que la decide. Las cifras publicadas (41 / 133 sobre 1.3.2) no se discuten; el 56 de aquí es la misma medición sobre el árbol actual. +- **Procedencia:** Registrado el 2026-09-20 a partir de la pregunta del propietario sobre la tabla de la página de estado real («cómo logramos la paridad»), tras correr los dos motores sobre los dos escenarios y cruzar cada regla de un solo motor contra el cuerpo de la política que la decide. Las cifras publicadas (41 / 133 sobre 1.3.2) no se discuten; el 56 de aquí es la misma medición sobre el árbol actual. **Cerrado el 2026-09-21**, un criterio por pull request: AC1 `b2840947` ([#788](https://github.com/beyondnetcode/evolith_arch32/pull/788)), AC2 `3d76f4a6` ([#793](https://github.com/beyondnetcode/evolith_arch32/pull/793)), AC3 `29c8a4ba` ([#795](https://github.com/beyondnetcode/evolith_arch32/pull/795)), AC4 `ea736a75` ([#797](https://github.com/beyondnetcode/evolith_arch32/pull/797)), AC5 `e5b4701c` ([#799](https://github.com/beyondnetcode/evolith_arch32/pull/799)). - **Criterios de aceptación:** - [x] **Una faceta suministrada ausente es `skipped` en OPA, nunca un veredicto.** El manifiesto del bundle lleva, por id de regla, las rutas de input que lee su política (`compile-opa-wasm.mjs` ya parsea el AST de cada política), y `OpaEvaluator` devuelve `skipped` con una clase que nombra la faceta ausente cuando el input no la lleva. **FALSABILIDAD:** sobre el satélite nuevo `--engine opa` pasa de 133 decididas a 60 como máximo, las siete entradas `supplied-facet-absent` de `engine-verdict-parity.baseline.json` se eliminan porque el guard 68 las reporta obsoletas, y suministrar la faceta por el canal de GT-694 sigue devolviendo `MTN-01` — una evaluación real, no una respuesta fija. Ambas salidas registradas. **CUMPLIDO en `b2840947`, y el falsador se movió más lejos de lo que pedía el criterio.** `compile-opa-wasm.mjs` compila un segundo entrypoint de manifiesto, `evolith/manifest/rule_input_paths` (200 de los 238 ids declarados enuncian qué leen; los otros 38 son ids de gate), extraído del AST del compilador por `.harness/scripts/lib/rego-rule-inputs.mjs` — lecturas directas, cabeceras, reglas auxiliares seguidas transitivamente — y `OpaEvaluator` devuelve `skipped` / `supplied-facet-absent` nombrando la faceta cuando el input no lleva nada de lo que la regla lee; `27-opa-parity-gate` hace fallar un bundle que deje de exponerlo. Medido sobre el mismo satélite nuevo: `--engine opa` pasó de **133 decididas a 10**, no a 60 — el «73» que registró esta fila contaba solo las reglas que el nativo saltaba, y 47 de las 57 reglas que ambos motores «decidían» eran también veredictos de OPA sobre input ausente (`DOD-*` sobre `context`, `TAX-*` sobre `repository`, `KI-R*` sobre `knowledge_id`/`review`, `INH-02` sobre `contracts`…); las 10 que quedan leen lo que el builder observa (`INH-06` falla por el `DECISIONS.md` ausente, `OBS-EVD-01/02/03` por los paquetes ausentes — `OBS-EVD-03` lee `packageJson`, no `scorecards` como dice la tabla de arriba; gana el AST). El guard 68 reportó **ocho** entradas obsoletas en la línea base, no siete: las siete `supplied-facet-absent` y `TAX-01`, cuya razón registrada (listas de exención distintas) era errónea — `repository-taxonomy.rego` lee `input.repository.files`, una faceta que ningún canal transporta, así que su `passed` era una lista ausente; las ocho eliminadas, quedan 3 conflictos (`CLI-EXIT-01/03`, `GOV-RULE-NON-EXECUTABLE`), decididas por ambos 28 → 14. Suministrar la faceta por el canal de GT-694 devuelve `MTN-01` `failed` con `applicationFiltering: false` y `passed` con `true` (`opa-supplied-facts.spec.ts`, bundle real). Sobrevive una excepción mantenida a mano, `ABSENCE_IS_A_FACT` (`qualityEvidence`, `evaluationDate`, `qualityAdmissibilityPolicy`, `evidence`, `waiver`, `tenantId`): facetas cuya ausencia ambos motores ya tratan como hecho (`PEA-01..04` pasan sobre una ejecución a secas en nativo y en Rego, ADR-0111) — el AC2 la lleva a los ficheros de regla. `GOV-ENGINE-COVERAGE` ya no le dice al lector que OPA «decide más». - [x] **Una sola declaración de evaluabilidad por regla, y ambos motores derivan de ella.** Cada regla de `*.rules.json` declara los hechos que necesita y su procedencia (`observed` / `supplied` / `runtime` / `external` / `none`); `RULE_TRIAGE` pasa a ser una proyección de ella (o una comprobación contra ella), y el manifiesto del bundle también. Un guard falla cuando un `.rego` lee una faceta que su regla no declara, cuando un handler nativo reclama una regla declarada no ejecutable, y cuando los dos motores clasifican distinto la misma regla. **FALSABILIDAD:** las 12 reglas que hoy el nativo clasifica como no ejecutables y OPA decide (`KI-R01..07`, `INH-03..05`, `PROT-03`, `PROT-06`) lo ponen en rojo hasta que un lado cambie; `PEA-01..04` siguen en verde sin tocarlas. **CUMPLIDO en `3d76f4a6`.** Cada regla de `*.rules.json` (415 de 415) declara `facts` — las facetas que lee su comprobación, por id desde el nuevo vocabulario `src/rulesets/schema/facets.json` (85 facetas, cada una con su procedencia: `observed` / `supplied` / `external` / `runtime`) — y ambos motores derivan de ella: `classifyRule` toma la clase de la procedencia más exigente (`runtime` > `external` > `supplied` > `observed`; `facts: []` es documentación tras un juicio o un placeholder del generador, sin especificar tras nada), `RULE_TRIAGE` ya no existe, y `npm run build:policy` rechaza un bundle cuando una política lee una faceta que su regla no declaró o cuando una entrada del vocabulario no la declara ninguna regla ni la lee ninguna política. Se añadió una clase nativa, `needs-supplied-facts` (una postura que solo los dueños pueden declarar; OPA la decide cuando se suministra, el nativo nunca). **FALSADOR, ambas salidas registradas:** declarar `facts: []` para las doce pone el chequeo del build en rojo con doce hallazgos, uno por regla nombrando las facetas que lee su política (`INH-03 … reads corePath, satellite.contracts, satellitePath which its facts (none) do not declare`); declarar lo que leen las políticas lo devuelve a verde y mueve las doce al denominador ejecutable; `PEA-01..04` siguen `native-handler` con `qualityEvidence` declarado. **Lo que se movió cuando la declaración sustituyó a la tabla — 44 de 415 reglas, sin implementar nada:** `unimplemented-native` 52 → 21 (25 de esas filas las decidían políticas sobre una postura declarada, el sistema de CI o una ejecución de tests, nunca un handler sobre el árbol), `needs-external-system` 20 → 27, `needs-runtime` 17 → 23, `needs-supplied-facts` 0 → 31, `documentation-only` 141 → 138, `underspecified` 14 → 4; las 73 reglas bloqueantes que no corren son las mismas 73, recosteadas (11 handlers, 16 adaptadores, 15 observadores de runtime, 27 posturas declaradas, 4 decisiones de autoría). El guard 68 declaró entonces obsoleto `GOV-RULE-NON-EXECUTABLE` — con las doce ejecutables, nada no ejecutable queda en alcance en este repositorio —, así que la línea base queda en `CLI-EXIT-01/03`. Sobre el satélite nuevo no cambió lo que decide ningún motor (nativo 56, OPA 10); las siete filas `KI-R` hacen fallar ahora la ejecución como `needs-supplied-facts` en vez de esconderse en el conteo de no ejecutables. Snapshot, mapeo ISO 5055 (backlog 21: 9 adoptables, 2 parciales, 10 por escribir) y el README de estándares rederivados. Un juicio queda en código: `ABSENCE_IS_A_FACT`, reducido a las cuatro facetas cuya ausencia ambos motores ya tratan como hecho. - [x] **La diferencia de cobertura es un ratchet por regla, en ambas direcciones y en ambos escenarios.** `68-validate-engine-verdict-parity.mjs` (o un hermano que reutilice su `deriveOutcomes`) deja en línea base cada id de `coverageOnly` con su propia razón medida y su seguimiento, sobre la raíz del repositorio Y sobre un satélite producido por `init`; una regla de un solo motor sin registrar falla, y una registrada que ahora deciden ambos falla hasta que se retire su entrada. **FALSABILIDAD:** quitar un `import` de `main.rego` lo pone en rojo nombrando los ids que el bundle dejó de decidir; restaurarlo lo devuelve a verde. **CUMPLIDO en `29c8a4ba`.** `73-validate-engine-coverage-parity.mjs` — hermano de 68 que reutiliza su `deriveOutcomes` — corre ambos motores sobre la raíz del repositorio y sobre un satélite que crea con `evolith init` en un directorio temporal, y sujeta cada regla que solo un motor decide a `engine-coverage-parity.baseline.json`: por escenario, por dirección, por regla, con la razón que dio el otro motor — la clase que enuncia su informe y las facetas que nombra su fila de salto cuando la fila existe; el manifiesto del bundle y el conjunto que emite el constructor de input cuando no — y un seguimiento por clase. Una regla sin registrar, una entrada obsoleta o una clase cambiada hacen fallar; `--write` regenera el fichero para revisión. Ambos escenarios leen el Core desde una exportación del árbol versionado más el bundle compilado — la primera corrida en CI discrepó del portátil que escribió la línea base (un directorio `coverage/` local decidía `EM-Y-01`/`QT-01`, el historial git decidía `DRIFT-01`, y la corrida sobre el árbol de trabajo resolvía las referencias de las reglas ADR-conformance contra la copia empaquetada de la CLI y fallaba 138 de ellas falsamente), así que los artefactos no versionados no pueden voltear una regla entre máquinas. Medido el 2026-09-20 sobre esa exportación: repositorio 82 solo-nativo (52 `supplied-facet-absent`, 26 `no-policy-in-bundle`, 4 en que la vía OPA saltó sin una razón propia — las `HXA-01/02/04/05` enrutadas al enforcer, archivadas como `opa-gave-no-reason` y no como deuda de handler) y 7 solo-OPA (`MCP-01..03` `unimplemented-native`, `OBS-EVD-01/02` `needs-runtime`, `OBS-EVD-03` `needs-external-system`, `MCP-05` handler que declinó); satélite `init` 49 solo-nativo (34 / 11 / 4) y las mismas 7 solo-OPA. **FALSADOR, ambas salidas registradas:** la prueba que nombraba este criterio — quitar un `import` de `main.rego` — no puede correr desde GT-675, porque el build del bundle rechaza una política alcanzable que nadie importa; la prueba equivalente es renombrar `OBS-EVD-03` en `telemetry-evidence.rego` para que el bundle deje de decidirla: el guard se puso en rojo en ambos escenarios nombrando el id (`opaOnly OBS-EVD-03 … no longer coverage-only`), y volvió a verde al restaurarla. Corre en el job `Test` tras 68 (≈17 s + ≈3 s); clasificado INSTRUMENTED por el guard 42 y en rojo en el sandbox vacío del guard 43; 11 tests unitarios. `known-limitations` dice ahora que CI registra las diferencias de cobertura por regla en vez de solo permitirlas. - [x] **El backlog nombrado queda implementado o declarado, nada de él dejado en `coverageOnly` por omisión:** handlers nativos para `MCP-05`, `OBS-EVD-01`, `OBS-EVD-02` (y, sobre el corpus, `MCP-01..04`, `DEP-08`, `TAX-07/08`); un `.rego` para las 11 reglas `no-policy-in-bundle` nombradas arriba; ambos para las 7 que no decide ningún motor; y una decisión registrada para las 138 reglas ADR-conformance y `MM-R*` — un gemelo `.rego` generado, o `documentation-only` en los dos motores. **CUMPLIDO en `ea736a75`.** Implementado: `TelemetryEvidenceRuleHandler` decide `OBS-EVD-01..03` desde las dependencias del satélite — el mismo proxy y las mismas listas de paquetes que `telemetry-evidence.rego`, luego el mismo veredicto (las reglas declaran `satellite.packageJson` y nada que no lean; `telemetryBackend` salió del vocabulario, 84 facetas); `McpRuleHandler` decide `MCP-05` desde la fuente del servidor y falla `MCP-01..03` por evidencia de humo ausente con las palabras de la propia política, donde antes saltaba; `MCP-04`, `DEP-08` y `TAX-07/08` ya las decidían ambos motores o estaban registradas como `supplied-facet-absent`. Sobre un scaffold en fase 0 las tres reglas de telemetría quedan **no aplicables** en vez de fallidas: declaran `appliesFromSdlcPhase: 3` (Construcción), que es lo que dice su texto — rutas de petición y servicios *en producción* — y lo que MTN-05 ya hacía para Diseño; el invariante de GT-571 de que un satélite recién inicializado no hace nada mal se cumplía en el motor nativo y se cumple ahora en los dos (`--engine opa` fallaba las tres en cada scaffold; CI cazó al gemelo nativo haciendo lo mismo antes de la anotación). La misma corrida cazó a `MCP-01..03` fallando un satélite recién creado validado contra un Core sin evidencia de humo — tres hallazgos bloqueantes dirigidos al Core, la queja de GT-571 al pie de la letra —, así que el pack MCP declara `audience: core` donde lo lee la aplicabilidad, como hacen los packs de la CLI y como su `scope: core-cli` siempre dijo; sobre el Core ambos motores siguen decidiendo las cinco. Declarado, en la regla: las siete que no decidía ningún motor (`SEC-INJ-01/02`, `SEC-PATH-01/02`, `SEC-TIMING-01/02`, `SEC-RL-03`) declaran `satellite.findings` — si `child_process.exec` recibe input interpolado o si una comparación de credenciales es en tiempo constante es el hallazgo de un escáner sobre el AST, no una regex sobre el árbol, que `MM-R10` en este mismo corpus prohíbe — y `QT-05`, que `SdlcRuleHandler` respondía `passed` con «requiere análisis en runtime», declara `satellite.testing` y no la reclama nadie. Registrado y exigido: `engine-coverage-decisions.json`, ocho decisiones — `SSDF-*` ×8, `SLSA-*` ×4, `SEC-RL-01/02`, `MM-R*` ×11 y las `HXA-01/02/04/05` enrutadas al enforcer como `native-only`, cada una con su porqué y con lo que la reabriría; las 138 reglas ADR-conformance generadas como un patrón `neither`, `documentation-only` en los dos motores ahora que `OpaEvaluator` enuncia la clase de la declaración para una regla sin política cuyo `facts: []` dice que no hay nada que comprobar; las siete reglas de escáner y `QT-05` como `neither` — y el guard 73 hace fallar una entrada de clase deuda sin decisión como solo-un-motor por omisión, y una decisión que las ejecuciones contradigan. **FALSADOR, observado antes de confiar en él:** la primera corrida del guard extendido se puso en rojo con `MM-R03 (decision modular-monolith-native-only)` — `modular-monolith.rego` la decide y ambos motores coinciden, cosa que el registro no decía — y en verde en cuanto el id salió de la decisión. Medido el 2026-09-21 sobre la exportación: solo-OPA **0 / 0** (era 7 / 7); solo-nativo 81 / 48 (era 82 / 49, salió `QT-05`) — 52 / 34 `supplied-facet-absent`, 25 / 10 `no-policy-in-bundle` con una decisión cada una, 4 `opa-gave-no-reason` con una. **No hecho tal como el criterio lo formuló al principio:** no se escribió ningún gemelo `.rego` para las 11 reglas sin política — cada una es una decisión registrada (`SSDF`/`SLSA`: el constructor de input no proyecta nada de lo que lee el handler, así que un gemelo es la misma comprobación tras una segunda proyección; `SEC-RL-01/02`: un escaneo de fuente; `MM-R*`: un grafo de imports que la vía OPA no lleva hasta que la costura de GT-514 lo alimente). Costes: `unimplemented-native` 21 → 14, reglas bloqueantes que no corren 73 → 71 (5 handlers, 21 adaptadores, 14 runtime, 27 posturas, 4 de autoría), backlog ISO 5055 14 (5 adoptables, 9 por escribir). El guard 68 ganó un conflicto registrado, `GOV-RULE-NON-EXECUTABLE`: en su corrida sin `--core` el motor nativo resuelve el corpus a la copia empaquetada de la CLI y falla las 138 reglas ADR-conformance (0 no ejecutables) mientras OPA ahora cuenta 138; con `--core .` ambos motores emiten la misma fila — llevar el 68 a la exportación es del AC5. - - [ ] **El informe y la página dicen lo mismo.** `GOV-ENGINE-COVERAGE` y `docs/known-limitations.es.md` enuncian la cobertura por motor con el desglose de facetas suministradas, y la portada o deja de necesitar `--engine opa` por razones de cobertura o dice qué razón queda. + - [x] **El informe y la página dicen lo mismo.** `GOV-ENGINE-COVERAGE` y `docs/known-limitations.es.md` enuncian la cobertura por motor con el desglose de facetas suministradas, y la portada o deja de necesitar `--engine opa` por razones de cobertura o dice qué razón queda. **CUMPLIDO en `e5b4701c` ([#799](https://github.com/beyondnetcode/evolith_arch32/pull/799)).** El informe: `GOV-ENGINE-COVERAGE` se dispara en cualquier motor que salte más de lo que comprueba y enuncia la cobertura de esa corrida con sus saltos desglosados por clase — *hecho no suministrado* (`needs-supplied-facts` / `supplied-facet-absent`, con el remedio `facts.satellite`), *falta adaptador*, *documentación*, *deuda del motor* — desde un nuevo `RuleCoverage.skippedByEvaluability` que viaja al informe JSON; la frase «un salto suele significar que el evaluador nativo no tiene handler» desaparece, porque era cierta de 14 reglas y se decía de 240. La página: `docs/known-limitations.md` / `.es.md` llevan el mismo desglose por motor y escenario en una tabla que `73-validate-engine-coverage-parity.mjs --write` renderiza entre marcadores `engine-coverage` desde un bloque de cobertura que ahora registra en la línea base; la corrida por defecto falla cuando la cobertura medida difiere del bloque registrado Y cuando la tabla de la página difiere de su render — la tabla no se puede teclear. Medido el 2026-09-21 sobre la exportación: este repositorio nativo 109 decididas / 246 saltadas (37 hecho no suministrado, 65 adaptador, 138 documentación, 6 deuda) frente a `--engine opa` 28 / 327 (146, 1, 138, 42); satélite recién salido de `init` nativo 50 / 101 (30, 61, 4, 6) frente a `--engine opa` 2 / 149 (117, 1, 0, 31); 60 y 264 no aplicables. La portada dice qué razón queda: `--engine opa` tiene la mayor cobertura **en la CLI publicada (1.3.2)**, anterior a GT-716; en el árbol el motor por defecto decide más en una ejecución a secas y `--engine opa` es para los hechos que el llamador suministra — lo único que el motor nativo nunca decide. El activo de la demo se queda: sus filas son una corrida literal de 1.3.2. El guard 68 pasó a la misma exportación con `--core` (el seguimiento del AC4): CLI-EXIT-01/03 y GOV-RULE-NON-EXECUTABLE quedaron obsoletos — artefactos de la corrida sin `--core` — y queda un conflicto, CLI-EXIT-02 (`cli-exit-code-taxonomy.rego` falla por un documento de hechos ausente, `core.cli.exitCodes`, que el árbol versionado no lleva, mientras el handler nativo lee la fuente), registrado con su seguimiento. **FALSADOR, ambas salidas registradas:** una cifra editada en la tabla de la página puso el guard 73 en rojo nombrando la página (`docs/known-limitations.md: its table differs from the render of the registered coverage`); `--write` la restauró y el guard volvió a verde. - **Dependencias:** GT-694 (el canal de facetas suministradas, COMPLETADO), GT-675 (el manifiesto del bundle, COMPLETADO), GT-704 (la línea base de veredictos que esta extiende, COMPLETADO); la costura del enforcer (GT-514) para `satellite.layers`. -- **Estado:** `PENDIENTE` +- **Estado:** `COMPLETADO` #### GT-717 diff --git a/reference/core/control-center/gaps/gap-reference-catalog.md b/reference/core/control-center/gaps/gap-reference-catalog.md index e95a60d06..f39760897 100644 --- a/reference/core/control-center/gaps/gap-reference-catalog.md +++ b/reference/core/control-center/gaps/gap-reference-catalog.md @@ -10443,15 +10443,15 @@ Both were fixed structurally rather than corrected: the rethrow now names BOTH f - **Affected files:** `src/packages/core-domain/src/application/validators/evaluators/opa-evaluator.ts`, `.harness/scripts/compile-opa-wasm.mjs`, `src/packages/core-domain/src/application/validators/rule-evaluability.ts`, `src/packages/core-domain/src/application/validators/evaluators/native-evaluator.ts` and `handlers/`, `src/packages/core-domain/src/application/validators/evaluators/opa-input-coverage.spec.ts`, `src/packages/core-domain/src/application/validators/rule-corpus-triage.spec.ts`, `src/rulesets/**/*.rules.json` and the rule schema, `.harness/scripts/ci/68-validate-engine-verdict-parity.mjs`, `.harness/scripts/ci/engine-verdict-parity.baseline.json`, `docs/known-limitations.md` - **Component:** `Core Domain` · **Criticality:** P1 · **Complexity:** L - **Principal:** `L` · **Interest:** `MED` · **Basis:** `estimate` -- **Provenance:** Registered 2026-09-20 from the owner's question over the known-limitations table ("how do we reach parity"), after running both engines over both scenarios and crossing every one-engine rule against the policy body that decides it. The published figures (41 / 133 on 1.3.2) are not disputed; the 56 here is the same measurement on the current tree. +- **Provenance:** Registered 2026-09-20 from the owner's question over the known-limitations table ("how do we reach parity"), after running both engines over both scenarios and crossing every one-engine rule against the policy body that decides it. The published figures (41 / 133 on 1.3.2) are not disputed; the 56 here is the same measurement on the current tree. **Closed 2026-09-21**, one criterion per pull request: AC1 `b2840947` ([#788](https://github.com/beyondnetcode/evolith_arch32/pull/788)), AC2 `3d76f4a6` ([#793](https://github.com/beyondnetcode/evolith_arch32/pull/793)), AC3 `29c8a4ba` ([#795](https://github.com/beyondnetcode/evolith_arch32/pull/795)), AC4 `ea736a75` ([#797](https://github.com/beyondnetcode/evolith_arch32/pull/797)), AC5 `e5b4701c` ([#799](https://github.com/beyondnetcode/evolith_arch32/pull/799)). - **Acceptance criteria:** - [x] **An absent supplied facet is `skipped` on OPA, never a verdict.** The bundle manifest carries, per rule id, the input paths its policy reads (`compile-opa-wasm.mjs` already parses each policy's AST), and `OpaEvaluator` reports `skipped` with a class that names the missing facet when the input does not carry it. **FALSIFIABILITY:** on the fresh satellite `--engine opa` moves from 133 decided to at most 60, the seven `supplied-facet-absent` entries of `engine-verdict-parity.baseline.json` are removed because guard 68 reports them stale, and supplying the facet through GT-694's channel still returns `MTN-01` — a real evaluation, not a fixed answer. Both outputs recorded. **MET in `b2840947`, and the falsifier moved further than the criterion asked.** `compile-opa-wasm.mjs` compiles a second manifest entrypoint, `evolith/manifest/rule_input_paths` (200 of the 238 declared ids state what they read; the 38 others are gate ids), extracted from the compiler's AST by `.harness/scripts/lib/rego-rule-inputs.mjs` — direct reads, heads, helper rules followed transitively — and `OpaEvaluator` reports `skipped` / `supplied-facet-absent` naming the facet when the input carries none of what the rule reads; `27-opa-parity-gate` fails a bundle that stops exposing it. Measured on the same fresh satellite: `--engine opa` went from **133 decided to 10**, not to 60 — the "73" this row registered counted only the rules the native engine skipped, and 47 of the 57 rules both engines "decided" were OPA verdicts on absent input as well (`DOD-*` on `context`, `TAX-*` on `repository`, `KI-R*` on `knowledge_id`/`review`, `INH-02` on `contracts`…); the 10 that remain read what the builder observes (`INH-06` fails on the missing `DECISIONS.md`, `OBS-EVD-01/02/03` on the missing packages — `OBS-EVD-03` reads `packageJson`, not `scorecards` as the table above says; the AST wins). Guard 68 reported **eight** baseline entries stale, not seven: the `supplied-facet-absent` seven and `TAX-01`, whose registered reason (differing exemption lists) was wrong — `repository-taxonomy.rego` reads `input.repository.files`, a facet no channel carries, so its `passed` was an absent list; all eight removed, 3 conflicts remain (`CLI-EXIT-01/03`, `GOV-RULE-NON-EXECUTABLE`), jointly decided 28 → 14. Supplying the facet through GT-694's channel returns `MTN-01` `failed` on `applicationFiltering: false` and `passed` on `true` (`opa-supplied-facts.spec.ts`, real bundle). One hand-kept exemption survives, `ABSENCE_IS_A_FACT` (`qualityEvidence`, `evaluationDate`, `qualityAdmissibilityPolicy`, `evidence`, `waiver`, `tenantId`): facets whose absence both engines already treat as a fact (`PEA-01..04` pass on a bare run natively and in Rego, ADR-0111) — AC2 moves it into the rule files. `GOV-ENGINE-COVERAGE` no longer tells the reader OPA "decides more". - [x] **One evaluability declaration per rule, and both engines derive from it.** Each rule in `*.rules.json` declares the facts it needs and their provenance (`observed` / `supplied` / `runtime` / `external` / `none`); `RULE_TRIAGE` becomes a projection of it (or a check against it), and so does the bundle manifest. A guard fails when a `.rego` reads a facet its rule does not declare, when a native handler claims a rule declared non-executable, and when the two engines class the same rule differently. **FALSIFIABILITY:** the 12 rules native classes non-executable and OPA decides today (`KI-R01..07`, `INH-03..05`, `PROT-03`, `PROT-06`) turn it red until one side changes; `PEA-01..04` stay green untouched. **MET in `3d76f4a6`.** Every rule in `*.rules.json` (415 of 415) declares `facts` — the facets its check reads, by id from the new vocabulary `src/rulesets/schema/facets.json` (85 facets, each with its provenance: `observed` / `supplied` / `external` / `runtime`) — and both engines derive from it: `classifyRule` takes the class of the most demanding provenance (`runtime` > `external` > `supplied` > `observed`; `facts: []` is documentation behind a judgement or a generator placeholder, underspecified behind nothing), `RULE_TRIAGE` is gone, and `npm run build:policy` refuses a bundle when a policy reads a facet its rule did not declare or when a vocabulary entry is declared by no rule and read by no policy. One native class was added, `needs-supplied-facts` (a posture only the owners can declare; OPA decides it when supplied, native never). **FALSIFIER, both outputs recorded:** declaring `facts: []` for the twelve turns the build check red with twelve findings, one per rule naming the facets its policy reads (`INH-03 … reads corePath, satellite.contracts, satellitePath which its facts (none) do not declare`); declaring what the policies read turns it green and moves all twelve into the executable denominator; `PEA-01..04` stay `native-handler` with `qualityEvidence` declared. **What moved when the declaration replaced the table — 44 of 415 rules, nothing implemented:** `unimplemented-native` 52 → 21 (25 of those rows were decided by policies over a declared posture, the CI system or a test run, never by a handler over the tree), `needs-external-system` 20 → 27, `needs-runtime` 17 → 23, `needs-supplied-facts` 0 → 31, `documentation-only` 141 → 138, `underspecified` 14 → 4; the 73 blocking rules that do not run are the same 73, re-costed (11 handlers, 16 adapters, 15 runtime observers, 27 declared postures, 4 authoring decisions). Guard 68 then declared `GOV-RULE-NON-EXECUTABLE` stale — with the twelve executable, nothing non-executable remains in scope on this repository — so the baseline is down to `CLI-EXIT-01/03`. On the fresh satellite nothing changed in what either engine decides (native 56, OPA 10); the seven `KI-R` rows now fail the run as `needs-supplied-facts` instead of hiding inside the non-executable count. Snapshot, ISO 5055 mapping (backlog 21: 9 adoptable, 2 partial, 10 to author) and the standards README re-derived. One judgement stays in code: `ABSENCE_IS_A_FACT`, down to the four facets whose absence both engines already treat as a fact. - [x] **The coverage difference is a per-rule ratchet, both directions, both scenarios.** `68-validate-engine-verdict-parity.mjs` (or a sibling reusing its `deriveOutcomes`) baselines every id in `coverageOnly` with its own measured reason and follow-up, over the repository root AND a satellite produced by `init`; an unregistered one-engine rule fails, and a registered one that both engines now decide fails until its entry is removed. **FALSIFIABILITY:** removing one `import` from `main.rego` turns it red naming the ids the bundle stopped deciding; restoring it turns it green. **MET in `29c8a4ba`.** `73-validate-engine-coverage-parity.mjs` — a sibling of 68 reusing its `deriveOutcomes` — runs both engines on the repository root and on a satellite it creates with `evolith init` in a temporary directory, and holds every rule only one engine decides to `engine-coverage-parity.baseline.json`: per scenario, per direction, per rule, with the reason the other engine gave — the class its report states and the facets its skip row names when the row exists; the bundle manifest and the input builder's emitted set when it does not — and a follow-up per class. An unregistered rule, a stale entry or a changed class fails; `--write` regenerates the file for review. Both scenarios read the Core from an export of the tracked tree plus the compiled bundle — the first CI run disagreed with the laptop that wrote the baseline (a local `coverage/` directory decided `EM-Y-01`/`QT-01`, git history decided `DRIFT-01`, and the working-tree run resolved the ADR-conformance rules' references against the CLI's bundled copy and failed 138 of them falsely), so untracked artifacts cannot flip a rule between machines. Measured 2026-09-20 on that export: repository 82 native-only (52 `supplied-facet-absent`, 26 `no-policy-in-bundle`, 4 where the OPA path skipped without a reason of its own — the enforcer-routed `HXA-01/02/04/05`, filed as `opa-gave-no-reason` rather than as handler debt) and 7 opa-only (`MCP-01..03` `unimplemented-native`, `OBS-EVD-01/02` `needs-runtime`, `OBS-EVD-03` `needs-external-system`, `MCP-05` handler declined); init satellite 49 native-only (34 / 11 / 4) and the same 7 opa-only. **FALSIFIER, both outputs recorded:** the probe this criterion named — removing an import from `main.rego` — cannot run since GT-675, because the bundle build refuses a reachable policy nobody imports; the equivalent probe is renaming `OBS-EVD-03` in `telemetry-evidence.rego` so the bundle stops deciding it: the guard went red on both scenarios naming the id (`opaOnly OBS-EVD-03 … no longer coverage-only`), and green again once restored. Runs in the `Test` job after 68 (≈17 s + ≈3 s); classified INSTRUMENTED by guard 42 and red in guard 43's empty sandbox; 11 unit tests. `known-limitations` now says CI registers coverage differences per rule rather than merely allowing them. - [x] **The named backlog is implemented or declared, none of it left in `coverageOnly` by omission:** native handlers for `MCP-05`, `OBS-EVD-01`, `OBS-EVD-02` (and, over the corpus, `MCP-01..04`, `DEP-08`, `TAX-07/08`); a `.rego` for the 11 `no-policy-in-bundle` rules named above; both for the 7 neither engine decides; and one recorded decision for the 138 ADR-conformance rules and `MM-R*` — a generated `.rego` twin, or `documentation-only` on both engines. **MET in `ea736a75`.** Implemented: `TelemetryEvidenceRuleHandler` decides `OBS-EVD-01..03` from the satellite's dependencies — the same proxy and the same package lists as `telemetry-evidence.rego`, so the same verdict (the rules declare `satellite.packageJson` and nothing they do not read; `telemetryBackend` left the vocabulary, 84 facets); `McpRuleHandler` decides `MCP-05` from the server source and fails `MCP-01..03` on absent smoke evidence with the policy's own words, where it used to skip; `MCP-04`, `DEP-08` and `TAX-07/08` were already decided by both engines or registered as `supplied-facet-absent`. On a phase-0 scaffold the three telemetry rules are **not applicable** rather than failed: they declare `appliesFromSdlcPhase: 3` (Construction), which is what their text says — *production* request paths and services — and what MTN-05 already did for Design; the GT-571 invariant that a freshly initialised satellite does nothing wrong held on the native engine and holds on both now (`--engine opa` used to fail the three on every scaffold; CI caught the native twin doing the same before the annotation). The same run caught `MCP-01..03` failing a fresh satellite validated against a Core with no smoke evidence — three blocking findings addressed to the Core, the GT-571 complaint verbatim — so the MCP pack declares `audience: core` where applicability reads it, as the CLI packs do and as its `scope: core-cli` always said; on the Core both engines still decide all five. Declared, in the rule: the seven neither engine decided (`SEC-INJ-01/02`, `SEC-PATH-01/02`, `SEC-TIMING-01/02`, `SEC-RL-03`) declare `satellite.findings` — whether `child_process.exec` receives interpolated input or a credential comparison is constant-time is a scanner's finding over the AST, not a regex over the tree, which `MM-R10` in this same corpus forbids — and `QT-05`, which `SdlcRuleHandler` answered `passed` with "requires runtime analysis", declares `satellite.testing` and is claimed by nobody. Recorded and enforced: `engine-coverage-decisions.json`, eight decisions — `SSDF-*` ×8, `SLSA-*` ×4, `SEC-RL-01/02`, `MM-R*` ×11 and the enforcer-routed `HXA-01/02/04/05` as `native-only`, each with why and what reopens it; the 138 generated ADR-conformance rules as a `neither` pattern, `documentation-only` on both engines now that `OpaEvaluator` states the declaration's class for a policy-less rule whose `facts: []` say there is nothing to check; the seven scanner rules and `QT-05` as `neither` — and guard 73 fails a debt-class entry with no decision as coverage-only by omission, and a decision the runs contradict. **FALSIFIER, observed before it was trusted:** the extended guard's first run went red on `MM-R03 (decision modular-monolith-native-only)` — `modular-monolith.rego` decides it and both engines agree, which the register had not said — and green once the id left the decision. Measured 2026-09-21 on the export: opa-only **0 / 0** (was 7 / 7); native-only 81 / 48 (was 82 / 49, `QT-05` left) — 52 / 34 `supplied-facet-absent`, 25 / 10 `no-policy-in-bundle` with a decision each, 4 `opa-gave-no-reason` with one. **Not done as the criterion first phrased it:** no `.rego` twin was authored for the 11 no-policy rules — each is a recorded decision instead (`SSDF`/`SLSA`: the input builder projects none of what the handler reads, so a twin is the same check behind a second projection; `SEC-RL-01/02`: a source scan; `MM-R*`: an import graph the OPA path does not carry until GT-514's seam feeds it). Costs: `unimplemented-native` 21 → 14, blocking rules that do not run 73 → 71 (5 handlers, 21 adapters, 14 runtime, 27 postures, 4 authoring), ISO 5055 backlog 14 (5 adoptable, 9 to author). Guard 68 gained one registered conflict, `GOV-RULE-NON-EXECUTABLE`: on its `--core`-less run the native engine resolves the corpus to the CLI's bundled copy and fails the 138 ADR-conformance rules (0 non-executable) while OPA now counts 138; with `--core .` both engines emit the identical row — moving 68 onto the export is AC5's. - - [ ] **The report and the page say the same thing.** `GOV-ENGINE-COVERAGE` and `docs/known-limitations.md` state coverage per engine with the supplied-facet split, and the front page either stops needing `--engine opa` for coverage reasons or says which reason remains. + - [x] **The report and the page say the same thing.** `GOV-ENGINE-COVERAGE` and `docs/known-limitations.md` state coverage per engine with the supplied-facet split, and the front page either stops needing `--engine opa` for coverage reasons or says which reason remains. **MET in `e5b4701c` ([#799](https://github.com/beyondnetcode/evolith_arch32/pull/799)).** The report: `GOV-ENGINE-COVERAGE` fires on either engine that skips more than it checks and states that run's coverage with its skips split by class — *fact not supplied* (`needs-supplied-facts` / `supplied-facet-absent`, with the `facts.satellite` remedy), *adapter needed*, *documentation*, *engine debt* — from a new `RuleCoverage.skippedByEvaluability` that travels into the JSON report; the sentence "a skip usually means the native evaluator has no handler" is gone, because it was true of 14 rules and said of 240. The page: `docs/known-limitations.md` / `.es.md` carry the same split per engine and scenario in a table `73-validate-engine-coverage-parity.mjs --write` renders between `engine-coverage` markers from a coverage block it now records in the baseline; the default run fails when the measured coverage differs from the registered block AND when the page's table differs from its render — the table cannot be typed. Measured 2026-09-21 on the export: this repository native 109 decided / 246 skipped (37 fact not supplied, 65 adapter, 138 documentation, 6 debt) against `--engine opa` 28 / 327 (146, 1, 138, 42); satellite fresh from `init` native 50 / 101 (30, 61, 4, 6) against `--engine opa` 2 / 149 (117, 1, 0, 31); 60 and 264 not applicable. The front page says which reason remains: `--engine opa` has the most coverage **on the published CLI (1.3.2)**, which predates GT-716; in the tree the default decides more on a bare run and `--engine opa` is for the facts the caller supplies — the one thing the native engine never decides. The demo asset stays: its rows are a verbatim 1.3.2 run. Guard 68 moved onto the same export with `--core` (AC4's follow-up): CLI-EXIT-01/03 and GOV-RULE-NON-EXECUTABLE went stale — artifacts of the `--core`-less run — and one conflict remains, CLI-EXIT-02 (`cli-exit-code-taxonomy.rego` fails on an absent fact document, `core.cli.exitCodes`, that the tracked tree does not carry, while the native handler reads the source), registered with its follow-up. **FALSIFIER, both outputs recorded:** one number edited in the page's table turned guard 73 red naming the page (`docs/known-limitations.md: its table differs from the render of the registered coverage`); `--write` restored it and the guard went green. - **Dependencies:** GT-694 (the supplied-facet channel, DONE), GT-675 (the bundle manifest, DONE), GT-704 (the verdict baseline this extends, DONE); the enforcer seam (GT-514) for `satellite.layers`. -- **Status:** `PENDING` +- **Status:** `DONE` #### GT-717 diff --git a/reference/core/control-center/gaps/gap-tracking.es.md b/reference/core/control-center/gaps/gap-tracking.es.md index 0334e8582..5d17ace7a 100644 --- a/reference/core/control-center/gaps/gap-tracking.es.md +++ b/reference/core/control-center/gaps/gap-tracking.es.md @@ -4,6 +4,7 @@ **Estado:** Seguimiento Activo **Responsable:** Evolith Architecture Board +**Última Actualización:** 2026-09-21 (**GT-716 cerrado: el informe, la página y la portada dicen lo mismo sobre lo que cubre cada motor.** `e5b4701c` ([#799](https://github.com/beyondnetcode/evolith_arch32/pull/799)) — AC5: `GOV-ENGINE-COVERAGE` se dispara en cualquier motor que salte más de lo que comprueba y enuncia la cobertura de esa corrida con sus saltos desglosados por clase (hecho no suministrado, falta adaptador, documentación, deuda del motor) desde un nuevo `skippedByEvaluability` en el informe; `docs/known-limitations.{md,es.md}` llevan el mismo desglose por motor y escenario en una tabla que el guard 73 renderiza desde un bloque de cobertura que registra en su línea base y comprueba en cada corrida — se observó una cifra editada poniéndolo en rojo y `--write` restaurándola. Medido el 2026-09-21 sobre la exportación: este repositorio nativo 109 / 246 frente a `--engine opa` 28 / 327; satélite `init` nativo 50 / 101 frente a 2 / 149 — el motor por defecto decide más en una ejecución a secas, en las dos filas. La portada dice qué razón queda: `--engine opa` tiene la mayor cobertura en la CLI publicada (1.3.2), anterior a GT-716; en el árbol es para los hechos que el llamador suministra. El guard 68 pasó a la misma exportación (`--core`): CLI-EXIT-01/03 y GOV-RULE-NON-EXECUTABLE obsoletos, CLI-EXIT-02 registrado con su seguimiento. Cinco criterios, cinco pull requests (#788, #793, #795, #797, #799). Contadores recalculados desde las filas: **689 / 715 completados · 3 en progreso · 2 pendientes · 21 diferidos**.) **Última Actualización:** 2026-09-21 (**Aterrizó el AC4 de GT-716: nada en el registro de cobertura está por omisión.** `ea736a75` — el backlog nombrado queda implementado o declarado, y `73-validate-engine-coverage-parity.mjs` rechaza ahora una entrada de clase deuda sin decisión registrada. Implementado: `OBS-EVD-01..03` decididas en nativo desde las dependencias del satélite (el mismo proxy que lee su política), `MCP-05` desde la fuente del servidor, `MCP-01..03` fallando por evidencia de humo ausente en los dos motores en vez de en uno — el lado solo-OPA de la línea base está **vacío en ambos escenarios** (era 7 / 7). Declarado: las siete reglas que no decidía ningún motor (`SEC-INJ/PATH/TIMING-01/02`, `SEC-RL-03`) nombran `satellite.findings`, el hallazgo de un escáner sobre el AST y no una regex sobre el árbol (`MM-R10`); `QT-05`, que el motor nativo respondía `passed` con «requiere análisis en runtime», nombra `satellite.testing` y no la decide ninguno. Registrado: `engine-coverage-decisions.json`, ocho decisiones (`SSDF`, `SLSA`, `SEC-RL-01/02`, `MM-R*`, las cláusulas `HXA` enrutadas al enforcer como `native-only`; las 138 reglas ADR-conformance generadas, las siete reglas de escáner y `QT-05` como `neither`), cada una contrastada con las ejecuciones — la primera corrida del guard se puso en rojo con `MM-R03`, que ambos motores deciden y el registro había reclamado, y en verde en cuanto salió. Las 11 reglas sin política recibieron decisiones con razones, no gemelos `.rego`. Backlog de handlers 21 → 14; reglas bloqueantes que no corren 73 → 71; el guard 68 ganó un conflicto registrado (`GOV-RULE-NON-EXECUTABLE`, un artefacto de resolución del corpus de su corrida sin `--core`, que el AC5 retirará). AC5 abierto; contadores sin cambio: **688 / 715 completados · 3 en progreso · 3 pendientes · 21 diferidos**.) **Última Actualización:** 2026-09-20 (**Aterrizó el AC3 de GT-716: lo que solo un motor decide es una diferencia registrada, no libre.** `29c8a4ba` — `73-validate-engine-coverage-parity.mjs` corre ambos motores sobre este repositorio y sobre un satélite que crea con `evolith init`, y sujeta cada regla de un solo motor a una línea base por escenario, por dirección, por regla, con la razón que dio el otro motor (la clase que enuncia su informe; las facetas que la política lee y una ejecución a secas no suministra). Una regla sin registrar, una entrada obsoleta o una clase cambiada hacen fallar. Ambos escenarios leen el Core desde una exportación del árbol versionado, porque la primera corrida en CI discrepó del portátil que escribió la línea base (un directorio `coverage/` local, el historial git y la copia empaquetada del corpus en la CLI habían decidido seis reglas y fallado 138 reglas ADR-conformance falsamente). Medido sobre esa exportación: repositorio 82 solo-nativo / 7 solo-OPA, satélite `init` 49 / 7 — 52 de las 82 son políticas que leen facetas que nadie suministró, 26 no tienen política alguna, y cuatro son las `HXA-01/02/04/05` enrutadas al enforcer, donde la vía OPA salta sin una razón propia. La prueba del criterio (quitar un `import` de `main.rego`) no puede correr desde GT-675 — el build la rechaza —, así que se registró la equivalente: renombrar `OBS-EVD-03` en su política y el guard se pone en rojo en ambos escenarios nombrando el id. AC4–AC5 abiertos; contadores sin cambio: **688 / 715 completados · 3 en progreso · 3 pendientes · 21 diferidos**.) **Última Actualización:** 2026-09-20 (**Aterrizó el AC2 de GT-716: una declaración de evaluabilidad por regla, y ambos motores derivan de ella.** `3d76f4a6` — cada regla de `*.rules.json` declara `facts`, las facetas que lee su comprobación, desde un vocabulario que dice dónde vive la verdad de cada una (`observed` / `supplied` / `external` / `runtime`); `classifyRule` deriva la clase nativa de la procedencia más exigente y la tabla de triaje por id de regla ya no existe; `npm run build:policy` rechaza un bundle cuya política lea una faceta que su regla no declaró. Declarar `facts: []` para las doce reglas que la tabla llamaba no ejecutables mientras Rego las decidía puso ese chequeo en rojo con doce hallazgos; declarar lo que leen las políticas lo devolvió a verde. **Lo que se movió, sin implementar nada: 44 de 415 reglas cambiaron de clase.** El «backlog de handlers» es 21, no 52 — 25 de esas filas las decidían políticas sobre una postura declarada, el sistema de CI o una ejecución de tests, nunca un handler sobre el árbol — y una nueva clase nativa, `needs-supplied-facts`, recoge las 31 reglas que necesitan una postura que solo los dueños del satélite pueden declarar. Las 73 reglas bloqueantes que no corren son las mismas 73, recosteadas. Snapshot, mapeo ISO 5055 y README de estándares rederivados; la línea base del guard 68 queda en `CLI-EXIT-01/03`. AC3–AC5 abiertos; contadores sin cambio: **688 / 715 completados · 3 en progreso · 3 pendientes · 21 diferidos**.) @@ -33,7 +34,7 @@ Este tablero es la única fuente de verdad para deuda técnica, gaps, oportunida | ID | Gap | En simple | Qué resuelve | Componente | Fase | Criticidad | Complejidad | Estado | |---|---|---|---|:---:|:---:|:---:|:---:|:---:| | [`GT-717`](./gap-reference-catalog.es.md#gt-717) | **Las dos vistas de GitHub Pages se publicaban a mano y llevaban tres meses envejeciendo; nada derivaba sus números del árbol.** Medido el 2026-09-19/20 contra `origin/gh-pages` (`2c8f487a`, tocado por última vez el 2026-07-07) y develop `6491c3bc`: el Atlas desplegado era anterior a la mudanza de taxonomía del 2026-07-04 — 18 de sus 21 enlaces a fuentes respondían 404 desde github.io porque `docHref` construía rutas `../` contra la raíz de Pages —, era monolingüe y tenía 4 escenarios sin explicaciones; el propio `architecture-map.json` del árbol nombraba 20 rutas de docs de las que 10 ya no existían. La vista maestra imprimía `8 controllers`, `26 tools · 9 resources`, `20 commands`, `5 KindEvaluators`, `PWA · Web · Mobile` + `BFF (NestJS · ADR-0075)` para un Tracker que es .NET 10 + React 19, y la cabecera `ADR-0101 · 0074 · 0075 · 0102`; su generador imprimía `12 hexagonal ports · 30 adapters` (línea 149) mientras el SVG versionado decía `11 of 20 ports · 53 adapters` bajo una anotación `` antepuesta a mano que el generador nunca emitió — frente a un árbol que mide 21 interfaces de puerto declaradas (20 ficheros) · 11 en el camino caliente (7 + 4) · 47 clases adaptador · 55 tools · 12 resources · 8 prompts · 38 ficheros de comando · 12 controladores · 33 endpoints · 12 kinds / 7 evaluadores. Ningún workflow referenciaba `gh-pages` (`git grep -l gh-pages 6491c3bc -- .github/workflows` = nada). **CERRADO 2026-09-20** en `a09a1fc7` + `7eac8445`: `.harness/scripts/pages/build-pages.mjs` deriva cada número impreso (`derive-page-metrics.mjs`; puertos y adaptadores por las funciones propias del guard 45), guarda los hechos de fuera del árbol en un `observed-facts.json` fechado, valida el modelo autorado, resuelve 129 `{{placeholders}}` distintos por idioma, rechaza cualquier `../` en el sitio construido, y `--check` más un self-test de 8 pruebas corren en el job requerido `Validate documentation`; el Atlas son 45 nodos · 62 aristas · 11 capítulos · 3 niveles de lectura · EN/ES (197 cadenas de UI), el tour del póster recorre los mismos `chapters[]` sobre regiones emitidas por el generador, y `pages.yml` proyecta `main` sobre `gh-pages` (acciones fijadas por SHA, permiso de escritura solo en el job de publicación). Observado en rojo antes de confiar: mergear develop (release 1.4.0) por debajo hizo fallar `--check` sobre el SVG versionado («generated files are stale»), y `--write-svg` movió las tarjetas de las puertas a 1.4.0 sin teclear un número. **La prosa se puso al día el mismo día en [#787](https://github.com/beyondnetcode/evolith_arch32/pull/787) → [#789](https://github.com/beyondnetcode/evolith_arch32/pull/789):** los conteos que las vistas tecleaban vivían también en `SECURITY.md`, `known-limitations`, la visión maestra, los READMEs de MCP / CLI / OPA / interfaces, el catálogo de tools y la línea base del scorecard (una integración de salida → dos; 47 tools → 55; 26 categorías → 21; 17 puertos → 21; 6 mutativas → 20; 8 contextos requeridos → 10), corregidos contra el árbol con un puntero al inventario generado allí donde un puntero basta; gitleaks se puso en rojo por el camino porque sus huellas llevan número de línea y las filas nuevas las movieron, y `gh-pages` no necesitó reconstruirse — su `metrics.json` ya derivaba los mismos números (una clave distinta: `commit`). | Las dos imágenes públicas del producto tenían tres meses, la mayoría de sus enlaces estaban muertos y cada número se había tecleado a mano. | Las dos páginas se construyen desde el repositorio en cada promoción a `main`, en los dos idiomas; un número que envejece pone en rojo el check requerido, no la imagen. | `Documentation` | Cross | P2 | L | `COMPLETADO` | -| [`GT-716`](./gap-reference-catalog.es.md#gt-716) | **Sobre un `evolith validate` a secas, la brecha de cobertura entre los dos motores es en su mayoría veredictos sobre facetas que nadie suministró, y nada en CI fija esa brecha en ninguna de las dos direcciones.** Medido el 2026-09-20 con la CLI construida desde este árbol (1.4.0, `b3df7e96`) sobre un satélite recién salido de `init`, 159 reglas en alcance: el nativo decide 56 y salta 103; `--engine opa` decide 133 y salta 26. Cruzado regla por regla, 76 reglas ejecutables las decide solo OPA, y el cuerpo de la política de **73** de ellas lee una faceta que una ejecución a secas nunca envía (`input.satellite.{git,runtime,testing,multiTenancy,ci,findings,protocol,scorecards,layers,contracts}`, `input.adapter`, `input.context.dod`, `input.user`, las métricas `QT-*`) — un hecho ausente leído como veredicto, la familia a la que pertenecen seis de los ocho conflictos de veredicto de la ejecución y que la línea base de GT-704 ya registra como `supplied-facet-absent`. Solo `MCP-05`, `OBS-EVD-01` y `OBS-EVD-02` se deciden desde el árbol: ese es el backlog real de handlers. En sentido contrario, el nativo decide 11 reglas para las que el bundle no declara política (`SSDF-*` ×7, `SEC-RL-01/02`, `QT-05`, `SLSA-HOSTED-L2`), 7 no las decide ninguno (`SEC-INJ/PATH/TIMING-*`, `SEC-RL-03`), y 12 que el nativo clasifica como no ejecutables (`KI-R01..07`, `INH-03..05`, `PROT-03/06`) OPA las decide igualmente — dos clasificaciones de la misma regla desde dos fuentes sin relación (`RULE_TRIAGE` y `declared_rule_ids`), que coinciden por construcción solo para `PEA-01..04`. Sobre el corpus completo el signo se invierte — nativo 247 / OPA 187 de 358, porque 138 reglas ADR-conformance y `MM-R*` tienen handler nativo y ningún `.rego` — y `68-validate-engine-verdict-parity.mjs` imprime ambas cifras como `coverageOnly` y no bloquea por ninguna. **AC1 cerrado el 2026-09-20 en `b2840947`:** el bundle enuncia ahora qué lee cada regla (`rule_input_paths`) y OPA salta una regla cuyo hecho la ejecución no suministró; medido sobre el mismo satélite, `--engine opa` pasó de 133 decididas a **10** — las 73 contadas aquí eran solo las reglas que el nativo saltaba; 47 de las decididas por ambos también eran veredictos sobre input ausente — y ocho conflictos de la línea base (los siete `supplied-facet-absent` más `TAX-01`, mal diagnosticado) quedaron obsoletos y se retiraron. **AC2 cerrado el 2026-09-20 en `3d76f4a6`:** cada regla declara `facts` y ambos motores derivan de ello (`classifyRule` desde la procedencia, el build del bundle rechazando lecturas no declaradas); la tabla de triaje ya no existe, 44 reglas cambiaron de clase sin implementar nada — el backlog de handlers es 21, no 52, y 31 reglas necesitan una postura declarada (`needs-supplied-facts`) — y las doce contradicciones están declaradas y son ejecutables. **AC3 cerrado el 2026-09-20 en `29c8a4ba`:** `73-validate-engine-coverage-parity.mjs` registra cada regla que solo un motor decide — por escenario (este repositorio, un satélite `init`), por dirección, con la razón del otro motor — en `engine-coverage-parity.baseline.json`; una regla sin registrar, una entrada obsoleta o una clase cambiada hacen fallar. **AC4 cerrado el 2026-09-21 en `ea736a75`:** el lado solo-OPA está vacío en ambos escenarios — `OBS-EVD-01..03` y `MCP-05` tienen gemelos nativos de sus políticas, `MCP-01..03` fallan por evidencia ausente en los dos motores — y cada entrada de deuda que queda lleva una decisión registrada (`engine-coverage-decisions.json`) que el guard 73 contrasta con las ejecuciones, haciendo fallar una entrada sin ella como solo-un-motor por omisión; las siete reglas de seguridad sin decidir y `QT-05` quedan declaradas (hallazgos de un escáner, una ejecución de tests) en vez de respondidas. | `--engine opa` parece comprobar más del doble que el motor por defecto, y sobre un repositorio nuevo casi todo ese extra son veredictos sobre hechos que nadie le dio; nada avisaría si la brecha creciera. | Un hecho ausente significa «no evaluado» en los dos motores; una sola declaración por regla alimenta la tabla de handlers, el manifiesto del bundle y el informe; y lo que un motor decide y el otro no es una línea base por regla que CI hace fallar, en ambas direcciones y en ambos escenarios. | `Core Domain` | Cross | P1 | L | `PENDIENTE` | +| [`GT-716`](./gap-reference-catalog.es.md#gt-716) | **Sobre un `evolith validate` a secas, la brecha de cobertura entre los dos motores es en su mayoría veredictos sobre facetas que nadie suministró, y nada en CI fija esa brecha en ninguna de las dos direcciones.** Medido el 2026-09-20 con la CLI construida desde este árbol (1.4.0, `b3df7e96`) sobre un satélite recién salido de `init`, 159 reglas en alcance: el nativo decide 56 y salta 103; `--engine opa` decide 133 y salta 26. Cruzado regla por regla, 76 reglas ejecutables las decide solo OPA, y el cuerpo de la política de **73** de ellas lee una faceta que una ejecución a secas nunca envía (`input.satellite.{git,runtime,testing,multiTenancy,ci,findings,protocol,scorecards,layers,contracts}`, `input.adapter`, `input.context.dod`, `input.user`, las métricas `QT-*`) — un hecho ausente leído como veredicto, la familia a la que pertenecen seis de los ocho conflictos de veredicto de la ejecución y que la línea base de GT-704 ya registra como `supplied-facet-absent`. Solo `MCP-05`, `OBS-EVD-01` y `OBS-EVD-02` se deciden desde el árbol: ese es el backlog real de handlers. En sentido contrario, el nativo decide 11 reglas para las que el bundle no declara política (`SSDF-*` ×7, `SEC-RL-01/02`, `QT-05`, `SLSA-HOSTED-L2`), 7 no las decide ninguno (`SEC-INJ/PATH/TIMING-*`, `SEC-RL-03`), y 12 que el nativo clasifica como no ejecutables (`KI-R01..07`, `INH-03..05`, `PROT-03/06`) OPA las decide igualmente — dos clasificaciones de la misma regla desde dos fuentes sin relación (`RULE_TRIAGE` y `declared_rule_ids`), que coinciden por construcción solo para `PEA-01..04`. Sobre el corpus completo el signo se invierte — nativo 247 / OPA 187 de 358, porque 138 reglas ADR-conformance y `MM-R*` tienen handler nativo y ningún `.rego` — y `68-validate-engine-verdict-parity.mjs` imprime ambas cifras como `coverageOnly` y no bloquea por ninguna. **AC1 cerrado el 2026-09-20 en `b2840947`:** el bundle enuncia ahora qué lee cada regla (`rule_input_paths`) y OPA salta una regla cuyo hecho la ejecución no suministró; medido sobre el mismo satélite, `--engine opa` pasó de 133 decididas a **10** — las 73 contadas aquí eran solo las reglas que el nativo saltaba; 47 de las decididas por ambos también eran veredictos sobre input ausente — y ocho conflictos de la línea base (los siete `supplied-facet-absent` más `TAX-01`, mal diagnosticado) quedaron obsoletos y se retiraron. **AC2 cerrado el 2026-09-20 en `3d76f4a6`:** cada regla declara `facts` y ambos motores derivan de ello (`classifyRule` desde la procedencia, el build del bundle rechazando lecturas no declaradas); la tabla de triaje ya no existe, 44 reglas cambiaron de clase sin implementar nada — el backlog de handlers es 21, no 52, y 31 reglas necesitan una postura declarada (`needs-supplied-facts`) — y las doce contradicciones están declaradas y son ejecutables. **AC3 cerrado el 2026-09-20 en `29c8a4ba`:** `73-validate-engine-coverage-parity.mjs` registra cada regla que solo un motor decide — por escenario (este repositorio, un satélite `init`), por dirección, con la razón del otro motor — en `engine-coverage-parity.baseline.json`; una regla sin registrar, una entrada obsoleta o una clase cambiada hacen fallar. **AC4 cerrado el 2026-09-21 en `ea736a75`:** el lado solo-OPA está vacío en ambos escenarios — `OBS-EVD-01..03` y `MCP-05` tienen gemelos nativos de sus políticas, `MCP-01..03` fallan por evidencia ausente en los dos motores — y cada entrada de deuda que queda lleva una decisión registrada (`engine-coverage-decisions.json`) que el guard 73 contrasta con las ejecuciones, haciendo fallar una entrada sin ella como solo-un-motor por omisión; las siete reglas de seguridad sin decidir y `QT-05` quedan declaradas (hallazgos de un escáner, una ejecución de tests) en vez de respondidas. **AC5 cerrado el 2026-09-21 en `e5b4701c`:** `GOV-ENGINE-COVERAGE` enuncia la cobertura de cada corrida desglosada por clase en cualquiera de los dos motores; la tabla de estado real la renderiza el guard 73 desde el bloque de cobertura que mide y se comprueba en cada corrida (una cifra tecleada lo pone en rojo); la portada dice qué razón queda para `--engine opa` — la CLI publicada (1.3.2) es anterior a GT-716; en el árbol es para hechos suministrados; el guard 68 mide la misma exportación (queda un conflicto registrado, CLI-EXIT-02). **Cerrado.** | `--engine opa` parece comprobar más del doble que el motor por defecto, y sobre un repositorio nuevo casi todo ese extra son veredictos sobre hechos que nadie le dio; nada avisaría si la brecha creciera. | Un hecho ausente significa «no evaluado» en los dos motores; una sola declaración por regla alimenta la tabla de handlers, el manifiesto del bundle y el informe; y lo que un motor decide y el otro no es una línea base por regla que CI hace fallar, en ambas direcciones y en ambos escenarios. | `Core Domain` | Cross | P1 | L | `COMPLETADO` | | [`GT-715`](./gap-reference-catalog.es.md#gt-715) | **La Core API rechazaba cualquier contexto de evaluación inline de más de 100 KB con un 500 enmascarado y sin línea de log, así que la llamada de conformidad del repositorio del Tracker nunca funcionó contra un repositorio real.** Medido el 2026-09-20 desde el entorno UAT del Tracker: `POST /products/{id}/evaluate-architecture` sobre un producto que apunta a este repositorio leyó 150 ficheros (~1 MB) de GitHub y se los envió al Core, que respondió `500 INTERNAL_ERROR "An unexpected error occurred"`; el Tracker registró un `synthetic BLOCKED`. Reproducido sobre la misma imagen (`main@142b8324`): 14 ficheros / cuerpo de 99.797 bytes → `200`, 15 ficheros / 101.578 bytes → `500`; un cuerpo sintético de 92.956 bytes → `200`, de 126.556 → `500`. El límite de 100 KB por omisión de Express para json, lanzado como un `PayloadTooLargeError` que no es `HttpException`, clasificado por mensaje, enmascarado para el cable y escrito en ninguna parte. **CERRADO 2026-09-20** en `3b276c9a`: `EVOLITH_MAX_BODY_BYTES` (2 MiB por defecto) registra los parsers explícitamente, los errores del body parser conservan el estado del parser y el 413 nombra los dos tamaños y la variable, y todo 5xx enmascarado se registra con su traza. Rojo primero: 4 de las 5 specs nuevas fallan sobre el filtro viejo; después: el mismo cuerpo de 1,1 MB → `200` con el veredicto de `gate-f1`, 3 MB → `413`. Promovido en #778 (`9c5deedf`) y redesplegado por el job `Deploy UAT (Coolify)` de la corrida 35490911533 el 2026-09-20; medido justo después: la misma llamada `evaluate-architecture` responde `200`, `provenance: core`, `status: COMPLETED`, `resultDecision: FAILED` — un veredicto real sobre 150 ficheros (gates f1–f5 fallidos por artefactos de fase ausentes), 174 ms en el Core. La portada conserva la captura de la compuerta de fase. | El Tracker no podía obtener un veredicto de arquitectura sobre ningún repositorio real: el Core rechazaba la petición por su tamaño y no le decía nada útil a nadie. | El Core evalúa un repositorio real enviado inline, rechaza con una razón que nombra cuando debe, y deja una traza que el operador puede leer. | `Core API` | Cross | P1 | S | `COMPLETADO` | | [`GT-714`](./gap-reference-catalog.es.md#gt-714) | **`gate evaluate` y `phase advance` en la CLI publicada necesitan un checkout de este repositorio en disco, porque el tarball trae las reglas pero no las definiciones de gate.** Medido el 2026-09-20 con `@beyondnet/evolith-cli@1.3.2` en un contenedor `node:20` limpio sobre un satélite recién salido de `init`: sin `--core` las dos órdenes salen con `1` y `ENOENT … reference/governance/sdlc/gates` (el tarball trae `rulesets/sdlc/phase-gates.rules.json` y el registro de artefactos en las rutas propias del paquete, pero el validador compone `/reference/governance/sdlc/gates` y `/src/rulesets/sdlc/artifact-registry.json`, y `findCorePath` cae en el propio satélite); con `--core ../evolith` las dos salen con `2` y el veredicto real del gate. GT-705 arregló el mismo defecto para el paquete MCP empaquetando los dos árboles e instalando un único resolutor; el paquete de la CLI no entró en ese cambio, y `sdlc gate-status` (sub-hallazgo de GT-461) no tiene `--core` en absoluto. Hasta que aterrice, la portada muestra las dos órdenes con `--core ../evolith` y lo explica en el pie. | Las dos órdenes que hacen de Evolith algo más que un linter no corren desde el paquete publicado sin un clon de este repositorio junto al proyecto. | `npx -y @beyondnet/evolith-cli gate evaluate --phase discovery` sobre un satélite nuevo da el veredicto, sin opción, y la portada pierde su salvedad. | `Evolith CLI` | Cross | P1 | S | `PENDIENTE` | | [`GT-713`](./gap-reference-catalog.es.md#gt-713) | **El análisis al que están ligadas las alertas de la pestaña Security solo lo produce la corrida `push` de `sdk-cli-ci.yml`, y su filtro de rutas saltaba casi todo el código.** El job `CodeQL SAST` es el único que sube el análisis `/language:javascript-typescript` para `refs/heads/main`; las corridas de pull request son diff-informed (se recortan al diff y nunca mueven las alertas de la rama) y la configuración por defecto "Code Quality" es otra suite. El trigger `push` estaba filtrado a `src/sdk/cli/**`, `.harness/**` y los lockfiles. Medido el 2026-09-19: la promoción `19d736da` (cambios solo bajo `src/packages` y `src/apps`) llegó a `main` sin análisis alguno, así que la pestaña siguió mostrando 10 alertas sobre código que ya no existía; `c5547114` hizo lo mismo 40 minutos después. Las dos necesitaron `gh workflow run sdk-cli-ci.yml --ref main` a mano. **CERRADO 2026-09-19** en `72aceb70`: el filtro cubre ahora `src/packages/**` y `src/apps/**` — todo lo que CodeQL escanea — y un push solo de documentación sigue sin gastar la corrida. | El escáner que decide qué muestra la pestaña Security no volvía a correr cuando cambiaba la mayor parte del código, así que la pestaña describía el commit anterior. | Una promoción de código a `main` re-analiza `main`; la pestaña está al día sin que nadie tenga que acordarse de lanzarla. | `Infra` | Cross | P2 | XS | `COMPLETADO` | @@ -749,7 +750,7 @@ Este tablero es la única fuente de verdad para deuda técnica, gaps, oportunida | [`GT-706`](./gap-reference-catalog.es.md#gt-706) | **Nada asegura que los `exports` que un paquete declara resuelvan dentro de su propio tarball, así que un productor publica una subruta fantasma y solo la descubre un consumidor — una publicación demasiado tarde.** `contracts@1.1.0` declaró una subruta de export que no incluía; el fallo salió en el smoke de sala limpia de `infra-providers@1.2.1`, **después de que `core-domain@1.3.1` ya estuviera irreversiblemente en el registry**, dejando la release a medio entregar y sin despublicar posible pasadas 72 horas. La comprobación que existe es real y tiene la forma equivocada: `npm-release.yml:213` calcula «prometidos» como `[pkg.main, ...bin]`, y **`exports` no está en esa lista**. FALSABILIDAD DEMOSTRADA, OBSERVADA EN VERDE: un paquete de dos ficheros que declara `"./ingest"` con solo `dist/index.js` en disco pasa esa aserción corrida literal — `exit=0`, mientras `require pkg/ingest` responde `MODULE_NOT_FOUND`. El smoke de sala limpia tampoco lo cubre, y no es defecto suyo: resuelve lo que un paquete IMPORTA, así que el fantasma del productor es invisible hasta el turno de un consumidor, que es después del paso irreversible. Exposición: 3 de 8 paquetes publicables declaran **23 subrutas de export**, ninguna asegurada, y dos declaran además un `./*` sin cota. **ARREGLADO 2026-08-16 — `.harness/scripts/ci/67-validate-declared-exports.mjs`, corriendo en tiempo de PR sobre todos los workspaces publicables Y por paquete dentro del bucle de release, justo antes de `npm publish`.** Recoge cada hoja de texto del árbol de condiciones, así que `types` cuenta tanto como `default`, e incluye `main`/`bin`, siendo un superconjunto de la aserción que sustituye. **La propia afirmación de esta fila sobre el registry la refutó el guard en su primera corrida:** «22 de 22 resuelven, 0 fantasmas» excluía las claves con comodín por su propio filtro, y una está MUERTA — `core-domain` declara `./infrastructure/adapters/*` **sin ningún directorio `adapters`**, 0 coincidencias en un packlist de 796 ficheros, `MODULE_NOT_FOUND` en el 1.3.1 publicado, y **ningún commit de este repositorio llevó jamás ese path**. Borrada, no ampliada: nunca hubo nada detrás. Falsabilidad observada por los dos lados — rojo con la fixture `./ingest`, con `core-domain` de verdad, y con un fichero presente en disco pero excluido por `files`; verde con la misma fixture en cuanto se incluye y con el árbol entero, **68 destinos declarados en 9 paquetes**. | Un paquete puede prometer una ruta de import que nunca incluyó, y quien se entera es el siguiente paquete en publicarse. | La release se niega a publicar un manifiesto que miente, antes de que nada sea irreversible. | `Infra` | Cross | P1 | S | `COMPLETADO` | -**Progreso:** 688 / 715 completados · 3 en progreso · 3 pendientes · 21 diferidos +**Progreso:** 689 / 715 completados · 3 en progreso · 2 pendientes · 21 diferidos **Oleada 2026-06-23 (auditoría profunda de Winston III):** Añadidos 14 gaps nuevos `GT-212`…`GT-225` del Winston Audit Playbook que cubren: higiene de estado ADR (GT-212), metadata + presupuestos operativos + corpus de guías por topología (GT-213, GT-217, GT-219), observabilidad + OpenAPI en controladores REST (GT-214, GT-215), paridad de input-schemas OPA + densidad de tests por topología (GT-216, GT-222), plantillas de rollback + on-call de Fase 05 (GT-218), cobertura de ramas CLI + paridad de envelope --format + limpieza de skip-list (GT-220, GT-224, GT-225), audit logging HTTP de MCP (GT-221), y tests e2e de paridad cross-surface (GT-223). diff --git a/reference/core/control-center/gaps/gap-tracking.md b/reference/core/control-center/gaps/gap-tracking.md index 84e36cba6..e848d36cb 100644 --- a/reference/core/control-center/gaps/gap-tracking.md +++ b/reference/core/control-center/gaps/gap-tracking.md @@ -4,6 +4,7 @@ **Status:** Active Tracking **Owner:** Evolith Architecture Board +**Last Updated:** 2026-09-21 (**GT-716 closed: the report, the page and the front page say the same thing about what each engine covers.** `e5b4701c` ([#799](https://github.com/beyondnetcode/evolith_arch32/pull/799)) — AC5: `GOV-ENGINE-COVERAGE` fires on either engine that skips more than it checks and states that run's coverage with its skips split by class (fact not supplied, adapter needed, documentation, engine debt) from a new `skippedByEvaluability` in the report; `docs/known-limitations.{md,es.md}` carry the same split per engine and scenario in a table guard 73 renders from a coverage block it records in its baseline and checks on every run — one edited number was observed turning it red, `--write` restoring it. Measured 2026-09-21 on the export: this repository native 109 / 246 vs `--engine opa` 28 / 327; fresh `init` satellite native 50 / 101 vs 2 / 149 — the default decides more on a bare run, on both rows. The front page says which reason remains: `--engine opa` has the most coverage on the published CLI (1.3.2), which predates GT-716; in the tree it is for the facts the caller supplies. Guard 68 moved onto the same export (`--core`): CLI-EXIT-01/03 and GOV-RULE-NON-EXECUTABLE stale, CLI-EXIT-02 registered with its follow-up. Five criteria, five pull requests (#788, #793, #795, #797, #799). Counters recomputed from the rows: **689 / 715 done · 3 in progress · 2 pending · 21 deferred**.) **Last Updated:** 2026-09-21 (**GT-716 AC4 landed: nothing in the coverage register is there by omission.** `ea736a75` — the named backlog is implemented or declared, and `73-validate-engine-coverage-parity.mjs` now refuses a debt-class entry with no recorded decision. Implemented: `OBS-EVD-01..03` decided natively from the satellite's dependencies (the same proxy their policy reads), `MCP-05` from the server source, `MCP-01..03` failing on absent smoke evidence on both engines instead of one — the OPA-only side of the baseline is **empty on both scenarios** (was 7 / 7). Declared: the seven rules neither engine decided (`SEC-INJ/PATH/TIMING-01/02`, `SEC-RL-03`) name `satellite.findings`, a scanner's finding over the AST rather than a regex over the tree (`MM-R10`); `QT-05`, which the native engine answered `passed` with "requires runtime analysis", names `satellite.testing` and is decided by neither. Recorded: `engine-coverage-decisions.json`, eight decisions (`SSDF`, `SLSA`, `SEC-RL-01/02`, `MM-R*`, the enforcer-routed `HXA` clauses as `native-only`; the 138 generated ADR-conformance rules, the seven scanner rules and `QT-05` as `neither`), each held to the runs — the guard's first run went red on `MM-R03`, which both engines decide and the register had claimed, and green once it left. The 11 no-policy rules got decisions with reasons, not `.rego` twins. Handler backlog 21 → 14; blocking rules that do not run 73 → 71; guard 68 gained one registered conflict (`GOV-RULE-NON-EXECUTABLE`, a corpus-resolution artifact of its `--core`-less run, AC5's to remove). AC5 open; counters unchanged: **688 / 715 done · 3 in progress · 3 pending · 21 deferred**.) **Last Updated:** 2026-09-20 (**GT-716 AC3 landed: what only one engine decides is a registered difference, not a free one.** `29c8a4ba` — `73-validate-engine-coverage-parity.mjs` runs both engines on this repository and on a satellite it creates with `evolith init`, and holds every coverage-only rule to a baseline per scenario, per direction, per rule, with the reason the other engine gave (the class its report states; the facets the policy reads that a bare run does not supply). An unregistered rule, a stale entry or a changed class fails. Both scenarios read the Core from an export of the tracked tree, because the first CI run disagreed with the laptop that wrote the baseline (a local `coverage/` directory, git history and the CLI's bundled corpus copy had decided six rules and failed 138 ADR-conformance rules falsely). Measured on that export: repository 82 native-only / 7 opa-only, init satellite 49 / 7 — 52 of the 82 are policies reading facets nobody supplied, 26 have no policy at all, and four are the enforcer-routed `HXA-01/02/04/05`, where the OPA path skips without a reason of its own. The criterion's probe (drop an import from `main.rego`) cannot run since GT-675 — the build refuses it — so the equivalent one was recorded: rename `OBS-EVD-03` in its policy and the guard goes red on both scenarios naming the id. AC4–AC5 open; counters unchanged: **688 / 715 done · 3 in progress · 3 pending · 21 deferred**.) **Last Updated:** 2026-09-20 (**GT-716 AC2 landed: one evaluability declaration per rule, and both engines derive from it.** `3d76f4a6` — every rule in `*.rules.json` declares `facts`, the facets its check reads, from a vocabulary that says where the truth of each one lives (`observed` / `supplied` / `external` / `runtime`); `classifyRule` derives the native class from the most demanding provenance and the triage table keyed by rule id is gone; `npm run build:policy` refuses a bundle whose policy reads a facet its rule did not declare. Declaring `facts: []` for the twelve rules the table called non-executable while Rego decided them turned that check red with twelve findings; declaring what the policies read turned it green. **What moved, with nothing implemented: 44 of 415 rules changed class.** The "handler backlog" is 21, not 52 — 25 of those rows were decided by policies over a declared posture, the CI system or a test run, never by a handler over the tree — and a new native class, `needs-supplied-facts`, holds the 31 rules that need a posture only the satellite's owners can declare. The 73 blocking rules that do not run are the same 73, re-costed. Snapshot, ISO 5055 mapping and the standards README re-derived; guard 68's baseline down to `CLI-EXIT-01/03`. AC3–AC5 open; counters unchanged: **688 / 715 done · 3 in progress · 3 pending · 21 deferred**.) @@ -33,7 +34,7 @@ This board is the single source of truth for technical debt, gaps, opportunities | ID | Gap | In plain terms | What it fixes | Component | Phase | Criticality | Complexity | Status | |---|---|---|---|:---:|:---:|:---:|:---:|:---:| | [`GT-717`](./gap-reference-catalog.md#gt-717) | **The two GitHub Pages views were published by hand and had aged three months; nothing derived their numbers from the tree.** Measured 2026-09-19/20 against `origin/gh-pages` (`2c8f487a`, last touched 2026-07-07) and develop `6491c3bc`: the deployed Atlas predated the 2026-07-04 taxonomy move — 18 of its 21 source links answered 404 from github.io because `docHref` built `../` paths against the Pages root — was monolingual and had 4 scenarios with no explanations; the tree's own `architecture-map.json` named 20 docs paths of which 10 no longer existed. The master view printed `8 controllers`, `26 tools · 9 resources`, `20 commands`, `5 KindEvaluators`, `PWA · Web · Mobile` + `BFF (NestJS · ADR-0075)` for a Tracker that is .NET 10 + React 19, and the header `ADR-0101 · 0074 · 0075 · 0102`; its generator printed `12 hexagonal ports · 30 adapters` (line 149) while the tracked SVG said `11 of 20 ports · 53 adapters` under a hand-prepended `` annotation the generator never emitted — against a tree measuring 21 declared port interfaces (20 files) · 11 on the hot path (7 + 4) · 47 adapter classes · 55 tools · 12 resources · 8 prompts · 38 command files · 12 controllers · 33 endpoints · 12 kinds / 7 evaluators. No workflow referenced `gh-pages` (`git grep -l gh-pages 6491c3bc -- .github/workflows` = nothing). **CLOSED 2026-09-20** in `a09a1fc7` + `7eac8445`: `.harness/scripts/pages/build-pages.mjs` derives every printed number (`derive-page-metrics.mjs`; ports and adapters through guard 45's own functions), keeps out-of-tree facts in a dated `observed-facts.json`, validates the authored model, resolves 129 distinct `{{placeholders}}` per language, refuses any `../` in the built site, and `--check` plus an 8-test self-test run in the required `Validate documentation` job; the Atlas is 45 nodes · 62 edges · 11 chapters · 3 reading levels · EN/ES (197 UI strings), the poster's tour rides the same `chapters[]` on generator-emitted regions, and `pages.yml` projects `main` onto `gh-pages` (SHA-pinned, write permission only on the publish job). Observed red before trusted: merging develop (release 1.4.0) underneath made `--check` fail on the tracked SVG ("generated files are stale"), and `--write-svg` moved the door cards to 1.4.0 with no number typed. **The prose caught up the same day in [#787](https://github.com/beyondnetcode/evolith_arch32/pull/787) → [#789](https://github.com/beyondnetcode/evolith_arch32/pull/789):** the counts the views had typed also lived in `SECURITY.md`, `known-limitations`, the vision master, the MCP / CLI / OPA / interfaces READMEs, the tools catalog and the scorecard baseline (one outbound integration → two; 47 tools → 55; 26 categories → 21; 17 ports → 21; 6 mutative → 20; 8 required contexts → 10), corrected against the tree with a pointer to the generated inventory wherever a pointer suffices; gitleaks went red on the way because its fingerprints carry line numbers and the new rows moved them, and `gh-pages` needed no rebuild — its `metrics.json` already derived the same numbers (one differing key: `commit`). | The two public pictures of the product were three months old, most of their links were dead, and every number on them had been typed by hand. | Both pages are built from the repository on every promotion to `main`, in both languages; a number that ages turns the required check red instead of the picture. | `Documentation` | Cross | P2 | L | `DONE` | -| [`GT-716`](./gap-reference-catalog.md#gt-716) | **On a bare `evolith validate` the coverage gap between the two engines is mostly verdicts on facets nobody supplied, and nothing in CI pins the gap in either direction.** Measured 2026-09-20 with the CLI built from this tree (1.4.0, `b3df7e96`) on a satellite fresh from `init`, 159 rules in scope: native decides 56 and skips 103; `--engine opa` decides 133 and skips 26. Crossed rule by rule, 76 executable rules are decided by OPA alone, and the policy body of **73** of them reads a facet a bare run never sends (`input.satellite.{git,runtime,testing,multiTenancy,ci,findings,protocol,scorecards,layers,contracts}`, `input.adapter`, `input.context.dod`, `input.user`, the `QT-*` metrics) — an absent fact read as a verdict, the family six of the run's eight verdict conflicts belong to and GT-704's baseline already carries as `supplied-facet-absent`. Only `MCP-05`, `OBS-EVD-01` and `OBS-EVD-02` are decided from the tree: that is the real handler backlog. The other way, native decides 11 rules the bundle declares no policy for (`SSDF-*` ×7, `SEC-RL-01/02`, `QT-05`, `SLSA-HOSTED-L2`), 7 are decided by neither (`SEC-INJ/PATH/TIMING-*`, `SEC-RL-03`), and 12 that native classes non-executable (`KI-R01..07`, `INH-03..05`, `PROT-03/06`) OPA decides anyway — two classifications of the same rule from two unrelated sources (`RULE_TRIAGE` and `declared_rule_ids`), agreeing by construction only for `PEA-01..04`. Over the whole corpus the sign flips — native 247 / OPA 187 of 358, because 138 ADR-conformance rules and `MM-R*` have a native handler and no `.rego` — and `68-validate-engine-verdict-parity.mjs` prints both figures as `coverageOnly` and gates neither. **AC1 closed 2026-09-20 in `b2840947`:** the bundle now states what each rule reads (`rule_input_paths`) and OPA skips a rule whose fact the run did not supply; measured on the same satellite, `--engine opa` went from 133 decided to **10** — the 73 counted here were only the rules native skipped; 47 of the jointly-decided ones were absent-input verdicts too — and eight baseline conflicts (the seven `supplied-facet-absent` plus `TAX-01`, misdiagnosed) went stale and were removed. **AC2 closed 2026-09-20 in `3d76f4a6`:** every rule declares `facts` and both engines derive from it (`classifyRule` from the provenance, the bundle build refusing undeclared reads); the triage table is gone, 44 rules changed class with nothing implemented — the handler backlog is 21, not 52, and 31 rules need a declared posture (`needs-supplied-facts`) — and the twelve contradictions are declared and executable. **AC3 closed 2026-09-20 in `29c8a4ba`:** `73-validate-engine-coverage-parity.mjs` registers every rule only one engine decides — per scenario (this repository, an `init` satellite), per direction, with the other engine's reason — in `engine-coverage-parity.baseline.json`; an unregistered rule, a stale entry or a changed class fails. **AC4 closed 2026-09-21 in `ea736a75`:** the OPA-only side is empty on both scenarios — `OBS-EVD-01..03` and `MCP-05` have native twins of their policies, `MCP-01..03` fail on absent evidence on both engines — and every remaining debt entry carries a recorded decision (`engine-coverage-decisions.json`) guard 73 holds to the runs, failing an entry without one as coverage-only by omission; the seven undecided security rules and `QT-05` are declared (a scanner's findings, a test run) rather than answered. | `--engine opa` looks like it checks more than twice what the default does, and on a fresh repository almost all of the extra is verdicts on facts nobody gave it; nothing would notice the gap growing. | An absent fact means "not evaluated" on both engines; one declaration per rule feeds the handler table, the bundle manifest and the report; and what one engine decides and the other does not is a per-rule baseline CI fails on, in both directions, on both scenarios. | `Core Domain` | Cross | P1 | L | `PENDING` | +| [`GT-716`](./gap-reference-catalog.md#gt-716) | **On a bare `evolith validate` the coverage gap between the two engines is mostly verdicts on facets nobody supplied, and nothing in CI pins the gap in either direction.** Measured 2026-09-20 with the CLI built from this tree (1.4.0, `b3df7e96`) on a satellite fresh from `init`, 159 rules in scope: native decides 56 and skips 103; `--engine opa` decides 133 and skips 26. Crossed rule by rule, 76 executable rules are decided by OPA alone, and the policy body of **73** of them reads a facet a bare run never sends (`input.satellite.{git,runtime,testing,multiTenancy,ci,findings,protocol,scorecards,layers,contracts}`, `input.adapter`, `input.context.dod`, `input.user`, the `QT-*` metrics) — an absent fact read as a verdict, the family six of the run's eight verdict conflicts belong to and GT-704's baseline already carries as `supplied-facet-absent`. Only `MCP-05`, `OBS-EVD-01` and `OBS-EVD-02` are decided from the tree: that is the real handler backlog. The other way, native decides 11 rules the bundle declares no policy for (`SSDF-*` ×7, `SEC-RL-01/02`, `QT-05`, `SLSA-HOSTED-L2`), 7 are decided by neither (`SEC-INJ/PATH/TIMING-*`, `SEC-RL-03`), and 12 that native classes non-executable (`KI-R01..07`, `INH-03..05`, `PROT-03/06`) OPA decides anyway — two classifications of the same rule from two unrelated sources (`RULE_TRIAGE` and `declared_rule_ids`), agreeing by construction only for `PEA-01..04`. Over the whole corpus the sign flips — native 247 / OPA 187 of 358, because 138 ADR-conformance rules and `MM-R*` have a native handler and no `.rego` — and `68-validate-engine-verdict-parity.mjs` prints both figures as `coverageOnly` and gates neither. **AC1 closed 2026-09-20 in `b2840947`:** the bundle now states what each rule reads (`rule_input_paths`) and OPA skips a rule whose fact the run did not supply; measured on the same satellite, `--engine opa` went from 133 decided to **10** — the 73 counted here were only the rules native skipped; 47 of the jointly-decided ones were absent-input verdicts too — and eight baseline conflicts (the seven `supplied-facet-absent` plus `TAX-01`, misdiagnosed) went stale and were removed. **AC2 closed 2026-09-20 in `3d76f4a6`:** every rule declares `facts` and both engines derive from it (`classifyRule` from the provenance, the bundle build refusing undeclared reads); the triage table is gone, 44 rules changed class with nothing implemented — the handler backlog is 21, not 52, and 31 rules need a declared posture (`needs-supplied-facts`) — and the twelve contradictions are declared and executable. **AC3 closed 2026-09-20 in `29c8a4ba`:** `73-validate-engine-coverage-parity.mjs` registers every rule only one engine decides — per scenario (this repository, an `init` satellite), per direction, with the other engine's reason — in `engine-coverage-parity.baseline.json`; an unregistered rule, a stale entry or a changed class fails. **AC4 closed 2026-09-21 in `ea736a75`:** the OPA-only side is empty on both scenarios — `OBS-EVD-01..03` and `MCP-05` have native twins of their policies, `MCP-01..03` fail on absent evidence on both engines — and every remaining debt entry carries a recorded decision (`engine-coverage-decisions.json`) guard 73 holds to the runs, failing an entry without one as coverage-only by omission; the seven undecided security rules and `QT-05` are declared (a scanner's findings, a test run) rather than answered. **AC5 closed 2026-09-21 in `e5b4701c`:** `GOV-ENGINE-COVERAGE` states each run's coverage split by class on either engine; the known-limitations table is rendered by guard 73 from the coverage block it measures and is checked on every run (a typed number turns it red); the front page says which reason remains for `--engine opa` — the published CLI (1.3.2) predates GT-716; in the tree it is for supplied facts; guard 68 measures the same export (one registered conflict left, CLI-EXIT-02). **Closed.** | `--engine opa` looks like it checks more than twice what the default does, and on a fresh repository almost all of the extra is verdicts on facts nobody gave it; nothing would notice the gap growing. | An absent fact means "not evaluated" on both engines; one declaration per rule feeds the handler table, the bundle manifest and the report; and what one engine decides and the other does not is a per-rule baseline CI fails on, in both directions, on both scenarios. | `Core Domain` | Cross | P1 | L | `DONE` | | [`GT-715`](./gap-reference-catalog.md#gt-715) | **The Core API rejected any inline evaluation context over 100 KB with a masked 500 and no log line, so the Tracker's repository-conformance call never worked against a real repository.** Measured 2026-09-20 from the Tracker's UAT environment: `POST /products/{id}/evaluate-architecture` on a product pointing at this repository read 150 files (~1 MB) from GitHub and posted them to the Core, which answered `500 INTERNAL_ERROR "An unexpected error occurred"`; the Tracker recorded a `synthetic BLOCKED`. Reproduced on the same image (`main@142b8324`): 14 files / 99,797-byte body → `200`, 15 files / 101,578 bytes → `500`; a synthetic 92,956-byte body → `200`, 126,556 → `500`. Express's 100 KB json default, thrown as a `PayloadTooLargeError` that is not an `HttpException`, classified by message, masked for the wire and written nowhere. **CLOSED 2026-09-20** in `3b276c9a`: `EVOLITH_MAX_BODY_BYTES` (2 MiB by default) registers the parsers explicitly, body-parser errors keep the parser's status and the 413 names both sizes and the variable, and every masked 5xx is logged with its stack. Red first: 4 of 5 new specs fail on the old filter; after: the same 1.1 MB payload → `200` with the `gate-f1` verdict, 3 MB → `413`. Promoted in #778 (`9c5deedf`) and redeployed by the `Deploy UAT (Coolify)` job of run 35490911533 on 2026-09-20; measured right after: the same `evaluate-architecture` call answers `200`, `provenance: core`, `status: COMPLETED`, `resultDecision: FAILED` — a real verdict on 150 files (gates f1–f5 failed for missing phase artifacts), 174 ms in the Core. The front page keeps the phase-gate capture. | The Tracker could not get an architecture verdict on any real repository: the Core refused the request for its size and said nothing useful to anyone. | The Core evaluates a real repository sent inline, refuses with a reason it names when it must, and leaves a trace the operator can read. | `Core API` | Cross | P1 | S | `DONE` | | [`GT-714`](./gap-reference-catalog.md#gt-714) | **`gate evaluate` and `phase advance` in the published CLI need a checkout of this repository on disk, because the tarball carries the rules but not the gate definitions.** Measured 2026-09-20 with `@beyondnet/evolith-cli@1.3.2` in a clean `node:20` container on a satellite fresh from `init`: without `--core` both commands exit `1` with `ENOENT … reference/governance/sdlc/gates` (the tarball ships `rulesets/sdlc/phase-gates.rules.json` and the artifact registry at the package's own paths, but the validator composes `/reference/governance/sdlc/gates` and `/src/rulesets/sdlc/artifact-registry.json`, and `findCorePath` falls back to the satellite itself); with `--core ../evolith` both exit `2` with the real gate verdict. GT-705 fixed the same defect for the MCP package by bundling both trees and installing one resolver; the CLI package was not part of that change, and `sdlc gate-status` (GT-461 sub-finding) has no `--core` at all. Until it lands, the front page shows the two commands with `--core ../evolith` and says why in the caption. | The two commands that make Evolith more than a linter do not run from the published package without a clone of this repository next to the project. | `npx -y @beyondnet/evolith-cli gate evaluate --phase discovery` on a fresh satellite gives the verdict, no flag, and the front page loses its caveat. | `Evolith CLI` | Cross | P1 | S | `PENDING` | | [`GT-713`](./gap-reference-catalog.md#gt-713) | **The analysis that the Security tab's alerts are keyed to is produced only by the `push` run of `sdk-cli-ci.yml`, and its path filter skipped most of the code.** The `CodeQL SAST` job is the sole uploader of the `/language:javascript-typescript` analysis for `refs/heads/main`; pull-request runs are diff-informed (they prune to the diff and never move the branch's alerts) and the default "Code Quality" setup is a different suite. The `push` trigger was filtered to `src/sdk/cli/**`, `.harness/**` and the lockfiles. Measured 2026-09-19: promotion `19d736da` (changes under `src/packages` and `src/apps` only) reached `main` with no analysis at all, so the tab kept reporting 10 alerts on code that no longer existed; `c5547114` did the same 40 minutes later. Both needed `gh workflow run sdk-cli-ci.yml --ref main` by hand. **CLOSED 2026-09-19** in `72aceb70`: the filter now spans `src/packages/**` and `src/apps/**` — every tree CodeQL scans — while a docs-only push still skips the run. | The scanner that decides what the Security tab shows was not re-run when most of the code changed, so the tab described the previous commit. | A code promotion to `main` re-analyses `main`; the tab is current without anyone remembering to dispatch it. | `Infra` | Cross | P2 | XS | `DONE` | @@ -749,7 +750,7 @@ This board is the single source of truth for technical debt, gaps, opportunities | [`GT-706`](./gap-reference-catalog.md#gt-706) | **Nothing asserts that a package's own declared `exports` resolve inside its own tarball, so a producer publishes a phantom subpath and only a consumer discovers it — one publish too late.** `contracts@1.1.0` declared an export subpath it did not ship; the failure surfaced at `infra-providers@1.2.1`'s clean-room smoke, **after `core-domain@1.3.1` was already irreversibly on the registry**, leaving the release half-shipped with no unpublish available after 72 hours. The check that exists is real and the wrong shape: `npm-release.yml:213` computes "promised" as `[pkg.main, ...bin]`, and **`exports` is not in that list**. PROVEN FALSIFIABLE, OBSERVED GREEN: a two-file package declaring `"./ingest"` with only `dist/index.js` on disk passes that assertion run verbatim — `exit=0`, while `require pkg/ingest` answers `MODULE_NOT_FOUND`. The clean-room smoke does not cover it either, and that is not its defect: it resolves what a package IMPORTS, so a producer's phantom is invisible until a consumer's turn, which is after the irreversible step. Exposure: 3 of 8 publishable packages declare **23 export subpaths**, none asserted, two of them also declaring an unbounded `./*`. **FIXED 2026-08-16 — `.harness/scripts/ci/67-validate-declared-exports.mjs`, run at PR time over every publishable workspace AND per package inside the release loop, immediately before `npm publish`.** It collects every string leaf of the condition tree, so `types` counts as much as `default`, and folds in `main`/`bin`, making it a superset of the assertion it replaces. **The row's own claim about the registry was refuted by the guard on its first run:** "22 of 22 resolve, 0 phantom" excluded wildcard keys by its own filter, and one is DEAD — `core-domain` declares `./infrastructure/adapters/*` with **no `adapters` directory at all**, 0 matches in a 796-file packlist, `MODULE_NOT_FOUND` on the published 1.3.1, and **no commit in this repository ever carried that path**. Deleted, not widened: there was never anything behind it. Falsifiability observed on both sides — red on the `./ingest` fixture, on `core-domain` for real, and on a file present on disk but excluded by `files`; green on the same fixture once it ships and on the whole tree, **68 declared targets across 9 packages**. | A package can promise an import path it never shipped, and the next package to publish is the one that finds out. | The release refuses to publish a manifest that lies, before anything becomes irreversible. | `Infra` | Cross | P1 | S | `DONE` | -**Progress:** 688 / 715 done · 3 in progress · 3 pending · 21 deferred +**Progress:** 689 / 715 done · 3 in progress · 2 pending · 21 deferred **Wave 2026-06-23 (Winston deep audit III):** Added 14 new gaps `GT-212`…`GT-225` from the Winston Audit Playbook covering: ADR status hygiene (GT-212), topology manifest metadata + operational budgets + guidance corpus (GT-213, GT-217, GT-219), REST controller observability + OpenAPI (GT-214, GT-215), OPA input-schema parity + per-topology test density (GT-216, GT-222), SDLC Phase 05 rollback + on-call templates (GT-218), CLI branch coverage + envelope format coverage + skip-list cleanup (GT-220, GT-224, GT-225), MCP HTTP audit logging (GT-221), and cross-surface parity e2e tests (GT-223). diff --git a/reference/core/control-center/maturity-reports/executive-summary.es.md b/reference/core/control-center/maturity-reports/executive-summary.es.md index 3072f9c22..c94a5560b 100644 --- a/reference/core/control-center/maturity-reports/executive-summary.es.md +++ b/reference/core/control-center/maturity-reports/executive-summary.es.md @@ -28,7 +28,7 @@ La forma correcta de usar este resumen es simple: si necesitas contexto, abre so | 1 | Bloqueadores P0 | Impiden afirmar readiness productivo o release mayor. | [GT-435](../gaps/gap-reference-catalog.es.md#gt-435) | | 2 | Área de mayor riesgo | `Governance` tiene la mayor carga ponderada abierta. | [GT-670](../gaps/gap-reference-catalog.es.md#gt-670), [GT-585](../gaps/gap-reference-catalog.es.md#gt-585), [GT-669](../gaps/gap-reference-catalog.es.md#gt-669), [GT-672](../gaps/gap-reference-catalog.es.md#gt-672), [GT-689](../gaps/gap-reference-catalog.es.md#gt-689), [GT-588](../gaps/gap-reference-catalog.es.md#gt-588), +1 | | 3 | Ganancias rápidas | Alta criticidad con complejidad XS/S. | [GT-684](../gaps/gap-reference-catalog.es.md#gt-684), [GT-710](../gaps/gap-reference-catalog.es.md#gt-710), [GT-714](../gaps/gap-reference-catalog.es.md#gt-714) | -| 4 | Ola P1 | Endurecimiento siguiente después de limpiar P0. | [GT-684](../gaps/gap-reference-catalog.es.md#gt-684), [GT-710](../gaps/gap-reference-catalog.es.md#gt-710), [GT-714](../gaps/gap-reference-catalog.es.md#gt-714), [GT-324](../gaps/gap-reference-catalog.es.md#gt-324), [GT-670](../gaps/gap-reference-catalog.es.md#gt-670), [GT-680](../gaps/gap-reference-catalog.es.md#gt-680), [GT-681](../gaps/gap-reference-catalog.es.md#gt-681), [GT-585](../gaps/gap-reference-catalog.es.md#gt-585), +3 | +| 4 | Ola P1 | Endurecimiento siguiente después de limpiar P0. | [GT-684](../gaps/gap-reference-catalog.es.md#gt-684), [GT-710](../gaps/gap-reference-catalog.es.md#gt-710), [GT-714](../gaps/gap-reference-catalog.es.md#gt-714), [GT-324](../gaps/gap-reference-catalog.es.md#gt-324), [GT-670](../gaps/gap-reference-catalog.es.md#gt-670), [GT-680](../gaps/gap-reference-catalog.es.md#gt-680), [GT-681](../gaps/gap-reference-catalog.es.md#gt-681), [GT-585](../gaps/gap-reference-catalog.es.md#gt-585), +2 | | 5 | P2/P3 | Solo después de estabilizar seguridad, CI, reglas y contratos. | [GT-444](../gaps/gap-reference-catalog.es.md#gt-444), [GT-464](../gaps/gap-reference-catalog.es.md#gt-464), [GT-674](../gaps/gap-reference-catalog.es.md#gt-674), [GT-687](../gaps/gap-reference-catalog.es.md#gt-687), [GT-692](../gaps/gap-reference-catalog.es.md#gt-692), [GT-536](../gaps/gap-reference-catalog.es.md#gt-536), +6 | ## Bloqueadores Actuales @@ -43,13 +43,13 @@ La forma correcta de usar este resumen es simple: si necesitas contexto, abre so |---|---:| | Fecha canónica del tablero | 2026-09-21 | | Gaps totales | 715 | -| Gaps cerrados | 688 | -| Gaps pendientes | 27 | +| Gaps cerrados | 689 | +| Gaps pendientes | 26 | | P0 abiertos | 1 | -| P1 abiertos | 11 | +| P1 abiertos | 10 | | P2 abiertos | 12 | -| Cierre total | 96.2% | -| Registros de evidencia de cierre | 670 | +| Cierre total | 96.4% | +| Registros de evidencia de cierre | 671 | | Readiness registrado | 4 PASS | | Área | Pendientes | P0 | P1 | Primeros IDs | @@ -58,7 +58,7 @@ La forma correcta de usar este resumen es simple: si necesitas contexto, abre so | `Cross` | 3 | 1 | 1 | [GT-435](../gaps/gap-reference-catalog.es.md#gt-435), [GT-448](../gaps/gap-reference-catalog.es.md#gt-448), [GT-651](../gaps/gap-reference-catalog.es.md#gt-651) | | `MCP Server` | 3 | 0 | 3 | [GT-684](../gaps/gap-reference-catalog.es.md#gt-684), [GT-680](../gaps/gap-reference-catalog.es.md#gt-680), [GT-681](../gaps/gap-reference-catalog.es.md#gt-681) | | `Infra` | 4 | 0 | 2 | [GT-710](../gaps/gap-reference-catalog.es.md#gt-710), [GT-324](../gaps/gap-reference-catalog.es.md#gt-324), [GT-464](../gaps/gap-reference-catalog.es.md#gt-464), [GT-692](../gaps/gap-reference-catalog.es.md#gt-692) | -| `Core Domain` | 3 | 0 | 1 | [GT-716](../gaps/gap-reference-catalog.es.md#gt-716), [GT-674](../gaps/gap-reference-catalog.es.md#gt-674), [GT-687](../gaps/gap-reference-catalog.es.md#gt-687) | +| `Evolith CLI` | 1 | 0 | 1 | [GT-714](../gaps/gap-reference-catalog.es.md#gt-714) | ## Fuente y Regla de Actualización diff --git a/reference/core/control-center/maturity-reports/executive-summary.md b/reference/core/control-center/maturity-reports/executive-summary.md index 7ad9c0fab..fb473369b 100644 --- a/reference/core/control-center/maturity-reports/executive-summary.md +++ b/reference/core/control-center/maturity-reports/executive-summary.md @@ -28,7 +28,7 @@ Use this summary with a simple rule: if you need context, open only the linked I | 1 | P0 blockers | They prevent production-readiness or major-release confidence. | [GT-435](../gaps/gap-reference-catalog.md#gt-435) | | 2 | Highest-risk area | `Governance` has the largest weighted open load. | [GT-670](../gaps/gap-reference-catalog.md#gt-670), [GT-585](../gaps/gap-reference-catalog.md#gt-585), [GT-669](../gaps/gap-reference-catalog.md#gt-669), [GT-672](../gaps/gap-reference-catalog.md#gt-672), [GT-689](../gaps/gap-reference-catalog.md#gt-689), [GT-588](../gaps/gap-reference-catalog.md#gt-588), +1 | | 3 | Quick wins | High criticality with XS/S complexity. | [GT-684](../gaps/gap-reference-catalog.md#gt-684), [GT-710](../gaps/gap-reference-catalog.md#gt-710), [GT-714](../gaps/gap-reference-catalog.md#gt-714) | -| 4 | P1 wave | Next hardening after P0 is cleared. | [GT-684](../gaps/gap-reference-catalog.md#gt-684), [GT-710](../gaps/gap-reference-catalog.md#gt-710), [GT-714](../gaps/gap-reference-catalog.md#gt-714), [GT-324](../gaps/gap-reference-catalog.md#gt-324), [GT-670](../gaps/gap-reference-catalog.md#gt-670), [GT-680](../gaps/gap-reference-catalog.md#gt-680), [GT-681](../gaps/gap-reference-catalog.md#gt-681), [GT-585](../gaps/gap-reference-catalog.md#gt-585), +3 | +| 4 | P1 wave | Next hardening after P0 is cleared. | [GT-684](../gaps/gap-reference-catalog.md#gt-684), [GT-710](../gaps/gap-reference-catalog.md#gt-710), [GT-714](../gaps/gap-reference-catalog.md#gt-714), [GT-324](../gaps/gap-reference-catalog.md#gt-324), [GT-670](../gaps/gap-reference-catalog.md#gt-670), [GT-680](../gaps/gap-reference-catalog.md#gt-680), [GT-681](../gaps/gap-reference-catalog.md#gt-681), [GT-585](../gaps/gap-reference-catalog.md#gt-585), +2 | | 5 | P2/P3 | Only after security, CI, rules, and contracts stabilize. | [GT-444](../gaps/gap-reference-catalog.md#gt-444), [GT-464](../gaps/gap-reference-catalog.md#gt-464), [GT-674](../gaps/gap-reference-catalog.md#gt-674), [GT-687](../gaps/gap-reference-catalog.md#gt-687), [GT-692](../gaps/gap-reference-catalog.md#gt-692), [GT-536](../gaps/gap-reference-catalog.md#gt-536), +6 | ## Current Blockers @@ -43,13 +43,13 @@ Use this summary with a simple rule: if you need context, open only the linked I |---|---:| | Canonical board date | 2026-09-21 | | Total gaps | 715 | -| Closed gaps | 688 | -| Open gaps | 27 | +| Closed gaps | 689 | +| Open gaps | 26 | | Open P0 | 1 | -| Open P1 | 11 | +| Open P1 | 10 | | Open P2 | 12 | -| Total closure | 96.2% | -| Closure evidence records | 670 | +| Total closure | 96.4% | +| Closure evidence records | 671 | | Recorded readiness | 4 PASS | | Area | Open | P0 | P1 | First IDs | @@ -58,7 +58,7 @@ Use this summary with a simple rule: if you need context, open only the linked I | `Cross` | 3 | 1 | 1 | [GT-435](../gaps/gap-reference-catalog.md#gt-435), [GT-448](../gaps/gap-reference-catalog.md#gt-448), [GT-651](../gaps/gap-reference-catalog.md#gt-651) | | `MCP Server` | 3 | 0 | 3 | [GT-684](../gaps/gap-reference-catalog.md#gt-684), [GT-680](../gaps/gap-reference-catalog.md#gt-680), [GT-681](../gaps/gap-reference-catalog.md#gt-681) | | `Infra` | 4 | 0 | 2 | [GT-710](../gaps/gap-reference-catalog.md#gt-710), [GT-324](../gaps/gap-reference-catalog.md#gt-324), [GT-464](../gaps/gap-reference-catalog.md#gt-464), [GT-692](../gaps/gap-reference-catalog.md#gt-692) | -| `Core Domain` | 3 | 0 | 1 | [GT-716](../gaps/gap-reference-catalog.md#gt-716), [GT-674](../gaps/gap-reference-catalog.md#gt-674), [GT-687](../gaps/gap-reference-catalog.md#gt-687) | +| `Evolith CLI` | 1 | 0 | 1 | [GT-714](../gaps/gap-reference-catalog.md#gt-714) | ## Source and Refresh Rule diff --git a/reference/core/control-center/maturity-reports/maturity-reconciliation.json b/reference/core/control-center/maturity-reports/maturity-reconciliation.json index 9e29a4e41..ddc27d425 100644 --- a/reference/core/control-center/maturity-reports/maturity-reconciliation.json +++ b/reference/core/control-center/maturity-reports/maturity-reconciliation.json @@ -4,13 +4,13 @@ "asOf": "2026-09-21", "gaps": { "total": 715, - "done": 688, - "pending": 3, + "done": 689, + "pending": 2, "inProgress": 3, "deferred": 21 }, "evidence": { - "closureRecords": 670, + "closureRecords": 671, "cliPackage": "@beyondnet/evolith-cli@1.4.0", "adrCount": 144, "rulesetCount": 184, diff --git a/reference/core/interfaces/how-to-construction.md b/reference/core/interfaces/how-to-construction.md index dee5f221a..d9fc4d09e 100644 --- a/reference/core/interfaces/how-to-construction.md +++ b/reference/core/interfaces/how-to-construction.md @@ -168,6 +168,13 @@ Response shape (captured live): "source": "", "unmatched": [] }, + "skippedByEvaluability": { + "documentation-only": "", + "needs-external-system": "", + "needs-runtime": "", + "needs-supplied-facts": "", + "unimplemented-native": "" + }, "skippedRuleIds": [ "" ], @@ -269,6 +276,13 @@ Response shape (captured live): "source": "", "unmatched": [] }, + "skippedByEvaluability": { + "documentation-only": "", + "needs-external-system": "", + "needs-runtime": "", + "needs-supplied-facts": "", + "unimplemented-native": "" + }, "skippedRuleIds": [ "" ], @@ -364,6 +378,13 @@ Response shape (captured live): "source": "", "unmatched": [] }, + "skippedByEvaluability": { + "documentation-only": "", + "needs-external-system": "", + "needs-runtime": "", + "needs-supplied-facts": "", + "unimplemented-native": "" + }, "skippedRuleIds": [ "" ], diff --git a/reference/core/interfaces/how-to-qa.md b/reference/core/interfaces/how-to-qa.md index 5cbe3d265..c467f679d 100644 --- a/reference/core/interfaces/how-to-qa.md +++ b/reference/core/interfaces/how-to-qa.md @@ -103,6 +103,13 @@ Response shape (captured live): "source": "", "unmatched": [] }, + "skippedByEvaluability": { + "documentation-only": "", + "needs-external-system": "", + "needs-runtime": "", + "needs-supplied-facts": "", + "unimplemented-native": "" + }, "skippedRuleIds": [ "" ], @@ -204,6 +211,13 @@ Response shape (captured live): "source": "", "unmatched": [] }, + "skippedByEvaluability": { + "documentation-only": "", + "needs-external-system": "", + "needs-runtime": "", + "needs-supplied-facts": "", + "unimplemented-native": "" + }, "skippedRuleIds": [ "" ], @@ -299,6 +313,13 @@ Response shape (captured live): "source": "", "unmatched": [] }, + "skippedByEvaluability": { + "documentation-only": "", + "needs-external-system": "", + "needs-runtime": "", + "needs-supplied-facts": "", + "unimplemented-native": "" + }, "skippedRuleIds": [ "" ], diff --git a/reference/core/sdlc/assets/master-view.svg b/reference/core/sdlc/assets/master-view.svg index fb3d8067f..f9c1f4f3f 100644 --- a/reference/core/sdlc/assets/master-view.svg +++ b/reference/core/sdlc/assets/master-view.svg @@ -430,7 +430,7 @@ 6 EVIDENCE — how honest is this picture (generated · dated · linked) -Gap board 688 / 715 · 27 not done +Gap board 689 / 715 · 26 not done P0 GT-435 · NO-GO generated 2026-09-21 diff --git a/src/packages/core-domain/src/application/validators/engine-coverage-advisory.spec.ts b/src/packages/core-domain/src/application/validators/engine-coverage-advisory.spec.ts index a382c57af..3e67d0d94 100644 --- a/src/packages/core-domain/src/application/validators/engine-coverage-advisory.spec.ts +++ b/src/packages/core-domain/src/application/validators/engine-coverage-advisory.spec.ts @@ -1,13 +1,11 @@ /** - * #628 — `evolith validate` with no flag runs the native evaluator, which decides - * materially fewer rules than `--engine opa` over the same corpus. Both totals - * were honest and every skip was published; what was missing was the sentence - * telling the reader the missing coverage belongs to the ENGINE THEY DID NOT - * CHOOSE rather than to their repository. + * #628 / GT-716 AC5 — `evolith validate` publishes every skip, and the row this + * suite pins is the sentence that says whom a skip belongs to: the ENGINE the + * reader chose, not their repository — and, since GT-716, WHY it happened. * - * These tests pin the two things that make the row worth having: it fires on the - * shape a reader misreads, and it stays quiet otherwise. A row on every run is - * noise that teaches people to skim past it. + * Two things make the row worth having: it fires on the shape a reader misreads + * (more skipped than checked) and stays quiet otherwise, and what it says is the + * run's own coverage split in the same terms the known-limitations page uses. */ import { RulesetValidatorService } from './ruleset-validator.service'; @@ -15,7 +13,7 @@ import type { RuleCoverage } from './ruleset-validator.types'; type Issue = { ruleId: string; blocking: boolean; severity: string; title: string; description: string }; -function coverage(checked: number, skipped: number, total: number): RuleCoverage { +function coverage(checked: number, skipped: number, total: number, split?: Record): RuleCoverage { return { rulesChecked: checked, rulesSkipped: skipped, @@ -23,6 +21,7 @@ function coverage(checked: number, skipped: number, total: number): RuleCoverage rulesTotal: total, skippedRuleIds: [], erroredRuleIds: [], + ...(split ? { skippedByEvaluability: split } : {}), } as unknown as RuleCoverage; } @@ -34,8 +33,8 @@ function advisoryFor(engineType: 'native' | 'opa', c: RuleCoverage): Issue | und }).engineCoverageAdvisory(c); } -describe('engine coverage advisory (#628)', () => { - it('fires when the native engine skips more than it checks', () => { +describe('engine coverage advisory (#628, GT-716 AC5)', () => { + it('fires when the native engine skips more than it checks, and attributes the gap to the engine', () => { const issue = advisoryFor('native', coverage(41, 118, 159)); expect(issue).toBeDefined(); @@ -47,17 +46,45 @@ describe('engine coverage advisory (#628)', () => { // The point of the row is the attribution, so it has to be in the title -- // a reader who only sees the issue table still gets it. expect(issue!.title).toContain('this is the engine, not your repository'); - expect(issue!.description).toContain('--engine opa'); + expect(issue!.title).toContain('native engine'); expect(issue!.description).toContain('41'); expect(issue!.description).toContain('118'); + expect(issue!.description).toContain('73-validate-engine-coverage-parity.mjs'); + }); + + it('fires on the OPA engine too, in its own name — GT-716 AC5: coverage is stated per engine', () => { + const issue = advisoryFor('opa', coverage(7, 152, 159)); + expect(issue).toBeDefined(); + expect(issue!.title).toContain('OPA engine'); + expect(issue!.description).toContain('--engine opa'); + }); + + it('states the split of its skips by class, in the words a reader can act on', () => { + const issue = advisoryFor('native', coverage(56, 103, 159, { + 'needs-supplied-facts': 31, 'needs-external-system': 20, 'needs-runtime': 14, 'documentation-only': 30, 'unimplemented-native': 8, + })); + const d = issue!.description; + expect(d).toContain('31 read a fact this run did not supply (needs-supplied-facts)'); + expect(d).toContain('`facts.satellite`'); + expect(d).toContain('the OPA engine decides it'); + expect(d).toContain('34 need an adapter'); + expect(d).toContain('30 carry no check'); + expect(d).toContain('8 have no native handler yet'); + // What this row can NOT say: it never ran the other engine. + expect(d).not.toMatch(/usually means the native evaluator has no handler/); }); - it('says nothing on the opa engine, however little it decided', () => { - expect(advisoryFor('opa', coverage(2, 157, 159))).toBeUndefined(); + it('on the OPA engine the supplied-fact class is the policy\'s, and the debt is a missing policy', () => { + const issue = advisoryFor('opa', coverage(7, 152, 159, { 'supplied-facet-absent': 120, 'no-policy-in-bundle': 25, 'documentation-only': 7 })); + const d = issue!.description; + expect(d).toContain('120 read a fact this run did not supply (supplied-facet-absent)'); + expect(d).toContain('the policy decides it'); + expect(d).toContain('25 have no policy in the bundle yet'); }); - it('says nothing when the native engine decided most of its scope', () => { + it('says nothing when the engine decided most of its scope', () => { expect(advisoryFor('native', coverage(133, 26, 159))).toBeUndefined(); + expect(advisoryFor('opa', coverage(133, 26, 159))).toBeUndefined(); }); it('does not fire on a tie, only when skips genuinely outnumber checks', () => { diff --git a/src/packages/core-domain/src/application/validators/rule-evaluation-engine.ts b/src/packages/core-domain/src/application/validators/rule-evaluation-engine.ts index d079e8447..ffd0abcc5 100644 --- a/src/packages/core-domain/src/application/validators/rule-evaluation-engine.ts +++ b/src/packages/core-domain/src/application/validators/rule-evaluation-engine.ts @@ -71,6 +71,14 @@ export function summarizeRuleCoverage(results: readonly RuleEvaluationResult[]): const evaluability = summarizeEvaluability(results.map(classifyResult)); + // GT-716 AC5 — the skips, by the class each one states. + const skippedByEvaluability: Record = {}; + for (const r of results) { + if (r.result !== 'skipped') continue; + const cls = classifyResult(r).evaluability; + skippedByEvaluability[cls] = (skippedByEvaluability[cls] ?? 0) + 1; + } + return { rulesChecked, rulesSkipped: skippedRuleIds.length, @@ -87,6 +95,7 @@ export function summarizeRuleCoverage(results: readonly RuleEvaluationResult[]): // not run, whether or not anything could ever run it. blockingSkippedRuleIds, perRuleset: evaluability.perRuleset.map(r => ({ ...r })), + skippedByEvaluability, }; } diff --git a/src/packages/core-domain/src/application/validators/ruleset-validator.service.ts b/src/packages/core-domain/src/application/validators/ruleset-validator.service.ts index 616f9f26c..20a3a09aa 100644 --- a/src/packages/core-domain/src/application/validators/ruleset-validator.service.ts +++ b/src/packages/core-domain/src/application/validators/ruleset-validator.service.ts @@ -42,6 +42,41 @@ export { SelectionReport, } from './ruleset-validator.types'; +/** + * GT-716 AC5 — the skips of one run, by the class each rule states, in the words a + * reader can act on. Classes are grouped by what closes them: a fact only the caller + * can supply (through `facts.satellite`, GT-694), an adapter over an external or + * running system, documentation that no engine can run, and the engine's own debt. + * The known-limitations table groups them the same way (guard 73's `SKIP_GROUPS`). + */ +const SPLIT_GROUPS = [ + 'needs-supplied-facts', 'supplied-facet-absent', 'needs-external-system', 'needs-runtime', + 'documentation-only', 'underspecified', 'no-policy-in-bundle', 'unimplemented-native', 'handler-declined', +]; + +function describeSkipSplit(byClass: Record | undefined, opa: boolean): string { + if (!byClass || Object.keys(byClass).length === 0) return '. '; + const n = (k: string) => byClass[k] ?? 0; + const supplied = n('needs-supplied-facts') + n('supplied-facet-absent'); + const adapters = n('needs-external-system') + n('needs-runtime'); + const docs = n('documentation-only') + n('underspecified'); + const debt = opa ? n('no-policy-in-bundle') : n('unimplemented-native') + n('handler-declined'); + const parts: string[] = []; + if (supplied > 0) { + parts.push( + `${supplied} read a fact this run did not supply (${opa ? 'supplied-facet-absent' : 'needs-supplied-facts'}) — ` + + "a posture only the repository's owners can declare; supply it through `facts.satellite` and " + + (opa ? 'the policy decides it' : 'the OPA engine decides it'), + ); + } + if (adapters > 0) parts.push(`${adapters} need an adapter over an external system or a running one (needs-external-system, needs-runtime)`); + if (docs > 0) parts.push(`${docs} carry no check any engine could run (documentation-only, underspecified)`); + if (debt > 0) parts.push(`${debt} have no ${opa ? 'policy in the bundle yet (no-policy-in-bundle)' : 'native handler yet (unimplemented-native)'}`); + const rest = Object.entries(byClass).filter(([k]) => !SPLIT_GROUPS.includes(k)).map(([k, v]) => `${v} ${k}`); + if (rest.length > 0) parts.push(rest.join(', ')); + return `: ${parts.join('; ')}. `; +} + @Injectable() export class RulesetValidatorService { private readonly logger: ILogger; @@ -313,6 +348,7 @@ export class RulesetValidatorService { // without parsing issue text. blockingSkippedRuleIds: coverage.blockingSkippedRuleIds, perRuleset: coverage.perRuleset, + skippedByEvaluability: coverage.skippedByEvaluability, // #628 — WHICH engine produced these numbers. Two engines ship and they do // not cover the same ground, so a coverage figure without an engine beside // it is not readable. @@ -549,49 +585,50 @@ export class RulesetValidatorService { } /** - * #628 -- `evolith validate` with no flag runs the native evaluator, which - * decided materially fewer rules than `--engine opa` over the same corpus when - * the row was written. (GT-716 later showed most of that extra reach was verdicts - * on facts nobody supplied, and the OPA engine now skips those instead; the - * advice below says "different", not "more", for that reason.) - * Both totals were honest and the skips were all published; what was missing - * was the sentence telling the reader that the missing coverage belongs to the - * ENGINE THEY DID NOT CHOOSE rather than to their repository. + * #628 / GT-716 AC5 -- the row that tells the reader whom the missing coverage + * belongs to. Both totals were always honest and every skip was published; what + * was missing was the sentence saying that a skip is the ENGINE's reach, not the + * repository's failure -- and, since GT-716, WHY each skip happened, because "no + * handler" was true of 14 rules and was being said of 240. * - * Deliberately narrow. It fires only on the native engine and only when skips - * outnumber checks, because that is the shape a reader misreads. A run where - * the engine decided most of what it was handed needs no explanation, and a - * row on every run is noise that teaches people to skim past it. + * It fires on either engine, and only when skips outnumber checks: that is the + * shape a reader misreads, and a row on every run is noise that teaches people + * to skim past it. What it states is this run's own coverage, in the terms the + * known-limitations page uses (`skippedByEvaluability`), so the report and the + * page say the same thing about the same engine. It cannot know what the other + * engine would have decided; it says where that is measured. * * Non-blocking. The engines are ALLOWED to differ on coverage -- - * `68-validate-engine-verdict-parity.mjs` holds them to agreement on facts, - * not on reach -- so this reports a fact about the run, it does not fail it. + * `68-validate-engine-verdict-parity.mjs` holds them to agreement on facts, and + * `73-validate-engine-coverage-parity.mjs` registers every rule only one of them + * decides -- so this reports a fact about the run, it does not fail it. */ private engineCoverageAdvisory(coverage: RuleCoverage): ValidationIssue | undefined { - if (this.engineType !== 'native') return undefined; if (coverage.rulesSkipped <= coverage.rulesChecked) return undefined; + if (coverage.rulesTotal <= 0) return undefined; - const share = coverage.rulesTotal > 0 - ? Math.round((coverage.rulesSkipped / coverage.rulesTotal) * 100) - : 0; + const opa = this.engineType === 'opa'; + const engine = opa ? 'OPA' : 'native'; + const share = Math.round((coverage.rulesSkipped / coverage.rulesTotal) * 100); + const split = describeSkipSplit(coverage.skippedByEvaluability, opa); return { ruleId: 'GOV-ENGINE-COVERAGE', severity: 'COULD', category: 'governance', title: - `The native engine skipped more rules than it checked ` + + `The ${engine} engine skipped more rules than it checked ` + `(${coverage.rulesSkipped} of ${coverage.rulesTotal}) — this is the engine, not your repository`, description: - `This run used the native evaluator, the default when no \`--engine\` is given. It decided ` + - `${coverage.rulesChecked} of the ${coverage.rulesTotal} rules in scope and skipped ` + - `${coverage.rulesSkipped} (${share}%). A skip here usually means the native evaluator has no ` + - 'handler for that rule, not that your repository failed to satisfy it. ' + - '`--engine opa` evaluates the compiled Rego bundle instead, which decides a DIFFERENT part of the ' + - 'same corpus: most of its policies read facts a caller supplies (`facts.satellite`, GT-694), and ' + - 'since GT-716 it reports a rule whose fact the run did not supply as skipped rather than deciding it ' + - 'on absent input. The two engines are held to agreement on the verdicts they both reach; they are ' + - 'not held to equal reach.', + (opa + ? 'This run used the compiled Rego bundle (`--engine opa`). ' + : 'This run used the native evaluator, the default when no `--engine` is given. ') + + `It decided ${coverage.rulesChecked} of the ${coverage.rulesTotal} rules in scope and skipped ` + + `${coverage.rulesSkipped} (${share}%)${split}` + + 'The other engine decides a DIFFERENT part of the same corpus: every rule only one engine decides ' + + 'is registered per rule by CI (`73-validate-engine-coverage-parity.mjs`), and the two are held to ' + + 'agreement on the verdicts they both reach, not to equal reach. The measured coverage of each engine, ' + + 'on this repository and on a fresh satellite, is on the known-limitations page.', blocking: false, }; } diff --git a/src/packages/core-domain/src/application/validators/ruleset-validator.types.ts b/src/packages/core-domain/src/application/validators/ruleset-validator.types.ts index 5b7d353c4..3372b778c 100644 --- a/src/packages/core-domain/src/application/validators/ruleset-validator.types.ts +++ b/src/packages/core-domain/src/application/validators/ruleset-validator.types.ts @@ -65,6 +65,15 @@ export interface RuleCoverage { blockingSkippedRuleIds?: string[]; /** GT-595 AC3 — `handled / executable / total` per ruleset file. */ perRuleset?: RulesetCoverageRatio[]; + + /** + * GT-716 AC5 — `rulesSkipped`, split by WHY: the evaluability class each skipped + * rule states (a posture nobody supplied, an adapter nobody wrote, documentation, + * a missing handler or policy). The report and the known-limitations page state + * coverage per engine in these terms; a bare "skipped N" was the number the front + * page misread as reach. Sums to `rulesSkipped`; classes with zero skips are absent. + */ + skippedByEvaluability?: Record; } /** GT-595 AC3 — the coverage of one `*.rules.json`, so the ratio is per ruleset. */ @@ -138,6 +147,8 @@ export interface ValidationResult { /** GT-595 AC2 — blocking rules that did not run. Non-empty ⇒ `status: 'failed'`. */ blockingSkippedRuleIds?: string[]; perRuleset?: RulesetCoverageRatio[]; + /** GT-716 AC5 — the run's skips by evaluability class; see `RuleCoverage.skippedByEvaluability`. */ + skippedByEvaluability?: Record; /** * GT-661 — WHY these rules were evaluated, not just how many. * diff --git a/src/tests/contract/sdk-type-contract.types.ts b/src/tests/contract/sdk-type-contract.types.ts index 7b85259fc..ee435a9b4 100644 --- a/src/tests/contract/sdk-type-contract.types.ts +++ b/src/tests/contract/sdk-type-contract.types.ts @@ -369,6 +369,13 @@ export const WIRE_VALIDATION_RESULT: WireCheck = { // verdict is entitled to the ids behind it without parsing issue text. blockingSkippedRuleIds: { required: false, declaredAs: 'string[]', accepts: isArray }, perRuleset: { required: false, declaredAs: 'RulesetCoverageRatio[]', accepts: isArray }, + // GT-716 AC5: `rulesSkipped` split by the evaluability class each skipped rule + // states — a posture nobody supplied, an adapter nobody wrote, documentation, a + // missing handler or policy. The reporter's GOV-ENGINE-COVERAGE row and the + // known-limitations table state coverage per engine in these terms; a consumer + // reading `rulesSkipped` alone reads reach it cannot attribute. Sums to + // `rulesSkipped`; classes with zero skips are absent. + skippedByEvaluability: { required: false, declaredAs: 'Record', accepts: isObject }, // GT-661: the SCOPE of the verdict. A consumer that reads `status: 'failed'` // without this cannot tell "the packs I adopted failed" from "the Core // evaluated all 402 of its own opinions and something failed" — measured on