diff --git a/docs/evidence-pipeline-contract.md b/docs/evidence-pipeline-contract.md index 59c54f0..73d0404 100644 --- a/docs/evidence-pipeline-contract.md +++ b/docs/evidence-pipeline-contract.md @@ -75,12 +75,19 @@ Run: ```bash python scripts/regenerate_artifacts.py --check +python scripts/validate_run_manifest.py data/processed/run_manifest.json +python -m pytest tests/test_run_manifest_contract.py python -m pytest tests/test_evidence_pipeline_schemas.py ``` The regeneration check compares byte-stable CSV, JSON, JSONL, and Markdown artifacts with fresh pipeline output. It also regenerates PNG visual snapshots to verify that the plotting path still runs, but it does not byte-compare those images because Matplotlib rendering can vary across platforms. -The schema test validates each schema file and checks that every committed JSON artifact and JSONL record listed in the schema matrix conforms to it. +The run-manifest validator selects v1 or v2 only from the exact embedded marker +and then validates against that schema; it never falls back to a default or +newest version. Its compatibility test covers representative v1 and v2 +fixtures plus missing, malformed, and unknown markers. The evidence-pipeline +schema test validates each schema file and checks that every committed JSON +artifact and JSONL record listed in the schema matrix conforms to it. ## Compatibility Matrix diff --git a/docs/schema-compatibility-matrix.md b/docs/schema-compatibility-matrix.md index 7420f1e..8fcc1bd 100644 --- a/docs/schema-compatibility-matrix.md +++ b/docs/schema-compatibility-matrix.md @@ -70,10 +70,41 @@ values derive from the same byte snapshot. These fingerprints are not signatures and do not imply live telemetry coverage. The strict v1 schema is preserved at `schemas/run_manifest.schema.json`. -Current writers identify `run-manifest/v2`, whose per-file maps are required. -Consumers must select the matching schema version; a strict v1 validator is -expected to reject a v2 manifest. The aggregate digest algorithm and all six -committed aggregate values remain unchanged from v1.2. +Current writers continue to emit only `run-manifest/v2`; this reader-side +routing contract does not add dual-write behavior or change committed +artifacts. A strict v1 validator is expected to reject a v2 manifest. The +aggregate digest algorithm and all six committed aggregate values remain +unchanged from v1.2. + +### Exact schema selection + +`telemetry_lab.run_manifest_contract.RUN_MANIFEST_SCHEMA_REGISTRY` is the +authoritative reader mapping: + +| Embedded marker | Selected schema | +| --- | --- | +| `run-manifest/v1` | `schemas/run_manifest.schema.json` | +| `run-manifest/v2` | `schemas/run_manifest.v2.schema.json` | + +`select_run_manifest_schema()` reads only +`artifact_schema_versions.run_manifest` and requires an exact registry key. +Missing objects or markers, non-string and blank markers, unknown versions, +case changes, and surrounding whitespace all fail closed. There is no +fallback to v1, v2, or the newest schema. Schema-shape validation happens only +after this selection, so a marker cannot silently choose a different contract +because its payload happens to fit that schema. + +Validate either historical v1 input or current v2 input from a checkout with: + +```bash +python scripts/validate_run_manifest.py path/to/run_manifest.json +``` + +The command reports the selected marker and repository-relative schema path on +success. It returns a nonzero status for invalid JSON, unreadable files, +version-selection failures, or schema validation failures. Representative v1 +and v2 inputs live under `tests/fixtures/run_manifests/`; the compatibility +test requires each fixture to validate only against its selected schema. ## Boundaries diff --git a/scripts/validate_run_manifest.py b/scripts/validate_run_manifest.py new file mode 100644 index 0000000..2f37fdc --- /dev/null +++ b/scripts/validate_run_manifest.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +from jsonschema import Draft202012Validator, FormatChecker +from jsonschema.exceptions import SchemaError, ValidationError + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPO_ROOT / "src" +if str(SRC_ROOT) not in sys.path: + sys.path.insert(0, str(SRC_ROOT)) + +from telemetry_lab.run_manifest_contract import ( # noqa: E402 + RunManifestSchemaSelection, + RunManifestVersionError, + select_run_manifest_schema, +) + + +class RunManifestValidationError(ValueError): + """Raised when a selected run-manifest schema rejects the document.""" + + +def validate_run_manifest(manifest_path: Path) -> RunManifestSchemaSelection: + """Load, route, and validate one run manifest from a repository checkout.""" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + selection = select_run_manifest_schema(manifest) + schema = json.loads( + (REPO_ROOT / selection.schema_path).read_text(encoding="utf-8") + ) + + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + errors = sorted( + validator.iter_errors(manifest), + key=lambda error: [str(part) for part in error.absolute_path], + ) + if errors: + summary = "\n".join(_format_error(error) for error in errors[:5]) + raise RunManifestValidationError( + f"manifest does not satisfy {selection.schema_version} " + f"({selection.schema_path.as_posix()}):\n{summary}" + ) + + return selection + + +def _format_error(error: ValidationError) -> str: + path = ".".join(str(part) for part in error.absolute_path) or "" + return f"{path}: {error.message}" + + +def main(argv: Sequence[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + manifest_path = Path(args.manifest) + + try: + selection = validate_run_manifest(manifest_path) + except ( + OSError, + UnicodeError, + json.JSONDecodeError, + RunManifestVersionError, + RunManifestValidationError, + SchemaError, + ) as exc: + print(f"[FAIL] {manifest_path}: {exc}", file=sys.stderr) + return 1 + + print( + f"[OK] {manifest_path}: {selection.schema_version} -> " + f"{selection.schema_path.as_posix()}" + ) + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Select the exact supported run-manifest schema from its embedded " + "version marker and validate the document without fallback." + ) + ) + parser.add_argument("manifest", help="Path to one run_manifest.json document.") + return parser + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/telemetry_lab/run_manifest_contract.py b/src/telemetry_lab/run_manifest_contract.py new file mode 100644 index 0000000..87ca77a --- /dev/null +++ b/src/telemetry_lab/run_manifest_contract.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import PurePosixPath +from types import MappingProxyType +from typing import Final + + +class RunManifestVersionError(ValueError): + """Raised when a run manifest cannot select one exact supported schema.""" + + +@dataclass(frozen=True) +class RunManifestSchemaSelection: + """The exact schema contract selected by a run manifest marker.""" + + schema_version: str + schema_path: PurePosixPath + + +RUN_MANIFEST_SCHEMA_REGISTRY: Final[Mapping[str, PurePosixPath]] = MappingProxyType( + { + "run-manifest/v1": PurePosixPath("schemas/run_manifest.schema.json"), + "run-manifest/v2": PurePosixPath("schemas/run_manifest.v2.schema.json"), + } +) + + +def select_run_manifest_schema(manifest: object) -> RunManifestSchemaSelection: + """Select a schema by exact version marker, rejecting missing or unknown markers.""" + if not isinstance(manifest, Mapping): + raise RunManifestVersionError("run manifest must be a JSON object") + + artifact_versions = manifest.get("artifact_schema_versions") + if not isinstance(artifact_versions, Mapping): + raise RunManifestVersionError( + "artifact_schema_versions must be a JSON object" + ) + if "run_manifest" not in artifact_versions: + raise RunManifestVersionError( + "artifact_schema_versions.run_manifest is required" + ) + + schema_version = artifact_versions["run_manifest"] + if not isinstance(schema_version, str) or not schema_version.strip(): + raise RunManifestVersionError( + "artifact_schema_versions.run_manifest must be a non-empty string" + ) + + schema_path = RUN_MANIFEST_SCHEMA_REGISTRY.get(schema_version) + if schema_path is None: + supported_versions = ", ".join(sorted(RUN_MANIFEST_SCHEMA_REGISTRY)) + raise RunManifestVersionError( + f"unsupported run manifest schema version {schema_version!r}; " + f"supported versions: {supported_versions}" + ) + + return RunManifestSchemaSelection( + schema_version=schema_version, + schema_path=schema_path, + ) diff --git a/tests/fixtures/run_manifests/v1.json b/tests/fixtures/run_manifests/v1.json new file mode 100644 index 0000000..9a3009c --- /dev/null +++ b/tests/fixtures/run_manifests/v1.json @@ -0,0 +1,10 @@ +{ + "tool_version": "1.2.0", + "demo_id": "window", + "input_digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "config_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "artifact_schema_versions": { + "run_manifest": "run-manifest/v1" + }, + "execution_mode": "synthetic-local" +} diff --git a/tests/fixtures/run_manifests/v2.json b/tests/fixtures/run_manifests/v2.json new file mode 100644 index 0000000..4b3b103 --- /dev/null +++ b/tests/fixtures/run_manifests/v2.json @@ -0,0 +1,16 @@ +{ + "tool_version": "1.2.0", + "demo_id": "window", + "input_digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "config_digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "input_file_digests": { + "data/raw/events.jsonl": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + }, + "config_file_digests": { + "configs/window.yaml": "sha256:3333333333333333333333333333333333333333333333333333333333333333" + }, + "artifact_schema_versions": { + "run_manifest": "run-manifest/v2" + }, + "execution_mode": "synthetic-local" +} diff --git a/tests/test_run_manifest_contract.py b/tests/test_run_manifest_contract.py new file mode 100644 index 0000000..05c7dfa --- /dev/null +++ b/tests/test_run_manifest_contract.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator, FormatChecker + +from telemetry_lab.run_manifest_contract import ( + RunManifestVersionError, + select_run_manifest_schema, +) + + +REPO_ROOT = Path(__file__).resolve().parents[1] +FIXTURE_ROOT = REPO_ROOT / "tests" / "fixtures" / "run_manifests" + + +def _load_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize( + ("fixture_name", "schema_version", "schema_path", "incompatible_schema_path"), + [ + ( + "v1.json", + "run-manifest/v1", + "schemas/run_manifest.schema.json", + "schemas/run_manifest.v2.schema.json", + ), + ( + "v2.json", + "run-manifest/v2", + "schemas/run_manifest.v2.schema.json", + "schemas/run_manifest.schema.json", + ), + ], +) +def test_select_run_manifest_schema_uses_exact_compatible_version( + fixture_name: str, + schema_version: str, + schema_path: str, + incompatible_schema_path: str, +) -> None: + manifest = _load_json(FIXTURE_ROOT / fixture_name) + + selection = select_run_manifest_schema(manifest) + + assert selection.schema_version == schema_version + assert selection.schema_path.as_posix() == schema_path + + selected_schema = _load_json(REPO_ROOT / selection.schema_path) + incompatible_schema = _load_json(REPO_ROOT / incompatible_schema_path) + selected_validator = Draft202012Validator( + selected_schema, + format_checker=FormatChecker(), + ) + incompatible_validator = Draft202012Validator( + incompatible_schema, + format_checker=FormatChecker(), + ) + + assert list(selected_validator.iter_errors(manifest)) == [] + assert list(incompatible_validator.iter_errors(manifest)) + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + ([], "run manifest must be a JSON object"), + ({}, "artifact_schema_versions must be a JSON object"), + ( + {"artifact_schema_versions": []}, + "artifact_schema_versions must be a JSON object", + ), + ( + {"artifact_schema_versions": {}}, + "artifact_schema_versions.run_manifest is required", + ), + ( + {"artifact_schema_versions": {"run_manifest": None}}, + "artifact_schema_versions.run_manifest must be a non-empty string", + ), + ( + {"artifact_schema_versions": {"run_manifest": " "}}, + "artifact_schema_versions.run_manifest must be a non-empty string", + ), + ], +) +def test_select_run_manifest_schema_rejects_missing_or_invalid_markers( + manifest: object, + message: str, +) -> None: + with pytest.raises(RunManifestVersionError, match=message): + select_run_manifest_schema(manifest) + + +@pytest.mark.parametrize( + "schema_version", + ["run-manifest/v3", "run-manifest/v2 ", "RUN-MANIFEST/V2"], +) +def test_select_run_manifest_schema_rejects_unknown_versions_without_fallback( + schema_version: str, +) -> None: + manifest = {"artifact_schema_versions": {"run_manifest": schema_version}} + + with pytest.raises(RunManifestVersionError) as exc_info: + select_run_manifest_schema(manifest) + + assert str(exc_info.value) == ( + f"unsupported run manifest schema version {schema_version!r}; " + "supported versions: run-manifest/v1, run-manifest/v2" + ) + + +@pytest.mark.parametrize( + ("fixture_name", "schema_version", "schema_path"), + [ + ("v1.json", "run-manifest/v1", "schemas/run_manifest.schema.json"), + ("v2.json", "run-manifest/v2", "schemas/run_manifest.v2.schema.json"), + ], +) +def test_validate_run_manifest_script_reports_selected_contract( + fixture_name: str, + schema_version: str, + schema_path: str, +) -> None: + fixture_path = FIXTURE_ROOT / fixture_name + + result = subprocess.run( + [sys.executable, "scripts/validate_run_manifest.py", str(fixture_path)], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert f"{schema_version} -> {schema_path}" in result.stdout + + +def test_validate_run_manifest_script_rejects_unknown_version(tmp_path: Path) -> None: + manifest_path = tmp_path / "unknown.json" + manifest_path.write_text( + json.dumps( + {"artifact_schema_versions": {"run_manifest": "run-manifest/v3"}} + ), + encoding="utf-8", + ) + + result = subprocess.run( + [sys.executable, "scripts/validate_run_manifest.py", str(manifest_path)], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert result.stdout == "" + assert "unsupported run manifest schema version 'run-manifest/v3'" in result.stderr