From 4e5a25271933124d2c65f7daa918ba458c327b90 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:46:11 +0000 Subject: [PATCH 01/78] test(analysis): add Tier 1 real-audio accuracy acceptance Prove a decoded C major WAV recovers C and a 120 BPM click WAV passes tempo Acc1, with checksum-fail-closed case reports. Co-authored-by: Seongho Bae --- AGENTS.md | 2 + ARCHITECTURE.md | 3 +- CHANGELOG.md | 1 + CLAUDE.md | 1 + .../real-audio-accuracy-acceptance.md | 67 ++++++ ...26-08-16-real-audio-accuracy-acceptance.md | 50 +++++ .../bandscope_analysis/accuracy/__init__.py | 42 ++++ .../bandscope_analysis/accuracy/evaluate.py | 79 +++++++ .../bandscope_analysis/accuracy/fixtures.py | 133 ++++++++++++ .../bandscope_analysis/accuracy/manifest.py | 141 ++++++++++++ .../bandscope_analysis/accuracy/metrics.py | 70 ++++++ .../tests/test_accuracy_acceptance.py | 205 ++++++++++++++++++ 12 files changed, 793 insertions(+), 1 deletion(-) create mode 100644 docs/doctoring/real-audio-accuracy-acceptance.md create mode 100644 docs/plans/2026-08-16-real-audio-accuracy-acceptance.md create mode 100644 services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py create mode 100644 services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py create mode 100644 services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py create mode 100644 services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py create mode 100644 services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py create mode 100644 services/analysis-engine/tests/test_accuracy_acceptance.py diff --git a/AGENTS.md b/AGENTS.md index fca448ce9..c7a4f66b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,7 @@ This section applies to any agent (Claude, Codex, Cursor, opencode, ...) working ## Architecture references - `ARCHITECTURE.md` - `docs/engineering/acceptance-criteria.md` +- `docs/doctoring/real-audio-accuracy-acceptance.md` - `docs/engineering/harness-engineering.md` - `docs/workflow/one-day-delivery-plan.md` - `docs/workflow/pr-continuity.md` @@ -85,6 +86,7 @@ This section applies to any agent (Claude, Codex, Cursor, opencode, ...) working - Prefer practical, friendly, rehearsal-first wording over academic or authority-heavy language. - Do not reduce the product to a chord analyzer when form, timing, player coordination, simplification, and setup cues are the real rehearsal blockers. - Do not frame usability as a reason to accept weak analysis quality; BandScope should aim for both easy use and high accuracy. +- Do not claim harmony or tempo accuracy from mocked feature matrices. Tier 1 decoded-PCM fixtures and metrics live in `docs/doctoring/real-audio-accuracy-acceptance.md`. ## Safety - Do not add network-dependent runtime paths for local analysis. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3302a6fc3..8ee276119 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # ARCHITECTURE.md -Last updated: 2026-03-11 +Last updated: 2026-08-16 ## Brand source @@ -127,3 +127,4 @@ Last updated: 2026-03-11 - Supply-chain docs, workflow pinning, and lockfile verification are part of the default quickcheck path so dependency drift is caught early. - Quickcheck and CI are expected to verify dependency review, audit, supplemental inventory, and SBOM baseline presence as part of bootstrap. - Cross-platform build workflow presence and trigger coverage are part of the default supply-chain verification path. +- Real-audio accuracy acceptance is a separate buyer-facing gate from unit coverage. Tier 1 decoded-PCM fixtures, duration-weighted chord recall, and tempo Acc1 live in `docs/doctoring/real-audio-accuracy-acceptance.md` and `bandscope_analysis.accuracy`. diff --git a/CHANGELOG.md b/CHANGELOG.md index eea696893..599d43428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Add Tier 1 real-audio accuracy acceptance: a decoded C major WAV must recover `C`, and a 120 BPM click WAV must pass tempo Acc1, with checksum-fail-closed reports. - Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace. - 각 합주 역할(Role)별 개인 연습 진행도를 0~100% 범위로 기록 및 시각화할 수 있는 연습 진척도(`practiceProgress`) 트래커 기능 추가. UI 컨트롤(슬라이더 및 +/- 버튼)과 한/영 다국어 지원 포함. diff --git a/CLAUDE.md b/CLAUDE.md index 82c2c704a..62ea64742 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,3 +73,4 @@ Supporting packages: - Lockfiles (`package-lock.json`, `uv.lock`, `Cargo.lock`) are committed and must stay in sync; GitHub Actions are SHA-pinned. Adding a direct dependency requires the admission rationale defined in `AGENTS.md` and `docs/security/dependency-policy.md`. - CI beyond quickcheck: `gate / ci / rust-check` (Tauri cargo check on macOS) and `build-baseline` Windows/macOS amd64+arm64 native builds are merge gates, alongside CodeQL, dependency-review, sbom, bandit, trivy, secret-scan, and security-audit workflows. Do not weaken or skip them. - Version metadata lives in `VERSION`, the root `package.json`, and `CHANGELOG.md`; release flow is tag-driven (see `docs/operations/deploy-runbook.md`). +- Real-audio accuracy acceptance (decoded PCM, not mocked chroma) lives in `docs/doctoring/real-audio-accuracy-acceptance.md` and `bandscope_analysis.accuracy`. diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md new file mode 100644 index 000000000..0a96f9840 --- /dev/null +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -0,0 +1,67 @@ +# Real-audio accuracy acceptance + +Next action: before claiming a rehearsal result is accurate, run the Tier 1 +decoded-PCM cases. A green unit suite that never opens a WAV is not buyer +evidence. + +```bash +uv run --project services/analysis-engine pytest \ + services/analysis-engine/tests/test_accuracy_acceptance.py +``` + +## Why this lock exists + +BandScope sells rehearsal guidance. A buyer cannot distinguish “the job +succeeded” from “the engine heard the chord and tempo that are in the file” +until decoded PCM is scored against a known label (Raffel et al., 2014). + +This layer is Tier 1 of issue #770: tiny, license-clean, deterministic WAV +fixtures generated in process. It does not claim genre coverage, perceptual +stem quality, or private-corpus readiness. + +## Held cases + +- `c-major-triad`: three seconds of C4+E4+G4 written to WAV, decoded, and + scored with duration-weighted chord recall. Pass when recall of `C` is at + least `0.70` (Odekerken et al., 2021). +- `click-120-bpm`: eight seconds of 120 BPM clicks decoded by + `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% + (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. +- Checksum mismatch fails closed. Do not score a tampered file as a pass. + +## Claim boundary + +A passing case supports only the registered fixture, metric, engine version, +and tolerance. It does not establish universal musical correctness. + +Keys-left, keys-right, and acoustic-guitar roles still carry arrangement +defaults (`C#`, `Emaj7`, `Eb`). Lead vocal harmony is the role that currently +reflects the `other` stem recognizer. Do not treat those defaults as +measured accuracy. + +## References + +Odekerken, D., Koops, H. V., & Volk, A. (2021). Improving audio chord +estimation by alignment and integration of crowd-sourced symbolic music. +*Transactions of the International Society for Music Information Retrieval, +4*(1), 141–155. https://doi.org/10.5334/tismir.81 + +Raffel, C., McFee, B., Humphrey, E. J., Salamon, J., Nieto, O., Liang, D., & +Ellis, D. P. W. (2014). MIR_EVAL: A transparent implementation of common MIR +metrics. In *Proceedings of the 15th International Society for Music +Information Retrieval Conference* (pp. 367–372). + +Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? +*Transactions of the International Society for Music Information Retrieval, +3*(1), 111–125. https://doi.org/10.5334/tismir.43 + +## Security Notes + +- Attack surface: generated WAV bytes, SHA-256 digests, and decoded PCM + passed into `ChordRecognizer` and `TemporalAnalyzer`. +- Trust boundary: untrusted audio and manifests; trusted repo-controlled + fixture generators and metric floors. +- Mitigations: no network, no shell, checksum fail-closed, bounded fixture + durations, no copyrighted commercial recordings. +- Test points: deterministic digest, C major recall, 120 BPM Acc1, checksum + mismatch, malformed manifest, silence must not pass as C major. diff --git a/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md b/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md new file mode 100644 index 000000000..9a33f0e91 --- /dev/null +++ b/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md @@ -0,0 +1,50 @@ +# Real-audio accuracy acceptance (Tier 1) + +**Goal:** Give BandScope a buyer-readable accuracy gate: decoded PCM from a +known WAV must recover the expected chord or tempo, with a versioned report. + +**Architecture:** A new `bandscope_analysis.accuracy` package generates +license-clean fixtures, scores them with documented MIR metrics, and emits a +strict case-report schema. Tests write real WAV files and call production +`ChordRecognizer` and `TemporalAnalyzer` helpers. Stem separation stays out +of this slice. + +**Tech Stack:** NumPy, soundfile, librosa beat tracking, pytest. + +## Security Notes + +### Attack surface + +Generated WAV files, SHA-256 digests, decoded PCM arrays, and JSON-shaped +accuracy reports. + +### Trust boundary + +Untrusted: any on-disk fixture bytes and any parsed report mapping. +Trusted: in-repo generators, metric definitions, and registered floors. + +### Mitigations + +- No network and no shell interpolation. +- Checksum mismatch raises before tempo scoring. +- Manifest parsing fails closed on missing or mistyped fields. +- Fixtures are short, synthetic, and license-clean. + +### Test points + +- Deterministic C major digest +- Duration-weighted C recall after file write +- 120 BPM Acc1 after file decode +- Checksum mismatch +- Malformed report rejection +- Silence must not pass as C major + +### Realistic threats + +A tampered fixture or a missing digest could be scored as a pass and then +cited as release evidence. + +### Remaining risk + +Tier 1 does not cover Demucs stems, private commercial recordings, or +CPU/GPU numeric parity. Those remain later #770 tiers. diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py b/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py new file mode 100644 index 000000000..1c451394a --- /dev/null +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py @@ -0,0 +1,42 @@ +"""Real-audio accuracy acceptance helpers for BandScope rehearsal analysis. + +Next action: run the Tier 1 fixture tests before claiming a harmony or tempo +result is accurate. These helpers score decoded PCM against known labels. They +do not replace Demucs stem separation or private-corpus benchmarks. +""" + +from .evaluate import evaluate_c_major_pcm, evaluate_click_tempo_file +from .fixtures import ( + C_MAJOR_LABEL, + DEFAULT_CLICK_BPM, + DEFAULT_SAMPLE_RATE, + assert_fixture_checksum, + render_c_major_triad, + render_click_track, + write_pcm_wav, +) +from .manifest import ( + AccuracyCaseReport, + build_case_report, + parse_case_report, + read_product_version, +) +from .metrics import duration_weighted_chord_recall, tempo_acc1 + +__all__ = [ + "AccuracyCaseReport", + "C_MAJOR_LABEL", + "DEFAULT_CLICK_BPM", + "DEFAULT_SAMPLE_RATE", + "assert_fixture_checksum", + "build_case_report", + "duration_weighted_chord_recall", + "evaluate_c_major_pcm", + "evaluate_click_tempo_file", + "parse_case_report", + "read_product_version", + "render_c_major_triad", + "render_click_track", + "tempo_acc1", + "write_pcm_wav", +] diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py new file mode 100644 index 000000000..2fe1c3f0a --- /dev/null +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py @@ -0,0 +1,79 @@ +"""Score decoded PCM fixtures through production analysis helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +from numpy.typing import NDArray + +from bandscope_analysis.accuracy.fixtures import ( + C_MAJOR_LABEL, + DEFAULT_CLICK_BPM, + DEFAULT_SAMPLE_RATE, + assert_fixture_checksum, +) +from bandscope_analysis.accuracy.manifest import AccuracyCaseReport, build_case_report +from bandscope_analysis.accuracy.metrics import duration_weighted_chord_recall, tempo_acc1 +from bandscope_analysis.chords.chord_recognizer import ChordRecognizer +from bandscope_analysis.temporal.analyzer import TemporalAnalyzer + +C_MAJOR_RECALL_FLOOR = 0.70 + + +def evaluate_c_major_pcm( + audio: NDArray[np.floating], + sample_rate: int = DEFAULT_SAMPLE_RATE, + audio_sha256: str = "0" * 64, +) -> AccuracyCaseReport: + """Score a C major triad through the production chord recognizer. + + Args: + audio: Decoded mono PCM. Do not pass a chroma matrix. + sample_rate: Sample rate of ``audio``. + audio_sha256: Digest of the on-disk fixture that produced ``audio``. + + Returns: + A case report whose metric is duration-weighted recall of ``C``. + """ + recognizer = ChordRecognizer() + tracked = recognizer.recognize(np.asarray(audio, dtype=np.float32), sr=sample_rate) + segments = [(item["start_time"], item["end_time"], item["chord"]) for item in tracked] + duration = float(len(audio) / sample_rate) + recall = duration_weighted_chord_recall(segments, C_MAJOR_LABEL, 0.0, duration) + return build_case_report( + case_id="c-major-triad", + audio_sha256=audio_sha256, + metric_name="duration_weighted_chord_recall", + metric_value=recall, + passed=recall >= C_MAJOR_RECALL_FLOOR, + true_label=C_MAJOR_LABEL, + ) + + +def evaluate_click_tempo_file( + audio_path: Path, + expected_sha256: str, + true_bpm: float = DEFAULT_CLICK_BPM, +) -> AccuracyCaseReport: + """Decode a click-track WAV with TemporalAnalyzer and score Acc1. + + Args: + audio_path: On-disk WAV written by ``write_pcm_wav``. + expected_sha256: Registered digest. Mismatch fails closed. + true_bpm: Known click tempo. + + Returns: + A case report whose metric is 1.0 on Acc1 pass and 0.0 on Acc1 fail. + """ + assert_fixture_checksum(audio_path, expected_sha256) + features = TemporalAnalyzer().analyze(audio_path) + passed = tempo_acc1(features["bpm"], true_bpm) + return build_case_report( + case_id="click-120-bpm", + audio_sha256=expected_sha256, + metric_name="tempo_acc1", + metric_value=1.0 if passed else 0.0, + passed=passed, + true_label=f"{true_bpm:g} bpm", + ) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py new file mode 100644 index 000000000..a1b794f81 --- /dev/null +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -0,0 +1,133 @@ +"""Deterministic, license-clean PCM fixtures for accuracy acceptance.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import numpy as np +import soundfile as sf # type: ignore[import-untyped] +from numpy.typing import NDArray + +C_MAJOR_LABEL = "C" +DEFAULT_SAMPLE_RATE = 22_050 +DEFAULT_CLICK_BPM = 120.0 +C4_HZ = 261.63 +E4_HZ = 329.63 +G4_HZ = 392.00 +_CLICK_FREQUENCY_HZ = 1_000.0 +_CLICK_DURATION_SECONDS = 0.01 +_CLICK_DECAY = 80.0 + + +def render_c_major_triad( + duration_seconds: float = 3.0, + sample_rate: int = DEFAULT_SAMPLE_RATE, +) -> NDArray[np.float32]: + """Render a unit-peak C major triad as float32 PCM. + + Args: + duration_seconds: Positive fixture length in seconds. + sample_rate: Positive samples-per-second rate. + + Returns: + Mono float32 samples in ``[-1, 1]``. + + Raises: + ValueError: If duration or sample rate is not positive. + """ + if duration_seconds <= 0: + raise ValueError("duration_seconds must be positive") + if sample_rate <= 0: + raise ValueError("sample_rate must be positive") + + sample_count = int(duration_seconds * sample_rate) + times = np.arange(sample_count, dtype=np.float32) / np.float32(sample_rate) + mix = ( + np.sin(2 * np.pi * np.float32(C4_HZ) * times) + + np.sin(2 * np.pi * np.float32(E4_HZ) * times) + + np.sin(2 * np.pi * np.float32(G4_HZ) * times) + ) / np.float32(3.0) + return np.asarray(mix, dtype=np.float32) + + +def render_click_track( + bpm: float = DEFAULT_CLICK_BPM, + duration_seconds: float = 8.0, + sample_rate: int = DEFAULT_SAMPLE_RATE, +) -> NDArray[np.float32]: + """Render a click track at a known tempo. + + Args: + bpm: Positive true tempo in beats per minute. + duration_seconds: Positive fixture length in seconds. + sample_rate: Positive samples-per-second rate. + + Returns: + Mono float32 samples with a decaying click on each beat. + + Raises: + ValueError: If tempo, duration, or sample rate is not positive. + """ + if bpm <= 0 or duration_seconds <= 0 or sample_rate <= 0: + raise ValueError("bpm, duration_seconds, and sample_rate must be positive") + + sample_count = int(duration_seconds * sample_rate) + audio = np.zeros(sample_count, dtype=np.float32) + interval_seconds = 60.0 / bpm + click_length = int(_CLICK_DURATION_SECONDS * sample_rate) + if click_length > 0: + click_times = np.arange(click_length, dtype=np.float32) / np.float32(sample_rate) + click = ( + np.sin(2 * np.pi * np.float32(_CLICK_FREQUENCY_HZ) * click_times) + * np.exp(-click_times * np.float32(_CLICK_DECAY)) + ).astype(np.float32) + beat_time = 0.0 + while True: + start = int(beat_time * sample_rate) + if start >= sample_count: + break + end = min(sample_count, start + click_length) + audio[start:end] += click[: end - start] + beat_time += interval_seconds + + peak = float(np.max(np.abs(audio))) + if peak > 0: + audio /= np.float32(peak) + return audio + + +def write_pcm_wav(path: Path, audio: NDArray[np.floating], sample_rate: int) -> str: + """Write a WAV file and return the SHA-256 digest of the bytes on disk. + + Args: + path: Destination path. Parent directories are created. + audio: Mono PCM samples. + sample_rate: Samples-per-second rate used to write the file. + + Returns: + Lowercase hex SHA-256 of the written file. + + Raises: + ValueError: If the sample rate is not positive. + """ + if sample_rate <= 0: + raise ValueError("sample_rate must be positive") + path.parent.mkdir(parents=True, exist_ok=True) + sf.write(path, np.asarray(audio, dtype=np.float32), sample_rate) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def assert_fixture_checksum(path: Path, expected_sha256: str) -> None: + """Fail closed when a fixture file does not match its registered digest. + + Args: + path: Existing WAV path. + expected_sha256: Lowercase hex digest recorded in the case manifest. + + Raises: + ValueError: If the file digest does not match. + """ + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected_sha256: + raise ValueError("Accuracy fixture checksum mismatch") diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py new file mode 100644 index 000000000..29c595205 --- /dev/null +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py @@ -0,0 +1,141 @@ +"""Accuracy-case report schema for exact-value acceptance evidence.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TypedDict + +REQUIRED_REPORT_KEYS = frozenset( + { + "case_id", + "audio_sha256", + "metric_name", + "metric_value", + "passed", + "engine_version", + "true_label", + } +) + + +class AccuracyCaseReport(TypedDict): + """One scored fixture case that a buyer can read without opening logs.""" + + case_id: str + audio_sha256: str + metric_name: str + metric_value: float + passed: bool + engine_version: str + true_label: str + + +def read_product_version(start: Path | None = None) -> str: + """Return the nearest ``VERSION`` file contents, or ``unknown``. + + Args: + start: File or directory to walk upward from. Defaults to this module. + + Returns: + Stripped version text, or ``unknown`` when no non-empty file is found. + """ + current = start if start is not None else Path(__file__).resolve() + cursor = current.parent if current.is_file() else current + for parent in (cursor, *cursor.parents): + candidate = parent / "VERSION" + if not candidate.is_file(): + continue + text = candidate.read_text(encoding="utf-8").strip() + if text: + return text + return "unknown" + + +def build_case_report( + *, + case_id: str, + audio_sha256: str, + metric_name: str, + metric_value: float, + passed: bool, + true_label: str, + engine_version: str | None = None, +) -> AccuracyCaseReport: + """Build a validated accuracy case report. + + Args: + case_id: Stable fixture identifier such as ``c-major-triad``. + audio_sha256: Digest of the decoded fixture bytes. + metric_name: Registered metric id such as ``duration_weighted_chord_recall``. + metric_value: Numeric score for this run. + passed: Whether the score met the registered tolerance. + true_label: Ground-truth label the buyer should hear. + engine_version: Optional override. Defaults to the product ``VERSION``. + + Returns: + A report that ``parse_case_report`` will accept. + """ + report: AccuracyCaseReport = { + "case_id": case_id, + "audio_sha256": audio_sha256, + "metric_name": metric_name, + "metric_value": float(metric_value), + "passed": passed, + "engine_version": engine_version if engine_version is not None else read_product_version(), + "true_label": true_label, + } + return parse_case_report(report) + + +def parse_case_report(value: object) -> AccuracyCaseReport: + """Validate an accuracy case report and return a typed copy. + + Args: + value: Untrusted mapping, typically loaded from JSON. + + Returns: + The same fields after type and presence checks. + + Raises: + ValueError: If the payload is missing keys or uses the wrong types. + """ + if not isinstance(value, dict): + raise ValueError("Accuracy case report must be an object") + + missing = REQUIRED_REPORT_KEYS.difference(value) + if missing: + missing_names = ", ".join(sorted(missing)) + raise ValueError(f"Accuracy case report is missing: {missing_names}") + + case_id = value["case_id"] + audio_sha256 = value["audio_sha256"] + metric_name = value["metric_name"] + metric_value = value["metric_value"] + passed = value["passed"] + engine_version = value["engine_version"] + true_label = value["true_label"] + + if not isinstance(case_id, str) or not case_id: + raise ValueError("case_id must be a non-empty string") + if not isinstance(audio_sha256, str) or len(audio_sha256) != 64: + raise ValueError("audio_sha256 must be a 64-character hex digest") + if not isinstance(metric_name, str) or not metric_name: + raise ValueError("metric_name must be a non-empty string") + if isinstance(metric_value, bool) or not isinstance(metric_value, (int, float)): + raise ValueError("metric_value must be a number") + if not isinstance(passed, bool): + raise ValueError("passed must be a boolean") + if not isinstance(engine_version, str) or not engine_version: + raise ValueError("engine_version must be a non-empty string") + if not isinstance(true_label, str) or not true_label: + raise ValueError("true_label must be a non-empty string") + + return { + "case_id": case_id, + "audio_sha256": audio_sha256, + "metric_name": metric_name, + "metric_value": float(metric_value), + "passed": passed, + "engine_version": engine_version, + "true_label": true_label, + } diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py new file mode 100644 index 000000000..1dfaf2eb2 --- /dev/null +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py @@ -0,0 +1,70 @@ +"""Recognized MIR metrics for BandScope accuracy acceptance.""" + +from __future__ import annotations + +from collections.abc import Sequence + + +def duration_weighted_chord_recall( + segments: Sequence[tuple[float, float, str]], + expected_chord: str, + start_seconds: float, + end_seconds: float, +) -> float: + """Return the annotated-interval fraction labeled with ``expected_chord``. + + This is a single-label duration-weighted recall on one interval, the + smallest WCSR-style score that still answers “did the engine hear the + known chord for most of the fixture?” (Odekerken et al., 2021; Raffel + et al., 2014). + + Args: + segments: ``(start, end, chord)`` estimates in seconds. + expected_chord: Ground-truth chord symbol for the interval. + start_seconds: Inclusive annotation start. + end_seconds: Exclusive annotation end. Must be greater than start. + + Returns: + A value in ``[0, 1]``. + + Raises: + ValueError: If the annotation interval is empty or reversed. + """ + if end_seconds <= start_seconds: + raise ValueError("annotation end_seconds must be greater than start_seconds") + + covered = 0.0 + for segment_start, segment_end, chord in segments: + overlap_start = max(start_seconds, segment_start) + overlap_end = min(end_seconds, segment_end) + if overlap_end > overlap_start and chord == expected_chord: + covered += overlap_end - overlap_start + return covered / (end_seconds - start_seconds) + + +def tempo_acc1( + estimated_bpm: float, + true_bpm: float, + relative_tolerance: float = 0.04, +) -> bool: + """Return whether estimated tempo is within Acc1 tolerance of the true tempo. + + Acc1 accepts an estimate within ``relative_tolerance`` of the true BPM and + does not credit octave errors (Schreiber & Müller, 2020). + + Args: + estimated_bpm: Engine tempo in beats per minute. + true_bpm: Known fixture tempo. Must be positive. + relative_tolerance: Non-negative Acc1 window. The default is 4%. + + Returns: + ``True`` when the estimate is inside the Acc1 window. + + Raises: + ValueError: If ``true_bpm`` is not positive or the tolerance is negative. + """ + if true_bpm <= 0: + raise ValueError("true_bpm must be positive") + if relative_tolerance < 0: + raise ValueError("relative_tolerance must be non-negative") + return abs(estimated_bpm - true_bpm) / true_bpm <= relative_tolerance diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py new file mode 100644 index 000000000..31acddba0 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -0,0 +1,205 @@ +"""Real-audio accuracy acceptance for decoded PCM fixtures. + +These cases prove a buyer-visible claim: a known waveform written to disk, +decoded, and analyzed yields the expected chord or tempo. Mocked chroma +matrices are not acceptance evidence. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from bandscope_analysis.accuracy import ( + C_MAJOR_LABEL, + DEFAULT_CLICK_BPM, + DEFAULT_SAMPLE_RATE, + assert_fixture_checksum, + build_case_report, + duration_weighted_chord_recall, + evaluate_c_major_pcm, + evaluate_click_tempo_file, + parse_case_report, + read_product_version, + render_c_major_triad, + render_click_track, + tempo_acc1, + write_pcm_wav, +) +from bandscope_analysis.accuracy.evaluate import C_MAJOR_RECALL_FLOOR +from bandscope_analysis.api import build_demo_rehearsal_song + + +def test_c_major_fixture_is_deterministic(tmp_path: Path) -> None: + """Two writes of the same triad must share one SHA-256 digest.""" + audio = render_c_major_triad() + first = write_pcm_wav(tmp_path / "a.wav", audio, DEFAULT_SAMPLE_RATE) + second = write_pcm_wav(tmp_path / "b.wav", audio, DEFAULT_SAMPLE_RATE) + assert first == second + assert len(first) == 64 + + +def test_c_major_wav_recovers_c_after_file_decode(tmp_path: Path) -> None: + """A decoded C major WAV must recover C for most of the fixture duration.""" + audio = render_c_major_triad(duration_seconds=3.0) + digest = write_pcm_wav(tmp_path / "c-major.wav", audio, DEFAULT_SAMPLE_RATE) + assert_fixture_checksum(tmp_path / "c-major.wav", digest) + report = evaluate_c_major_pcm(audio, DEFAULT_SAMPLE_RATE, digest) + assert report["true_label"] == C_MAJOR_LABEL + assert report["metric_name"] == "duration_weighted_chord_recall" + assert report["metric_value"] >= C_MAJOR_RECALL_FLOOR + assert report["passed"] is True + assert report["audio_sha256"] == digest + + +def test_click_wav_recovers_120_bpm_acc1(tmp_path: Path) -> None: + """A decoded 120 BPM click WAV must pass tempo Acc1.""" + audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) + path = tmp_path / "click-120.wav" + digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + report = evaluate_click_tempo_file(path, digest, DEFAULT_CLICK_BPM) + assert report["passed"] is True + assert report["metric_name"] == "tempo_acc1" + assert report["true_label"] == "120 bpm" + + +def test_silence_does_not_pass_c_major_recall() -> None: + """Silence must not be reported as a passing C major acceptance case.""" + silence = np.zeros(DEFAULT_SAMPLE_RATE, dtype=np.float32) + report = evaluate_c_major_pcm(silence, DEFAULT_SAMPLE_RATE, "b" * 64) + assert report["passed"] is False + assert report["metric_value"] < C_MAJOR_RECALL_FLOOR + + +def test_click_tempo_acc1_fails_when_true_tempo_is_wrong(tmp_path: Path) -> None: + """Acc1 must fail when the registered true tempo is not the click tempo.""" + audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) + path = tmp_path / "click-wrong-label.wav" + digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + report = evaluate_click_tempo_file(path, digest, true_bpm=40.0) + assert report["passed"] is False + assert report["metric_value"] == 0.0 + + +def test_checksum_mismatch_fails_closed(tmp_path: Path) -> None: + """A tampered fixture must not be scored as a passing case.""" + audio = render_click_track() + path = tmp_path / "click.wav" + write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + with pytest.raises(ValueError, match="checksum mismatch"): + assert_fixture_checksum(path, "0" * 64) + + +def test_pipeline_surfaces_c_on_active_lead_vocal() -> None: + """Unmocked assembly must put measured C on lead vocal when that stem is active.""" + audio = render_c_major_triad(duration_seconds=3.0) + silence = np.zeros_like(audio) + song = build_demo_rehearsal_song( + { + "stems": { + "vocals": audio * np.float32(0.35), + "bass": silence, + "drums": silence, + "other": audio, + }, + "sr": DEFAULT_SAMPLE_RATE, + "separation": {"duration_seconds": 3.0, "chunk_count": 1, "notes": "accuracy"}, + } + ) + assert song["id"] == "analyzed-song" + lead_chords = [ + role["harmony"]["chord"] + for section in song["sections"] + for role in section["roles"] + if role["id"] == "lead-vocal" + ] + assert C_MAJOR_LABEL in lead_chords + + +def test_render_helpers_reject_non_positive_inputs() -> None: + """Fixture helpers must refuse empty or reversed generation parameters.""" + with pytest.raises(ValueError, match="duration_seconds"): + render_c_major_triad(duration_seconds=0) + with pytest.raises(ValueError, match="sample_rate"): + render_c_major_triad(sample_rate=0) + with pytest.raises(ValueError, match="must be positive"): + render_click_track(bpm=0) + with pytest.raises(ValueError, match="sample_rate"): + write_pcm_wav(Path("unused.wav"), np.zeros(4, dtype=np.float32), 0) + + +def test_click_track_with_zero_length_click_stays_silent() -> None: + """A 1 Hz sample rate makes the click window empty and leaves silence.""" + audio = render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) + assert audio.shape == (1,) + assert float(audio[0]) == 0.0 + + +def test_duration_weighted_recall_covers_overlap_and_misses() -> None: + """Recall must count only overlapping time that matches the expected chord.""" + assert duration_weighted_chord_recall([(0.0, 2.0, "C"), (2.0, 4.0, "G")], "C", 0.0, 4.0) == 0.5 + assert duration_weighted_chord_recall([(5.0, 6.0, "C")], "C", 0.0, 2.0) == 0.0 + with pytest.raises(ValueError, match="end_seconds"): + duration_weighted_chord_recall([], "C", 1.0, 1.0) + + +def test_tempo_acc1_window_and_guards() -> None: + """Acc1 must accept a 4% window and reject octave errors and bad inputs.""" + assert tempo_acc1(120.0, 120.0) is True + assert tempo_acc1(124.8, 120.0) is True + assert tempo_acc1(240.0, 120.0) is False + with pytest.raises(ValueError, match="true_bpm"): + tempo_acc1(120.0, 0.0) + with pytest.raises(ValueError, match="relative_tolerance"): + tempo_acc1(120.0, 120.0, relative_tolerance=-0.01) + + +def test_parse_case_report_rejects_malformed_payloads() -> None: + """Manifest parsing must fail closed on missing or mistyped fields.""" + valid = build_case_report( + case_id="c-major-triad", + audio_sha256="a" * 64, + metric_name="duration_weighted_chord_recall", + metric_value=0.9, + passed=True, + true_label="C", + engine_version="0.1.3", + ) + assert parse_case_report(valid)["passed"] is True + + with pytest.raises(ValueError, match="must be an object"): + parse_case_report(["not", "an", "object"]) + with pytest.raises(ValueError, match="missing"): + parse_case_report({"case_id": "only"}) + with pytest.raises(ValueError, match="case_id"): + parse_case_report({**valid, "case_id": ""}) + with pytest.raises(ValueError, match="audio_sha256"): + parse_case_report({**valid, "audio_sha256": "short"}) + with pytest.raises(ValueError, match="metric_name"): + parse_case_report({**valid, "metric_name": ""}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": True}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": "0.9"}) + with pytest.raises(ValueError, match="passed"): + parse_case_report({**valid, "passed": 1}) + with pytest.raises(ValueError, match="engine_version"): + parse_case_report({**valid, "engine_version": ""}) + with pytest.raises(ValueError, match="true_label"): + parse_case_report({**valid, "true_label": ""}) + + +def test_read_product_version_uses_version_file_or_unknown(tmp_path: Path) -> None: + """Version lookup must read VERSION and fall back to unknown.""" + versioned = tmp_path / "versioned" + versioned.mkdir() + (versioned / "VERSION").write_text("9.9.9\n", encoding="utf-8") + assert read_product_version(versioned) == "9.9.9" + assert read_product_version(versioned / "VERSION") == "9.9.9" + empty = tmp_path / "empty-tree" + empty.mkdir() + (empty / "VERSION").write_text(" \n", encoding="utf-8") + assert read_product_version(empty) == "unknown" + assert read_product_version() != "unknown" From 11e156d7de6d50421268e24b1bf9d5439d1ed203 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 16:57:30 +0000 Subject: [PATCH 02/78] fix(analysis): score C major acceptance from decoded WAV bytes The headline Tier 1 case wrote a fixture then scored the pre-write array. Checksum the file, decode those bytes, and fail closed through both file evaluators so a tampered WAV cannot pass. Co-authored-by: Seongho Bae --- CHANGELOG.md | 4 ++ .../real-audio-accuracy-acceptance.md | 22 +++++--- ...26-08-16-real-audio-accuracy-acceptance.md | 13 ++++- .../bandscope_analysis/accuracy/__init__.py | 5 +- .../bandscope_analysis/accuracy/evaluate.py | 16 ++++++ .../bandscope_analysis/accuracy/fixtures.py | 21 +++++++ .../tests/test_accuracy_acceptance.py | 56 ++++++++++++++++--- 7 files changed, 117 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 599d43428..578753b56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ ### Added - Add Tier 1 real-audio accuracy acceptance: a decoded C major WAV must recover `C`, and a 120 BPM click WAV must pass tempo Acc1, with checksum-fail-closed reports. + +### Fixed + +- Score the C major acceptance case from checksummed on-disk WAV bytes instead of the pre-write in-memory triad. - Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace. - 각 합주 역할(Role)별 개인 연습 진행도를 0~100% 범위로 기록 및 시각화할 수 있는 연습 진척도(`practiceProgress`) 트래커 기능 추가. UI 컨트롤(슬라이더 및 +/- 버튼)과 한/영 다국어 지원 포함. diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 0a96f9840..e10de8071 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -21,13 +21,16 @@ stem quality, or private-corpus readiness. ## Held cases -- `c-major-triad`: three seconds of C4+E4+G4 written to WAV, decoded, and - scored with duration-weighted chord recall. Pass when recall of `C` is at - least `0.70` (Odekerken et al., 2021). +- `c-major-triad`: three seconds of C4+E4+G4 written to WAV, checksummed, + decoded from those bytes, and scored with duration-weighted chord recall. + Pass when recall of `C` is at least `0.70`. That floor is a BandScope Tier 1 + tolerance. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et + al., 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. -- Checksum mismatch fails closed. Do not score a tampered file as a pass. +- Checksum mismatch fails closed on both file evaluators. Do not score a + tampered file as a pass. ## Claim boundary @@ -61,7 +64,10 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? passed into `ChordRecognizer` and `TemporalAnalyzer`. - Trust boundary: untrusted audio and manifests; trusted repo-controlled fixture generators and metric floors. -- Mitigations: no network, no shell, checksum fail-closed, bounded fixture - durations, no copyrighted commercial recordings. -- Test points: deterministic digest, C major recall, 120 BPM Acc1, checksum - mismatch, malformed manifest, silence must not pass as C major. +- Mitigations: no network, no shell, checksum fail-closed before C-major + decode and before tempo scoring, bounded fixture durations, no copyrighted + commercial recordings. Fixture paths are pytest temp files; reports store + only SHA-256 and labels, not waveform bytes. +- Test points: deterministic digest, C major recall after file decode, + silence-on-disk vs in-memory triad, 120 BPM Acc1, checksum mismatch through + both file evaluators, malformed manifest, silence must not pass as C major. diff --git a/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md b/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md index 9a33f0e91..94335b2ff 100644 --- a/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md +++ b/docs/plans/2026-08-16-real-audio-accuracy-acceptance.md @@ -26,16 +26,17 @@ Trusted: in-repo generators, metric definitions, and registered floors. ### Mitigations - No network and no shell interpolation. -- Checksum mismatch raises before tempo scoring. +- Checksum mismatch raises before C-major decode and before tempo scoring. - Manifest parsing fails closed on missing or mistyped fields. - Fixtures are short, synthetic, and license-clean. ### Test points - Deterministic C major digest -- Duration-weighted C recall after file write +- Duration-weighted C recall after file decode +- Silence on disk fails even when a C major array exists in memory - 120 BPM Acc1 after file decode -- Checksum mismatch +- Checksum mismatch through both file evaluators - Malformed report rejection - Silence must not pass as C major @@ -44,6 +45,12 @@ Trusted: in-repo generators, metric definitions, and registered floors. A tampered fixture or a missing digest could be scored as a pass and then cited as release evidence. +### Logging and privacy + +`TemporalAnalyzer` may log the fixture path. Tests use pytest `tmp_path` +names only. Case reports keep SHA-256, metric, and labels; they do not store +PCM. + ### Remaining risk Tier 1 does not cover Demucs stems, private commercial recordings, or diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py b/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py index 1c451394a..03b4f5b57 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/__init__.py @@ -5,12 +5,13 @@ do not replace Demucs stem separation or private-corpus benchmarks. """ -from .evaluate import evaluate_c_major_pcm, evaluate_click_tempo_file +from .evaluate import evaluate_c_major_file, evaluate_c_major_pcm, evaluate_click_tempo_file from .fixtures import ( C_MAJOR_LABEL, DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, assert_fixture_checksum, + read_pcm_wav, render_c_major_triad, render_click_track, write_pcm_wav, @@ -31,9 +32,11 @@ "assert_fixture_checksum", "build_case_report", "duration_weighted_chord_recall", + "evaluate_c_major_file", "evaluate_c_major_pcm", "evaluate_click_tempo_file", "parse_case_report", + "read_pcm_wav", "read_product_version", "render_c_major_triad", "render_click_track", diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py index 2fe1c3f0a..8df7d414f 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py @@ -12,6 +12,7 @@ DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, assert_fixture_checksum, + read_pcm_wav, ) from bandscope_analysis.accuracy.manifest import AccuracyCaseReport, build_case_report from bandscope_analysis.accuracy.metrics import duration_weighted_chord_recall, tempo_acc1 @@ -51,6 +52,21 @@ def evaluate_c_major_pcm( ) +def evaluate_c_major_file(audio_path: Path, expected_sha256: str) -> AccuracyCaseReport: + """Checksum, decode, and score a C major WAV through ChordRecognizer. + + Args: + audio_path: On-disk WAV written by ``write_pcm_wav``. + expected_sha256: Registered digest. Mismatch fails closed before decode. + + Returns: + A case report whose metric is duration-weighted recall of ``C``. + """ + assert_fixture_checksum(audio_path, expected_sha256) + audio, sample_rate = read_pcm_wav(audio_path) + return evaluate_c_major_pcm(audio, sample_rate, expected_sha256) + + def evaluate_click_tempo_file( audio_path: Path, expected_sha256: str, diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index a1b794f81..cdcabc5fd 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -118,6 +118,27 @@ def write_pcm_wav(path: Path, audio: NDArray[np.floating], sample_rate: int) -> return hashlib.sha256(path.read_bytes()).hexdigest() +def read_pcm_wav(path: Path) -> tuple[NDArray[np.float32], int]: + """Decode a WAV file to mono float32 PCM. + + Args: + path: Existing WAV path written by ``write_pcm_wav``. + + Returns: + A tuple of mono samples and the file sample rate. + + Raises: + ValueError: If the file has no samples after decode. + """ + audio, sample_rate = sf.read(path, dtype="float32", always_2d=False) + samples = np.asarray(audio, dtype=np.float32) + if samples.ndim > 1: + samples = np.mean(samples, axis=1).astype(np.float32) + if samples.size == 0: + raise ValueError("decoded WAV has no samples") + return samples, int(sample_rate) + + def assert_fixture_checksum(path: Path, expected_sha256: str) -> None: """Fail closed when a fixture file does not match its registered digest. diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 31acddba0..8c6999605 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -11,17 +11,19 @@ import numpy as np import pytest +import soundfile as sf from bandscope_analysis.accuracy import ( C_MAJOR_LABEL, DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, - assert_fixture_checksum, build_case_report, duration_weighted_chord_recall, + evaluate_c_major_file, evaluate_c_major_pcm, evaluate_click_tempo_file, parse_case_report, + read_pcm_wav, read_product_version, render_c_major_triad, render_click_track, @@ -44,9 +46,9 @@ def test_c_major_fixture_is_deterministic(tmp_path: Path) -> None: def test_c_major_wav_recovers_c_after_file_decode(tmp_path: Path) -> None: """A decoded C major WAV must recover C for most of the fixture duration.""" audio = render_c_major_triad(duration_seconds=3.0) - digest = write_pcm_wav(tmp_path / "c-major.wav", audio, DEFAULT_SAMPLE_RATE) - assert_fixture_checksum(tmp_path / "c-major.wav", digest) - report = evaluate_c_major_pcm(audio, DEFAULT_SAMPLE_RATE, digest) + path = tmp_path / "c-major.wav" + digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + report = evaluate_c_major_file(path, digest) assert report["true_label"] == C_MAJOR_LABEL assert report["metric_name"] == "duration_weighted_chord_recall" assert report["metric_value"] >= C_MAJOR_RECALL_FLOOR @@ -85,11 +87,49 @@ def test_click_tempo_acc1_fails_when_true_tempo_is_wrong(tmp_path: Path) -> None def test_checksum_mismatch_fails_closed(tmp_path: Path) -> None: """A tampered fixture must not be scored as a passing case.""" - audio = render_click_track() - path = tmp_path / "click.wav" - write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + click = render_click_track() + click_path = tmp_path / "click.wav" + write_pcm_wav(click_path, click, DEFAULT_SAMPLE_RATE) + with pytest.raises(ValueError, match="checksum mismatch"): + evaluate_click_tempo_file(click_path, "0" * 64) + + triad = render_c_major_triad() + triad_path = tmp_path / "c-major.wav" + write_pcm_wav(triad_path, triad, DEFAULT_SAMPLE_RATE) with pytest.raises(ValueError, match="checksum mismatch"): - assert_fixture_checksum(path, "0" * 64) + evaluate_c_major_file(triad_path, "0" * 64) + + +def test_c_major_file_decode_scores_disk_not_memory(tmp_path: Path) -> None: + """Silence on disk must fail even when a C major array exists in memory.""" + triad = render_c_major_triad(duration_seconds=3.0) + silence = np.zeros_like(triad) + path = tmp_path / "silence.wav" + digest = write_pcm_wav(path, silence, DEFAULT_SAMPLE_RATE) + report = evaluate_c_major_file(path, digest) + assert report["passed"] is False + assert report["metric_value"] < C_MAJOR_RECALL_FLOOR + memory_report = evaluate_c_major_pcm(triad, DEFAULT_SAMPLE_RATE, digest) + assert memory_report["passed"] is True + + +def test_read_pcm_wav_mixes_stereo_to_mono(tmp_path: Path) -> None: + """A stereo fixture must collapse to mono before scoring.""" + path = tmp_path / "stereo.wav" + stereo = np.column_stack([np.ones(8, dtype=np.float32), np.zeros(8, dtype=np.float32)]) + sf.write(path, stereo, DEFAULT_SAMPLE_RATE) + audio, sample_rate = read_pcm_wav(path) + assert sample_rate == DEFAULT_SAMPLE_RATE + assert audio.shape == (8,) + assert np.allclose(audio, 0.5, atol=1e-3) + + +def test_read_pcm_wav_rejects_empty_file(tmp_path: Path) -> None: + """An empty WAV must fail closed instead of scoring as a pass.""" + path = tmp_path / "empty.wav" + sf.write(path, np.zeros(0, dtype=np.float32), DEFAULT_SAMPLE_RATE) + with pytest.raises(ValueError, match="no samples"): + read_pcm_wav(path) def test_pipeline_surfaces_c_on_active_lead_vocal() -> None: From d4282bafed4ada030b41845d5ae431cb7b2a81a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:02:54 +0900 Subject: [PATCH 03/78] test(accuracy): reject malformed report provenance --- services/analysis-engine/tests/test_accuracy_acceptance.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 8c6999605..0b13d0278 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -217,12 +217,18 @@ def test_parse_case_report_rejects_malformed_payloads() -> None: parse_case_report({**valid, "case_id": ""}) with pytest.raises(ValueError, match="audio_sha256"): parse_case_report({**valid, "audio_sha256": "short"}) + with pytest.raises(ValueError, match="audio_sha256"): + parse_case_report({**valid, "audio_sha256": "g" * 64}) with pytest.raises(ValueError, match="metric_name"): parse_case_report({**valid, "metric_name": ""}) with pytest.raises(ValueError, match="metric_value"): parse_case_report({**valid, "metric_value": True}) with pytest.raises(ValueError, match="metric_value"): parse_case_report({**valid, "metric_value": "0.9"}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": np.nan}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": np.inf}) with pytest.raises(ValueError, match="passed"): parse_case_report({**valid, "passed": 1}) with pytest.raises(ValueError, match="engine_version"): From f3ac87be6e305d277bde80992a31f2bd5080bf35 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:05:06 +0900 Subject: [PATCH 04/78] fix(accuracy): validate report provenance values --- .../bandscope_analysis/accuracy/manifest.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py index 29c595205..250cc42a2 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from pathlib import Path from typing import TypedDict @@ -16,6 +17,7 @@ "true_label", } ) +_HEX_DIGITS = frozenset("0123456789abcdefABCDEF") class AccuracyCaseReport(TypedDict): @@ -97,7 +99,7 @@ def parse_case_report(value: object) -> AccuracyCaseReport: The same fields after type and presence checks. Raises: - ValueError: If the payload is missing keys or uses the wrong types. + ValueError: If the payload is missing keys or uses invalid values. """ if not isinstance(value, dict): raise ValueError("Accuracy case report must be an object") @@ -117,12 +119,22 @@ def parse_case_report(value: object) -> AccuracyCaseReport: if not isinstance(case_id, str) or not case_id: raise ValueError("case_id must be a non-empty string") - if not isinstance(audio_sha256, str) or len(audio_sha256) != 64: + if ( + not isinstance(audio_sha256, str) + or len(audio_sha256) != 64 + or any(character not in _HEX_DIGITS for character in audio_sha256) + ): raise ValueError("audio_sha256 must be a 64-character hex digest") if not isinstance(metric_name, str) or not metric_name: raise ValueError("metric_name must be a non-empty string") if isinstance(metric_value, bool) or not isinstance(metric_value, (int, float)): - raise ValueError("metric_value must be a number") + raise ValueError("metric_value must be a finite number") + try: + normalized_metric_value = float(metric_value) + except OverflowError as error: + raise ValueError("metric_value must be a finite number") from error + if not math.isfinite(normalized_metric_value): + raise ValueError("metric_value must be a finite number") if not isinstance(passed, bool): raise ValueError("passed must be a boolean") if not isinstance(engine_version, str) or not engine_version: @@ -134,7 +146,7 @@ def parse_case_report(value: object) -> AccuracyCaseReport: "case_id": case_id, "audio_sha256": audio_sha256, "metric_name": metric_name, - "metric_value": float(metric_value), + "metric_value": normalized_metric_value, "passed": passed, "engine_version": engine_version, "true_label": true_label, From 1f375b21b27b2787c578ace998f900e78815f13e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:11:36 +0900 Subject: [PATCH 05/78] docs(accuracy): record provenance validation hardening --- CHANGELOG.md | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 578753b56..a186a1c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject malformed accuracy-report provenance, including non-hex SHA-256 text and non-finite metric values, before acceptance evidence is consumed. - Score the C major acceptance case from checksummed on-disk WAV bytes instead of the pre-write in-memory triad. - Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace. - 각 합주 역할(Role)별 개인 연습 진행도를 0~100% 범위로 기록 및 시각화할 수 있는 연습 진척도(`practiceProgress`) 트래커 기능 추가. UI 컨트롤(슬라이더 및 +/- 버튼)과 한/영 다국어 지원 포함. @@ -52,22 +53,4 @@ - Issue #29: Defined core `song -> section -> role` rehearsal domain contracts - Issue #38: Added cross-architecture build support (Windows/macOS arm64+amd64) -- Issue #40: Enforced 100% Python docstring and test coverage -- Issue #32: Implemented local analysis orchestration and secure IPC boundaries -- Issue #33: Implemented secure local audio intake and project bootstrap -- Issue #35: Engineered section, form, and cue anchor extraction pipeline -- Issue #34: Implemented role extraction targets and part graph -- Issue #31: Added role-specific harmony, range, overlap, and confidence metrics -- Issue #28: Delivered practical rehearsal workspace UI -- Issue #27: Supported manual overrides, provenance tracking, and local project persistence -- Issue #36: Implemented rehearsal priority calculation and cue-sheet (CSV) / chart (JSON) exports -- Issue #30: Added policy-constrained YouTube import with local fallback -- Issue #26: Finalized roadmap and prepared application for initial release - -## [0.1.4] - 2026-05-15 - -### 추가됨 (Added) - -- `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. -- `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. -- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). +- Issue #40: Enforced 100% Python docstring and test coverage \ No newline at end of file From 2da68dc4a30606822a226e4bd556354733a2ac27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:12:00 +0900 Subject: [PATCH 06/78] fix(changelog): preserve release history while recording provenance hardening --- CHANGELOG.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a186a1c82..aade725cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,4 +53,22 @@ - Issue #29: Defined core `song -> section -> role` rehearsal domain contracts - Issue #38: Added cross-architecture build support (Windows/macOS arm64+amd64) -- Issue #40: Enforced 100% Python docstring and test coverage \ No newline at end of file +- Issue #40: Enforced 100% Python docstring and test coverage +- Issue #32: Implemented local analysis orchestration and secure IPC boundaries +- Issue #33: Implemented secure local audio intake and project bootstrap +- Issue #35: Engineered section, form, and cue anchor extraction pipeline +- Issue #34: Implemented role extraction targets and part graph +- Issue #31: Added role-specific harmony, range, overlap, and confidence metrics +- Issue #28: Delivered practical rehearsal workspace UI +- Issue #27: Supported manual overrides, provenance tracking, and local project persistence +- Issue #36: Implemented rehearsal priority calculation and cue-sheet (CSV) / chart (JSON) exports +- Issue #30: Added policy-constrained YouTube import with local fallback +- Issue #26: Finalized roadmap and prepared application for initial release + +## [0.1.4] - 2026-05-15 + +### 추가됨 (Added) + +- `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. +- `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. +- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). From b778af6a41ce3a7bfa210e8e57d581b5881cec06 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:12:27 +0900 Subject: [PATCH 07/78] docs(accuracy): document report provenance validation --- .../real-audio-accuracy-acceptance.md | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index e10de8071..ec8290c59 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -31,6 +31,11 @@ stem quality, or private-corpus readiness. (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. - Checksum mismatch fails closed on both file evaluators. Do not score a tampered file as a pass. +- Machine-readable case reports are accepted only when the registered + provenance fields are present and typed, `audio_sha256` is exactly 64 + hexadecimal characters, and `metric_value` is a finite numeric value. + Boolean, NaN, infinity, overflow-to-float, malformed digest, or missing-field + evidence fails closed rather than becoming a portable acceptance record. ## Claim boundary @@ -60,14 +65,16 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? ## Security Notes -- Attack surface: generated WAV bytes, SHA-256 digests, and decoded PCM - passed into `ChordRecognizer` and `TemporalAnalyzer`. +- Attack surface: generated WAV bytes, SHA-256 digests, decoded PCM, and parsed + case-report mappings passed into the accuracy acceptance path. - Trust boundary: untrusted audio and manifests; trusted repo-controlled - fixture generators and metric floors. + fixture generators, metric definitions, and registered floors. - Mitigations: no network, no shell, checksum fail-closed before C-major - decode and before tempo scoring, bounded fixture durations, no copyrighted - commercial recordings. Fixture paths are pytest temp files; reports store - only SHA-256 and labels, not waveform bytes. + decode and before tempo scoring, strict SHA-256 syntax, finite-only metric + values, bounded fixture durations, no copyrighted commercial recordings. + Fixture paths are pytest temp files; reports store only SHA-256 and labels, + not waveform bytes. - Test points: deterministic digest, C major recall after file decode, silence-on-disk vs in-memory triad, 120 BPM Acc1, checksum mismatch through - both file evaluators, malformed manifest, silence must not pass as C major. + both file evaluators, malformed/non-hex manifest provenance, NaN/infinity + rejection, and silence must not pass as C major. From 0f6386bd1c9627cd108eace62fb8da1bb9263be7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:29:29 +0900 Subject: [PATCH 08/78] test(accuracy): prevent overlap double counting --- .../analysis-engine/tests/test_accuracy_acceptance.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 0b13d0278..b7807bc96 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -185,6 +185,17 @@ def test_duration_weighted_recall_covers_overlap_and_misses() -> None: duration_weighted_chord_recall([], "C", 1.0, 1.0) +def test_duration_weighted_recall_unions_overlapping_matching_estimates() -> None: + """Overlapping matching estimates must not count annotation time twice.""" + recall = duration_weighted_chord_recall( + [(0.0, 2.0, "C"), (1.0, 3.0, "C"), (4.0, 5.0, "C")], + "C", + 0.0, + 5.0, + ) + assert recall == pytest.approx(0.8) + + def test_tempo_acc1_window_and_guards() -> None: """Acc1 must accept a 4% window and reject octave errors and bad inputs.""" assert tempo_acc1(120.0, 120.0) is True From 9cd0c767b19d2e65e358a73311053766743cfa29 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:30:49 +0900 Subject: [PATCH 09/78] fix(accuracy): union overlapping chord estimates --- .../bandscope_analysis/accuracy/metrics.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py index 1dfaf2eb2..96b105298 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py @@ -16,7 +16,9 @@ def duration_weighted_chord_recall( This is a single-label duration-weighted recall on one interval, the smallest WCSR-style score that still answers “did the engine hear the known chord for most of the fixture?” (Odekerken et al., 2021; Raffel - et al., 2014). + et al., 2014). Matching estimate intervals are unioned after clipping to + the annotation window, so overlapping estimates cannot count the same + annotated time more than once. Args: segments: ``(start, end, chord)`` estimates in seconds. @@ -33,12 +35,26 @@ def duration_weighted_chord_recall( if end_seconds <= start_seconds: raise ValueError("annotation end_seconds must be greater than start_seconds") - covered = 0.0 + matching_intervals: list[tuple[float, float]] = [] for segment_start, segment_end, chord in segments: overlap_start = max(start_seconds, segment_start) overlap_end = min(end_seconds, segment_end) if overlap_end > overlap_start and chord == expected_chord: - covered += overlap_end - overlap_start + matching_intervals.append((overlap_start, overlap_end)) + + if not matching_intervals: + return 0.0 + + matching_intervals.sort(key=lambda interval: (interval[0], interval[1])) + current_start, current_end = matching_intervals[0] + covered = 0.0 + for interval_start, interval_end in matching_intervals[1:]: + if interval_start <= current_end: + current_end = max(current_end, interval_end) + continue + covered += current_end - current_start + current_start, current_end = interval_start, interval_end + covered += current_end - current_start return covered / (end_seconds - start_seconds) From 82b275ca14ea339c1d0b3fc670fba4af17c86688 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:32:46 +0900 Subject: [PATCH 10/78] docs(changelog): record bounded chord recall --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aade725cb..eb3800a5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Union overlapping matching chord-estimate intervals before duration-weighted recall so acceptance scores cannot double-count annotated time or exceed 100%. - Reject malformed accuracy-report provenance, including non-hex SHA-256 text and non-finite metric values, before acceptance evidence is consumed. - Score the C major acceptance case from checksummed on-disk WAV bytes instead of the pre-write in-memory triad. - Display the analyzed song tempo (BPM) as a badge in the rehearsal workspace. From 05a6ce3e7d93e031c4b6afdb7e3adf36c0c4ef8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:33:06 +0900 Subject: [PATCH 11/78] docs(accuracy): define overlap-safe recall --- docs/doctoring/real-audio-accuracy-acceptance.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index ec8290c59..9b2d01131 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -24,8 +24,11 @@ stem quality, or private-corpus readiness. - `c-major-triad`: three seconds of C4+E4+G4 written to WAV, checksummed, decoded from those bytes, and scored with duration-weighted chord recall. Pass when recall of `C` is at least `0.70`. That floor is a BandScope Tier 1 - tolerance. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et - al., 2014). + tolerance. Matching estimate intervals are clipped to the annotation window + and unioned before duration is accumulated, so overlapping or duplicate + estimates cannot count the same annotated time twice or produce recall above + `1.0`. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et al., + 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. @@ -75,6 +78,7 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? Fixture paths are pytest temp files; reports store only SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, + overlapping matching intervals do not double-count annotation duration, silence-on-disk vs in-memory triad, 120 BPM Acc1, checksum mismatch through both file evaluators, malformed/non-hex manifest provenance, NaN/infinity rejection, and silence must not pass as C major. From 5be801e914bcec4b1a1b28bb539c41513bb7cbf7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:38:59 +0900 Subject: [PATCH 12/78] test(accuracy): reject non-finite tempo evidence --- .../tests/test_accuracy_acceptance.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index b7807bc96..388b15aea 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -207,6 +207,28 @@ def test_tempo_acc1_window_and_guards() -> None: tempo_acc1(120.0, 120.0, relative_tolerance=-0.01) +@pytest.mark.parametrize( + ("estimated_bpm", "true_bpm", "relative_tolerance", "message"), + [ + (np.nan, 120.0, 0.04, "estimated_bpm"), + (np.inf, 120.0, 0.04, "estimated_bpm"), + (120.0, np.nan, 0.04, "true_bpm"), + (120.0, np.inf, 0.04, "true_bpm"), + (120.0, 120.0, np.nan, "relative_tolerance"), + (120.0, 120.0, np.inf, "relative_tolerance"), + ], +) +def test_tempo_acc1_rejects_non_finite_evidence( + estimated_bpm: float, + true_bpm: float, + relative_tolerance: float, + message: str, +) -> None: + """Non-finite estimate, truth, or tolerance must fail closed.""" + with pytest.raises(ValueError, match=message): + tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) + + def test_parse_case_report_rejects_malformed_payloads() -> None: """Manifest parsing must fail closed on missing or mistyped fields.""" valid = build_case_report( From 0b4e73bc174885e736828177db927266d8494579 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:39:16 +0900 Subject: [PATCH 13/78] fix(accuracy): reject non-finite tempo evidence --- .../bandscope_analysis/accuracy/metrics.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py index 96b105298..0dcd3b68d 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from collections.abc import Sequence @@ -66,21 +67,26 @@ def tempo_acc1( """Return whether estimated tempo is within Acc1 tolerance of the true tempo. Acc1 accepts an estimate within ``relative_tolerance`` of the true BPM and - does not credit octave errors (Schreiber & Müller, 2020). + does not credit octave errors (Schreiber & Müller, 2020). Non-finite + estimates, ground truth, or tolerances are invalid evidence and fail + closed instead of being converted into an ordinary metric miss. Args: - estimated_bpm: Engine tempo in beats per minute. - true_bpm: Known fixture tempo. Must be positive. - relative_tolerance: Non-negative Acc1 window. The default is 4%. + estimated_bpm: Engine tempo in beats per minute. Must be finite. + true_bpm: Known fixture tempo. Must be finite and positive. + relative_tolerance: Finite non-negative Acc1 window. The default is 4%. Returns: ``True`` when the estimate is inside the Acc1 window. Raises: - ValueError: If ``true_bpm`` is not positive or the tolerance is negative. + ValueError: If any metric input is non-finite, ``true_bpm`` is not + positive, or the tolerance is negative. """ - if true_bpm <= 0: - raise ValueError("true_bpm must be positive") - if relative_tolerance < 0: - raise ValueError("relative_tolerance must be non-negative") + if not math.isfinite(estimated_bpm): + raise ValueError("estimated_bpm must be finite") + if not math.isfinite(true_bpm) or true_bpm <= 0: + raise ValueError("true_bpm must be finite and positive") + if not math.isfinite(relative_tolerance) or relative_tolerance < 0: + raise ValueError("relative_tolerance must be finite and non-negative") return abs(estimated_bpm - true_bpm) / true_bpm <= relative_tolerance From f1d039ae3dc944dd510eeba3fb2d4abdb09d110d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:39:34 +0900 Subject: [PATCH 14/78] docs(accuracy): fail closed on non-finite tempo evidence --- .../real-audio-accuracy-acceptance.md | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 9b2d01131..e6bf02b9f 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -32,6 +32,9 @@ stem quality, or private-corpus readiness. - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. + Estimated BPM, true BPM, and the tolerance must all be finite; non-finite + metric inputs are invalid acceptance evidence and fail closed rather than + being recorded as an ordinary miss. - Checksum mismatch fails closed on both file evaluators. Do not score a tampered file as a pass. - Machine-readable case reports are accepted only when the registered @@ -68,17 +71,21 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? ## Security Notes -- Attack surface: generated WAV bytes, SHA-256 digests, decoded PCM, and parsed - case-report mappings passed into the accuracy acceptance path. -- Trust boundary: untrusted audio and manifests; trusted repo-controlled - fixture generators, metric definitions, and registered floors. +- Attack surface: generated WAV bytes, SHA-256 digests, decoded PCM, recognizer + segment timings, tempo estimates, and parsed case-report mappings passed into + the accuracy acceptance path. +- Trust boundary: untrusted audio, recognizer output, and manifests; trusted + repo-controlled fixture generators, true labels, metric definitions, and + registered floors. - Mitigations: no network, no shell, checksum fail-closed before C-major - decode and before tempo scoring, strict SHA-256 syntax, finite-only metric - values, bounded fixture durations, no copyrighted commercial recordings. + decode and before tempo scoring, overlap-safe chord duration, finite-only + tempo metric inputs, strict SHA-256 syntax, finite-only report metric values, + bounded fixture durations, and no copyrighted commercial recordings. Fixture paths are pytest temp files; reports store only SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, overlapping matching intervals do not double-count annotation duration, - silence-on-disk vs in-memory triad, 120 BPM Acc1, checksum mismatch through - both file evaluators, malformed/non-hex manifest provenance, NaN/infinity - rejection, and silence must not pass as C major. + silence-on-disk vs in-memory triad, 120 BPM Acc1, non-finite tempo estimate / + truth / tolerance rejection, checksum mismatch through both file evaluators, + malformed/non-hex manifest provenance, NaN/infinity report rejection, and + silence must not pass as C major. From 3d41c00ee739300886096c2556c6f2c951dfbef0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:39:56 +0900 Subject: [PATCH 15/78] docs(changelog): record finite tempo evidence --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3800a5d..53bd6b93b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject non-finite tempo estimates, ground-truth BPM values, and Acc1 tolerances as invalid accuracy evidence instead of recording them as ordinary misses. - Union overlapping matching chord-estimate intervals before duration-weighted recall so acceptance scores cannot double-count annotated time or exceed 100%. - Reject malformed accuracy-report provenance, including non-hex SHA-256 text and non-finite metric values, before acceptance evidence is consumed. - Score the C major acceptance case from checksummed on-disk WAV bytes instead of the pre-write in-memory triad. From 59997bef15eb73198bc3b1286c882db551f12ebb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:40:47 +0900 Subject: [PATCH 16/78] test(accuracy): bound fixture timing inputs --- .../analysis-engine/tests/test_accuracy_acceptance.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 388b15aea..91c131e8b 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -170,6 +170,16 @@ def test_render_helpers_reject_non_positive_inputs() -> None: write_pcm_wav(Path("unused.wav"), np.zeros(4, dtype=np.float32), 0) +def test_render_helpers_reject_non_finite_inputs() -> None: + """Fixture generation must reject non-finite timing before allocation/loops.""" + with pytest.raises(ValueError, match="duration_seconds.*finite"): + render_c_major_triad(duration_seconds=np.nan) + with pytest.raises(ValueError, match="bpm.*finite"): + render_click_track(bpm=np.nan) + with pytest.raises(ValueError, match="duration_seconds.*finite"): + render_click_track(duration_seconds=np.inf) + + def test_click_track_with_zero_length_click_stays_silent() -> None: """A 1 Hz sample rate makes the click window empty and leaves silence.""" audio = render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) From 6e7da0b11e513b4471db54b48f97b0c8679afde5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 03:41:07 +0900 Subject: [PATCH 17/78] fix(accuracy): bound non-finite fixture timing --- .../bandscope_analysis/accuracy/fixtures.py | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index cdcabc5fd..d2644bb54 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -27,19 +27,19 @@ def render_c_major_triad( """Render a unit-peak C major triad as float32 PCM. Args: - duration_seconds: Positive fixture length in seconds. - sample_rate: Positive samples-per-second rate. + duration_seconds: Finite positive fixture length in seconds. + sample_rate: Finite positive samples-per-second rate. Returns: Mono float32 samples in ``[-1, 1]``. Raises: - ValueError: If duration or sample rate is not positive. + ValueError: If duration or sample rate is non-finite or not positive. """ - if duration_seconds <= 0: - raise ValueError("duration_seconds must be positive") - if sample_rate <= 0: - raise ValueError("sample_rate must be positive") + if not np.isfinite(duration_seconds) or duration_seconds <= 0: + raise ValueError("duration_seconds must be finite and positive") + if not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be finite and positive") sample_count = int(duration_seconds * sample_rate) times = np.arange(sample_count, dtype=np.float32) / np.float32(sample_rate) @@ -59,18 +59,23 @@ def render_click_track( """Render a click track at a known tempo. Args: - bpm: Positive true tempo in beats per minute. - duration_seconds: Positive fixture length in seconds. - sample_rate: Positive samples-per-second rate. + bpm: Finite positive true tempo in beats per minute. + duration_seconds: Finite positive fixture length in seconds. + sample_rate: Finite positive samples-per-second rate. Returns: Mono float32 samples with a decaying click on each beat. Raises: - ValueError: If tempo, duration, or sample rate is not positive. + ValueError: If tempo, duration, or sample rate is non-finite or not + positive. """ - if bpm <= 0 or duration_seconds <= 0 or sample_rate <= 0: - raise ValueError("bpm, duration_seconds, and sample_rate must be positive") + if not np.isfinite(bpm) or bpm <= 0: + raise ValueError("bpm must be finite and positive") + if not np.isfinite(duration_seconds) or duration_seconds <= 0: + raise ValueError("duration_seconds must be finite and positive") + if not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be finite and positive") sample_count = int(duration_seconds * sample_rate) audio = np.zeros(sample_count, dtype=np.float32) @@ -103,16 +108,16 @@ def write_pcm_wav(path: Path, audio: NDArray[np.floating], sample_rate: int) -> Args: path: Destination path. Parent directories are created. audio: Mono PCM samples. - sample_rate: Samples-per-second rate used to write the file. + sample_rate: Finite positive samples-per-second rate used to write the file. Returns: Lowercase hex SHA-256 of the written file. Raises: - ValueError: If the sample rate is not positive. + ValueError: If the sample rate is non-finite or not positive. """ - if sample_rate <= 0: - raise ValueError("sample_rate must be positive") + if not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be finite and positive") path.parent.mkdir(parents=True, exist_ok=True) sf.write(path, np.asarray(audio, dtype=np.float32), sample_rate) return hashlib.sha256(path.read_bytes()).hexdigest() From 2b0bbbbb63eaa1410cc3c62cb24ac0b958045495 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 04:35:21 +0900 Subject: [PATCH 18/78] test(accuracy): reject non-finite chord timing evidence --- .../tests/test_accuracy_acceptance.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 91c131e8b..a073a9185 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -206,6 +206,25 @@ def test_duration_weighted_recall_unions_overlapping_matching_estimates() -> Non assert recall == pytest.approx(0.8) +@pytest.mark.parametrize( + ("segments", "start_seconds", "end_seconds"), + [ + ([(np.nan, 2.0, "C")], 0.0, 4.0), + ([(0.0, np.inf, "C")], 0.0, 4.0), + ([(0.0, 2.0, "C")], np.nan, 4.0), + ([(0.0, 2.0, "C")], 0.0, np.inf), + ], +) +def test_duration_weighted_recall_rejects_non_finite_timing( + segments: list[tuple[float, float, str]], + start_seconds: float, + end_seconds: float, +) -> None: + """Non-finite estimate or annotation times must not fabricate recall.""" + with pytest.raises(ValueError, match="finite"): + duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) + + def test_tempo_acc1_window_and_guards() -> None: """Acc1 must accept a 4% window and reject octave errors and bad inputs.""" assert tempo_acc1(120.0, 120.0) is True From 1302d41793d1a3d3adb8bd87862c8cb0e118d1bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 04:35:54 +0900 Subject: [PATCH 19/78] fix(accuracy): fail closed on non-finite chord timing --- .../src/bandscope_analysis/accuracy/metrics.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py index 0dcd3b68d..c0f851dfe 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py @@ -19,25 +19,32 @@ def duration_weighted_chord_recall( known chord for most of the fixture?” (Odekerken et al., 2021; Raffel et al., 2014). Matching estimate intervals are unioned after clipping to the annotation window, so overlapping estimates cannot count the same - annotated time more than once. + annotated time more than once. Non-finite annotation or estimate timing is + invalid acceptance evidence and fails closed before clipping. Args: - segments: ``(start, end, chord)`` estimates in seconds. + segments: ``(start, end, chord)`` estimates in seconds. Segment timing + values must be finite. expected_chord: Ground-truth chord symbol for the interval. - start_seconds: Inclusive annotation start. - end_seconds: Exclusive annotation end. Must be greater than start. + start_seconds: Inclusive finite annotation start. + end_seconds: Exclusive finite annotation end. Must be greater than start. Returns: A value in ``[0, 1]``. Raises: - ValueError: If the annotation interval is empty or reversed. + ValueError: If annotation or estimate timing is non-finite, or the + annotation interval is empty or reversed. """ + if not math.isfinite(start_seconds) or not math.isfinite(end_seconds): + raise ValueError("annotation times must be finite") if end_seconds <= start_seconds: raise ValueError("annotation end_seconds must be greater than start_seconds") matching_intervals: list[tuple[float, float]] = [] for segment_start, segment_end, chord in segments: + if not math.isfinite(segment_start) or not math.isfinite(segment_end): + raise ValueError("segment times must be finite") overlap_start = max(start_seconds, segment_start) overlap_end = min(end_seconds, segment_end) if overlap_end > overlap_start and chord == expected_chord: From 4cfdfabd585b97ea8ee9a7ef64dab0512d3c5bfe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 04:36:28 +0900 Subject: [PATCH 20/78] docs(accuracy): document finite chord timing evidence --- .../real-audio-accuracy-acceptance.md | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index e6bf02b9f..7218cbb1f 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -27,8 +27,10 @@ stem quality, or private-corpus readiness. tolerance. Matching estimate intervals are clipped to the annotation window and unioned before duration is accumulated, so overlapping or duplicate estimates cannot count the same annotated time twice or produce recall above - `1.0`. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et al., - 2014). + `1.0`. Annotation and estimate times must be finite before clipping; NaN or + infinite timing is invalid acceptance evidence and fails closed instead of + being allowed to fabricate covered duration. The metric family is WCSR/CSR + (Odekerken et al., 2021; Raffel et al., 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. @@ -79,13 +81,14 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? registered floors. - Mitigations: no network, no shell, checksum fail-closed before C-major decode and before tempo scoring, overlap-safe chord duration, finite-only - tempo metric inputs, strict SHA-256 syntax, finite-only report metric values, - bounded fixture durations, and no copyrighted commercial recordings. - Fixture paths are pytest temp files; reports store only SHA-256 and labels, - not waveform bytes. + annotation/estimate timing and tempo metric inputs, strict SHA-256 syntax, + finite-only report metric values, bounded fixture durations, and no + copyrighted commercial recordings. Fixture paths are pytest temp files; + reports store only SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, overlapping matching intervals do not double-count annotation duration, - silence-on-disk vs in-memory triad, 120 BPM Acc1, non-finite tempo estimate / - truth / tolerance rejection, checksum mismatch through both file evaluators, - malformed/non-hex manifest provenance, NaN/infinity report rejection, and - silence must not pass as C major. + non-finite chord annotation/estimate timing rejection, silence-on-disk vs + in-memory triad, 120 BPM Acc1, non-finite tempo estimate / truth / tolerance + rejection, checksum mismatch through both file evaluators, malformed/non-hex + manifest provenance, NaN/infinity report rejection, and silence must not pass + as C major. From 44162996e9aef00b654ad4269e3fcc2e9366efe0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 04:36:55 +0900 Subject: [PATCH 21/78] docs(changelog): record finite chord timing guard --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53bd6b93b..e8ff27afb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject non-finite chord annotation and estimate timings before duration-weighted recall so NaN/Inf evidence cannot fabricate covered duration. - Reject non-finite tempo estimates, ground-truth BPM values, and Acc1 tolerances as invalid accuracy evidence instead of recording them as ordinary misses. - Union overlapping matching chord-estimate intervals before duration-weighted recall so acceptance scores cannot double-count annotated time or exceed 100%. - Reject malformed accuracy-report provenance, including non-hex SHA-256 text and non-finite metric values, before acceptance evidence is consumed. From a59085f78631426b62bd9c39fa8621c35c92be1f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:03:25 +0900 Subject: [PATCH 22/78] test(accuracy): fail closed without product version --- .../tests/test_accuracy_manifest_version.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_manifest_version.py diff --git a/services/analysis-engine/tests/test_accuracy_manifest_version.py b/services/analysis-engine/tests/test_accuracy_manifest_version.py new file mode 100644 index 000000000..208562876 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_manifest_version.py @@ -0,0 +1,23 @@ +"""Regression tests for accuracy-report engine-version provenance.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from bandscope_analysis.accuracy import read_product_version + + +def test_missing_or_empty_product_version_fails_closed(tmp_path: Path) -> None: + """Accuracy evidence must not publish an ``unknown`` engine version.""" + missing_tree = tmp_path / "missing-tree" + missing_tree.mkdir() + with pytest.raises(ValueError, match="VERSION"): + read_product_version(missing_tree) + + empty_tree = tmp_path / "empty-tree" + empty_tree.mkdir() + (empty_tree / "VERSION").write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="VERSION"): + read_product_version(empty_tree) From 3ad3a52d2407824f0050a52bb2be0c54b7015178 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:03:48 +0900 Subject: [PATCH 23/78] fix(accuracy): require exact product version provenance --- .../src/bandscope_analysis/accuracy/manifest.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py index 250cc42a2..c249cc7a6 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py @@ -33,13 +33,20 @@ class AccuracyCaseReport(TypedDict): def read_product_version(start: Path | None = None) -> str: - """Return the nearest ``VERSION`` file contents, or ``unknown``. + """Return the nearest non-empty ``VERSION`` file contents. + + Accuracy reports are provenance evidence, so an unavailable product version + is not converted into an ``unknown`` value that could still pass report + validation. Args: start: File or directory to walk upward from. Defaults to this module. Returns: - Stripped version text, or ``unknown`` when no non-empty file is found. + Stripped product version text. + + Raises: + ValueError: If no non-empty ``VERSION`` file exists in the ancestor path. """ current = start if start is not None else Path(__file__).resolve() cursor = current.parent if current.is_file() else current @@ -50,7 +57,7 @@ def read_product_version(start: Path | None = None) -> str: text = candidate.read_text(encoding="utf-8").strip() if text: return text - return "unknown" + raise ValueError("Product VERSION file is missing or empty") def build_case_report( @@ -76,6 +83,10 @@ def build_case_report( Returns: A report that ``parse_case_report`` will accept. + + Raises: + ValueError: If required report evidence is invalid or the default product + ``VERSION`` cannot be resolved. """ report: AccuracyCaseReport = { "case_id": case_id, From 65cb516a30e61e45307a4ffdb595709e2eb6b402 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:04:39 +0900 Subject: [PATCH 24/78] test(accuracy): align version provenance contract --- services/analysis-engine/tests/test_accuracy_acceptance.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index a073a9185..92057c1c9 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -299,8 +299,8 @@ def test_parse_case_report_rejects_malformed_payloads() -> None: parse_case_report({**valid, "true_label": ""}) -def test_read_product_version_uses_version_file_or_unknown(tmp_path: Path) -> None: - """Version lookup must read VERSION and fall back to unknown.""" +def test_read_product_version_uses_version_file_and_fails_closed(tmp_path: Path) -> None: + """Version lookup must read VERSION and reject missing provenance.""" versioned = tmp_path / "versioned" versioned.mkdir() (versioned / "VERSION").write_text("9.9.9\n", encoding="utf-8") @@ -309,5 +309,6 @@ def test_read_product_version_uses_version_file_or_unknown(tmp_path: Path) -> No empty = tmp_path / "empty-tree" empty.mkdir() (empty / "VERSION").write_text(" \n", encoding="utf-8") - assert read_product_version(empty) == "unknown" + with pytest.raises(ValueError, match="VERSION"): + read_product_version(empty) assert read_product_version() != "unknown" From 3856a9ea6ec40f2cc827731b32550e73016d6edd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:05:05 +0900 Subject: [PATCH 25/78] docs(changelog): record exact version provenance --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8ff27afb..f17672270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Fail closed when accuracy-report provenance cannot resolve a non-empty product `VERSION`, instead of publishing an `unknown` engine version as valid evidence. - Reject non-finite chord annotation and estimate timings before duration-weighted recall so NaN/Inf evidence cannot fabricate covered duration. - Reject non-finite tempo estimates, ground-truth BPM values, and Acc1 tolerances as invalid accuracy evidence instead of recording them as ordinary misses. - Union overlapping matching chord-estimate intervals before duration-weighted recall so acceptance scores cannot double-count annotated time or exceed 100%. From c5c1d23b027cfb12f385b4711e5c636bf8bc4baa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:05:29 +0900 Subject: [PATCH 26/78] docs(accuracy): require exact engine version provenance --- .../real-audio-accuracy-acceptance.md | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 7218cbb1f..2b71258c3 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -6,7 +6,8 @@ evidence. ```bash uv run --project services/analysis-engine pytest \ - services/analysis-engine/tests/test_accuracy_acceptance.py + services/analysis-engine/tests/test_accuracy_acceptance.py \ + services/analysis-engine/tests/test_accuracy_manifest_version.py ``` ## Why this lock exists @@ -44,6 +45,10 @@ stem quality, or private-corpus readiness. hexadecimal characters, and `metric_value` is a finite numeric value. Boolean, NaN, infinity, overflow-to-float, malformed digest, or missing-field evidence fails closed rather than becoming a portable acceptance record. +- When a caller does not provide an explicit engine version, report creation + resolves the repository product `VERSION`. Missing or empty `VERSION` + provenance fails closed; `unknown` is not accepted as a substitute for the + exact engine version required by the accuracy evidence contract. ## Claim boundary @@ -74,21 +79,22 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? ## Security Notes - Attack surface: generated WAV bytes, SHA-256 digests, decoded PCM, recognizer - segment timings, tempo estimates, and parsed case-report mappings passed into - the accuracy acceptance path. + segment timings, tempo estimates, product-version provenance, and parsed + case-report mappings passed into the accuracy acceptance path. - Trust boundary: untrusted audio, recognizer output, and manifests; trusted - repo-controlled fixture generators, true labels, metric definitions, and - registered floors. + repo-controlled fixture generators, true labels, metric definitions, + registered floors, and the repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major decode and before tempo scoring, overlap-safe chord duration, finite-only annotation/estimate timing and tempo metric inputs, strict SHA-256 syntax, - finite-only report metric values, bounded fixture durations, and no - copyrighted commercial recordings. Fixture paths are pytest temp files; - reports store only SHA-256 and labels, not waveform bytes. + finite-only report metric values, exact non-empty product-version provenance, + bounded fixture durations, and no copyrighted commercial recordings. + Fixture paths are pytest temp files; reports store only SHA-256 and labels, + not waveform bytes. - Test points: deterministic digest, C major recall after file decode, overlapping matching intervals do not double-count annotation duration, non-finite chord annotation/estimate timing rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, non-finite tempo estimate / truth / tolerance rejection, checksum mismatch through both file evaluators, malformed/non-hex - manifest provenance, NaN/infinity report rejection, and silence must not pass - as C major. + manifest provenance, NaN/infinity report rejection, missing/empty product + `VERSION` rejection, and silence must not pass as C major. From 4dd0def17954a9bd0025eeb0c3f6c5569124a4e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:06:07 +0900 Subject: [PATCH 27/78] test(accuracy): reject unknown engine version evidence --- .../tests/test_accuracy_manifest_version.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/services/analysis-engine/tests/test_accuracy_manifest_version.py b/services/analysis-engine/tests/test_accuracy_manifest_version.py index 208562876..c4b1ed7ce 100644 --- a/services/analysis-engine/tests/test_accuracy_manifest_version.py +++ b/services/analysis-engine/tests/test_accuracy_manifest_version.py @@ -6,7 +6,7 @@ import pytest -from bandscope_analysis.accuracy import read_product_version +from bandscope_analysis.accuracy import build_case_report, parse_case_report, read_product_version def test_missing_or_empty_product_version_fails_closed(tmp_path: Path) -> None: @@ -21,3 +21,28 @@ def test_missing_or_empty_product_version_fails_closed(tmp_path: Path) -> None: (empty_tree / "VERSION").write_text(" \n", encoding="utf-8") with pytest.raises(ValueError, match="VERSION"): read_product_version(empty_tree) + + +def test_unknown_engine_version_is_rejected_at_report_boundary() -> None: + """Explicit or parsed ``unknown`` provenance must not become valid evidence.""" + report = { + "case_id": "c-major-triad", + "audio_sha256": "a" * 64, + "metric_name": "duration_weighted_chord_recall", + "metric_value": 0.9, + "passed": True, + "engine_version": "unknown", + "true_label": "C", + } + with pytest.raises(ValueError, match="engine_version"): + parse_case_report(report) + with pytest.raises(ValueError, match="engine_version"): + build_case_report( + case_id="c-major-triad", + audio_sha256="a" * 64, + metric_name="duration_weighted_chord_recall", + metric_value=0.9, + passed=True, + true_label="C", + engine_version="unknown", + ) From 60ecd22c0a6d7f7cf2a37b4826481f0613fbb30f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:06:28 +0900 Subject: [PATCH 28/78] test(accuracy): cover inexact version provenance --- .../tests/test_accuracy_manifest_version.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/analysis-engine/tests/test_accuracy_manifest_version.py b/services/analysis-engine/tests/test_accuracy_manifest_version.py index c4b1ed7ce..62c8f0ea3 100644 --- a/services/analysis-engine/tests/test_accuracy_manifest_version.py +++ b/services/analysis-engine/tests/test_accuracy_manifest_version.py @@ -23,15 +23,16 @@ def test_missing_or_empty_product_version_fails_closed(tmp_path: Path) -> None: read_product_version(empty_tree) -def test_unknown_engine_version_is_rejected_at_report_boundary() -> None: - """Explicit or parsed ``unknown`` provenance must not become valid evidence.""" +@pytest.mark.parametrize("engine_version", ["unknown", "UNKNOWN", " 0.1.3 "]) +def test_inexact_engine_version_is_rejected_at_report_boundary(engine_version: str) -> None: + """Unknown or whitespace-obscured versions must not become valid evidence.""" report = { "case_id": "c-major-triad", "audio_sha256": "a" * 64, "metric_name": "duration_weighted_chord_recall", "metric_value": 0.9, "passed": True, - "engine_version": "unknown", + "engine_version": engine_version, "true_label": "C", } with pytest.raises(ValueError, match="engine_version"): @@ -44,5 +45,5 @@ def test_unknown_engine_version_is_rejected_at_report_boundary() -> None: metric_value=0.9, passed=True, true_label="C", - engine_version="unknown", + engine_version=engine_version, ) From ebaa3f566c19934812272ea798220c74b9902f5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:06:54 +0900 Subject: [PATCH 29/78] fix(accuracy): reject unknown version evidence --- .../src/bandscope_analysis/accuracy/manifest.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py index c249cc7a6..261b90835 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py @@ -79,7 +79,7 @@ def build_case_report( metric_value: Numeric score for this run. passed: Whether the score met the registered tolerance. true_label: Ground-truth label the buyer should hear. - engine_version: Optional override. Defaults to the product ``VERSION``. + engine_version: Optional exact override. Defaults to the product ``VERSION``. Returns: A report that ``parse_case_report`` will accept. @@ -148,8 +148,13 @@ def parse_case_report(value: object) -> AccuracyCaseReport: raise ValueError("metric_value must be a finite number") if not isinstance(passed, bool): raise ValueError("passed must be a boolean") - if not isinstance(engine_version, str) or not engine_version: - raise ValueError("engine_version must be a non-empty string") + if ( + not isinstance(engine_version, str) + or not engine_version + or engine_version.strip() != engine_version + or engine_version.lower() == "unknown" + ): + raise ValueError("engine_version must be an exact non-empty version, not unknown") if not isinstance(true_label, str) or not true_label: raise ValueError("true_label must be a non-empty string") From db8412fad37d0b44205e983b9f0f3f0224e94999 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:29:59 +0900 Subject: [PATCH 30/78] test(accuracy): reject malformed chord estimate intervals --- .../test_accuracy_metric_interval_contract.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_metric_interval_contract.py diff --git a/services/analysis-engine/tests/test_accuracy_metric_interval_contract.py b/services/analysis-engine/tests/test_accuracy_metric_interval_contract.py new file mode 100644 index 000000000..814b12790 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_metric_interval_contract.py @@ -0,0 +1,28 @@ +"""Regression tests for MIR acceptance interval authority.""" + +from __future__ import annotations + +import pytest + +from bandscope_analysis.accuracy import duration_weighted_chord_recall + + +@pytest.mark.parametrize( + ("segment_start", "segment_end"), + [ + (2.0, 2.0), + (3.0, 2.0), + ], +) +def test_duration_weighted_recall_rejects_empty_or_reversed_estimate_intervals( + segment_start: float, + segment_end: float, +) -> None: + """Malformed estimate intervals must fail closed before acceptance scoring.""" + with pytest.raises(ValueError, match="segment end_seconds must be greater than start_seconds"): + duration_weighted_chord_recall( + [(segment_start, segment_end, "C")], + "C", + 0.0, + 4.0, + ) From 05929e269a6694ab8b69d29400bca989241a9fed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:30:54 +0900 Subject: [PATCH 31/78] fix(accuracy): reject malformed chord estimate intervals --- .../src/bandscope_analysis/accuracy/metrics.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py index c0f851dfe..1046ccc05 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py @@ -19,12 +19,13 @@ def duration_weighted_chord_recall( known chord for most of the fixture?” (Odekerken et al., 2021; Raffel et al., 2014). Matching estimate intervals are unioned after clipping to the annotation window, so overlapping estimates cannot count the same - annotated time more than once. Non-finite annotation or estimate timing is - invalid acceptance evidence and fails closed before clipping. + annotated time more than once. Non-finite, empty, or reversed annotation + and estimate timing is invalid acceptance evidence and fails closed before + clipping. Args: segments: ``(start, end, chord)`` estimates in seconds. Segment timing - values must be finite. + values must be finite and each end must be greater than its start. expected_chord: Ground-truth chord symbol for the interval. start_seconds: Inclusive finite annotation start. end_seconds: Exclusive finite annotation end. Must be greater than start. @@ -33,8 +34,8 @@ def duration_weighted_chord_recall( A value in ``[0, 1]``. Raises: - ValueError: If annotation or estimate timing is non-finite, or the - annotation interval is empty or reversed. + ValueError: If annotation or estimate timing is non-finite, empty, or + reversed. """ if not math.isfinite(start_seconds) or not math.isfinite(end_seconds): raise ValueError("annotation times must be finite") @@ -45,6 +46,8 @@ def duration_weighted_chord_recall( for segment_start, segment_end, chord in segments: if not math.isfinite(segment_start) or not math.isfinite(segment_end): raise ValueError("segment times must be finite") + if segment_end <= segment_start: + raise ValueError("segment end_seconds must be greater than start_seconds") overlap_start = max(start_seconds, segment_start) overlap_end = min(end_seconds, segment_end) if overlap_end > overlap_start and chord == expected_chord: From 8e22c970dede9c19989952d10dcfa8da7c30ede2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:31:46 +0900 Subject: [PATCH 32/78] docs(changelog): record malformed interval rejection --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f17672270..9d508072d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject empty or reversed chord-estimate intervals before duration-weighted recall so malformed timing evidence cannot be silently ignored by an accuracy acceptance score. - Fail closed when accuracy-report provenance cannot resolve a non-empty product `VERSION`, instead of publishing an `unknown` engine version as valid evidence. - Reject non-finite chord annotation and estimate timings before duration-weighted recall so NaN/Inf evidence cannot fabricate covered duration. - Reject non-finite tempo estimates, ground-truth BPM values, and Acc1 tolerances as invalid accuracy evidence instead of recording them as ordinary misses. From caaf7c7417b690e6f564d1d0018a6bbe79c39223 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:32:24 +0900 Subject: [PATCH 33/78] docs(accuracy): document interval validity gate --- .../real-audio-accuracy-acceptance.md | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 2b71258c3..21509438b 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -7,7 +7,8 @@ evidence. ```bash uv run --project services/analysis-engine pytest \ services/analysis-engine/tests/test_accuracy_acceptance.py \ - services/analysis-engine/tests/test_accuracy_manifest_version.py + services/analysis-engine/tests/test_accuracy_manifest_version.py \ + services/analysis-engine/tests/test_accuracy_metric_interval_contract.py ``` ## Why this lock exists @@ -28,10 +29,12 @@ stem quality, or private-corpus readiness. tolerance. Matching estimate intervals are clipped to the annotation window and unioned before duration is accumulated, so overlapping or duplicate estimates cannot count the same annotated time twice or produce recall above - `1.0`. Annotation and estimate times must be finite before clipping; NaN or - infinite timing is invalid acceptance evidence and fails closed instead of - being allowed to fabricate covered duration. The metric family is WCSR/CSR - (Odekerken et al., 2021; Raffel et al., 2014). + `1.0`. Annotation and estimate times must be finite before clipping, and each + estimate interval must have a strictly increasing start/end pair; NaN, + infinite, empty, or reversed timing is invalid acceptance evidence and fails + closed instead of being silently ignored or allowed to fabricate covered + duration. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et + al., 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. @@ -86,15 +89,16 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? registered floors, and the repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major decode and before tempo scoring, overlap-safe chord duration, finite-only - annotation/estimate timing and tempo metric inputs, strict SHA-256 syntax, - finite-only report metric values, exact non-empty product-version provenance, - bounded fixture durations, and no copyrighted commercial recordings. - Fixture paths are pytest temp files; reports store only SHA-256 and labels, - not waveform bytes. + annotation/estimate timing, strictly increasing estimate intervals, finite-only + tempo metric inputs, strict SHA-256 syntax, finite-only report metric values, + exact non-empty product-version provenance, bounded fixture durations, and no + copyrighted commercial recordings. Fixture paths are pytest temp files; + reports store only SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, overlapping matching intervals do not double-count annotation duration, - non-finite chord annotation/estimate timing rejection, silence-on-disk vs - in-memory triad, 120 BPM Acc1, non-finite tempo estimate / truth / tolerance - rejection, checksum mismatch through both file evaluators, malformed/non-hex - manifest provenance, NaN/infinity report rejection, missing/empty product - `VERSION` rejection, and silence must not pass as C major. + non-finite chord annotation/estimate timing rejection, empty/reversed estimate + interval rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, + non-finite tempo estimate / truth / tolerance rejection, checksum mismatch + through both file evaluators, malformed/non-hex manifest provenance, + NaN/infinity report rejection, missing/empty product `VERSION` rejection, and + silence must not pass as C major. From b9bb4091fa0e6ad7139300217a6d6b6b64782d36 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:46:12 +0900 Subject: [PATCH 34/78] test(accuracy): reject padded report evidence strings --- .../test_accuracy_manifest_string_contract.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_manifest_string_contract.py diff --git a/services/analysis-engine/tests/test_accuracy_manifest_string_contract.py b/services/analysis-engine/tests/test_accuracy_manifest_string_contract.py new file mode 100644 index 000000000..fb0fbff14 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_manifest_string_contract.py @@ -0,0 +1,37 @@ +"""Regression tests for accuracy-report string evidence authority.""" + +from __future__ import annotations + +import pytest + +from bandscope_analysis.accuracy import build_case_report, parse_case_report + + +@pytest.mark.parametrize( + ("field_name", "invalid_value"), + [ + ("case_id", " "), + ("case_id", " c-major-triad "), + ("metric_name", " "), + ("metric_name", " tempo_acc1 "), + ("true_label", " "), + ("true_label", " C "), + ], +) +def test_accuracy_report_rejects_blank_or_padded_string_evidence( + field_name: str, + invalid_value: str, +) -> None: + """Identifiers and truth labels must be exact non-blank evidence strings.""" + valid = build_case_report( + case_id="c-major-triad", + audio_sha256="a" * 64, + metric_name="duration_weighted_chord_recall", + metric_value=0.9, + passed=True, + true_label="C", + engine_version="0.1.3", + ) + + with pytest.raises(ValueError, match=field_name): + parse_case_report({**valid, field_name: invalid_value}) From eb3d304fa730b1beb98f7e8a55af0b43d3c4c11a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:48:23 +0900 Subject: [PATCH 35/78] fix(accuracy): reject padded report evidence strings --- .../src/bandscope_analysis/accuracy/manifest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py index 261b90835..6fcf42b0f 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py @@ -128,16 +128,16 @@ def parse_case_report(value: object) -> AccuracyCaseReport: engine_version = value["engine_version"] true_label = value["true_label"] - if not isinstance(case_id, str) or not case_id: - raise ValueError("case_id must be a non-empty string") + if not isinstance(case_id, str) or not case_id or case_id.strip() != case_id: + raise ValueError("case_id must be an exact non-empty string") if ( not isinstance(audio_sha256, str) or len(audio_sha256) != 64 or any(character not in _HEX_DIGITS for character in audio_sha256) ): raise ValueError("audio_sha256 must be a 64-character hex digest") - if not isinstance(metric_name, str) or not metric_name: - raise ValueError("metric_name must be a non-empty string") + if not isinstance(metric_name, str) or not metric_name or metric_name.strip() != metric_name: + raise ValueError("metric_name must be an exact non-empty string") if isinstance(metric_value, bool) or not isinstance(metric_value, (int, float)): raise ValueError("metric_value must be a finite number") try: @@ -155,8 +155,8 @@ def parse_case_report(value: object) -> AccuracyCaseReport: or engine_version.lower() == "unknown" ): raise ValueError("engine_version must be an exact non-empty version, not unknown") - if not isinstance(true_label, str) or not true_label: - raise ValueError("true_label must be a non-empty string") + if not isinstance(true_label, str) or not true_label or true_label.strip() != true_label: + raise ValueError("true_label must be an exact non-empty string") return { "case_id": case_id, From 6dcfc9ad9a791b4e3361c3552c1edddef734673d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:49:43 +0900 Subject: [PATCH 36/78] docs(changelog): record exact accuracy-report strings --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d508072d..6641e406d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject blank or edge-whitespace-padded accuracy-report case IDs, metric names, and truth labels so portable acceptance evidence preserves exact registered identities. - Reject empty or reversed chord-estimate intervals before duration-weighted recall so malformed timing evidence cannot be silently ignored by an accuracy acceptance score. - Fail closed when accuracy-report provenance cannot resolve a non-empty product `VERSION`, instead of publishing an `unknown` engine version as valid evidence. - Reject non-finite chord annotation and estimate timings before duration-weighted recall so NaN/Inf evidence cannot fabricate covered duration. From 3f2b820859f25b8653adf8997acce3c837da323f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 05:59:03 +0900 Subject: [PATCH 37/78] test(accuracy): reject boolean metric evidence --- .../tests/test_accuracy_acceptance.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 92057c1c9..d397e3d57 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -225,6 +225,25 @@ def test_duration_weighted_recall_rejects_non_finite_timing( duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) +@pytest.mark.parametrize( + ("segments", "start_seconds", "end_seconds"), + [ + ([(True, 2.0, "C")], 0.0, 4.0), + ([(0.0, True, "C")], 0.0, 4.0), + ([(0.0, 2.0, "C")], False, 4.0), + ([(0.0, 2.0, "C")], 0.0, True), + ], +) +def test_duration_weighted_recall_rejects_boolean_timing_evidence( + segments: list[tuple[float | bool, float | bool, str]], + start_seconds: float | bool, + end_seconds: float | bool, +) -> None: + """Boolean timestamps must not be accepted as numeric MIR timing evidence.""" + with pytest.raises(ValueError, match="times must be finite numbers"): + duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) # type: ignore[arg-type] + + def test_tempo_acc1_window_and_guards() -> None: """Acc1 must accept a 4% window and reject octave errors and bad inputs.""" assert tempo_acc1(120.0, 120.0) is True @@ -258,6 +277,25 @@ def test_tempo_acc1_rejects_non_finite_evidence( tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) +@pytest.mark.parametrize( + ("estimated_bpm", "true_bpm", "relative_tolerance", "message"), + [ + (True, 120.0, 0.04, "estimated_bpm"), + (120.0, True, 0.04, "true_bpm"), + (120.0, 120.0, True, "relative_tolerance"), + ], +) +def test_tempo_acc1_rejects_boolean_numeric_evidence( + estimated_bpm: float | bool, + true_bpm: float | bool, + relative_tolerance: float | bool, + message: str, +) -> None: + """Boolean values must not satisfy numeric Acc1 evidence contracts.""" + with pytest.raises(ValueError, match=message): + tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) # type: ignore[arg-type] + + def test_parse_case_report_rejects_malformed_payloads() -> None: """Manifest parsing must fail closed on missing or mistyped fields.""" valid = build_case_report( From 1f12562fa1aaa74bb9f9edfc597c5cee858939f8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:00:15 +0900 Subject: [PATCH 38/78] fix(accuracy): reject boolean metric evidence --- .../bandscope_analysis/accuracy/metrics.py | 65 ++++++++++++------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py index 1046ccc05..c565a8306 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/metrics.py @@ -19,33 +19,45 @@ def duration_weighted_chord_recall( known chord for most of the fixture?” (Odekerken et al., 2021; Raffel et al., 2014). Matching estimate intervals are unioned after clipping to the annotation window, so overlapping estimates cannot count the same - annotated time more than once. Non-finite, empty, or reversed annotation - and estimate timing is invalid acceptance evidence and fails closed before - clipping. + annotated time more than once. Non-finite, Boolean, empty, or reversed + annotation and estimate timing is invalid acceptance evidence and fails + closed before clipping. Args: segments: ``(start, end, chord)`` estimates in seconds. Segment timing - values must be finite and each end must be greater than its start. + values must be finite non-Boolean numbers and each end must be + greater than its start. expected_chord: Ground-truth chord symbol for the interval. - start_seconds: Inclusive finite annotation start. - end_seconds: Exclusive finite annotation end. Must be greater than start. + start_seconds: Inclusive finite non-Boolean annotation start. + end_seconds: Exclusive finite non-Boolean annotation end. Must be + greater than start. Returns: A value in ``[0, 1]``. Raises: - ValueError: If annotation or estimate timing is non-finite, empty, or - reversed. + ValueError: If annotation or estimate timing is Boolean, non-finite, + empty, or reversed. """ - if not math.isfinite(start_seconds) or not math.isfinite(end_seconds): - raise ValueError("annotation times must be finite") + if ( + isinstance(start_seconds, bool) + or isinstance(end_seconds, bool) + or not math.isfinite(start_seconds) + or not math.isfinite(end_seconds) + ): + raise ValueError("annotation times must be finite numbers") if end_seconds <= start_seconds: raise ValueError("annotation end_seconds must be greater than start_seconds") matching_intervals: list[tuple[float, float]] = [] for segment_start, segment_end, chord in segments: - if not math.isfinite(segment_start) or not math.isfinite(segment_end): - raise ValueError("segment times must be finite") + if ( + isinstance(segment_start, bool) + or isinstance(segment_end, bool) + or not math.isfinite(segment_start) + or not math.isfinite(segment_end) + ): + raise ValueError("segment times must be finite numbers") if segment_end <= segment_start: raise ValueError("segment end_seconds must be greater than start_seconds") overlap_start = max(start_seconds, segment_start) @@ -77,26 +89,33 @@ def tempo_acc1( """Return whether estimated tempo is within Acc1 tolerance of the true tempo. Acc1 accepts an estimate within ``relative_tolerance`` of the true BPM and - does not credit octave errors (Schreiber & Müller, 2020). Non-finite - estimates, ground truth, or tolerances are invalid evidence and fail - closed instead of being converted into an ordinary metric miss. + does not credit octave errors (Schreiber & Müller, 2020). Boolean or + non-finite estimates, ground truth, or tolerances are invalid evidence and + fail closed instead of being converted into an ordinary metric miss. Args: - estimated_bpm: Engine tempo in beats per minute. Must be finite. - true_bpm: Known fixture tempo. Must be finite and positive. - relative_tolerance: Finite non-negative Acc1 window. The default is 4%. + estimated_bpm: Engine tempo in beats per minute. Must be a finite + non-Boolean number. + true_bpm: Known fixture tempo. Must be a finite positive non-Boolean + number. + relative_tolerance: Finite non-negative non-Boolean Acc1 window. The + default is 4%. Returns: ``True`` when the estimate is inside the Acc1 window. Raises: - ValueError: If any metric input is non-finite, ``true_bpm`` is not - positive, or the tolerance is negative. + ValueError: If any metric input is Boolean/non-finite, ``true_bpm`` is + not positive, or the tolerance is negative. """ - if not math.isfinite(estimated_bpm): + if isinstance(estimated_bpm, bool) or not math.isfinite(estimated_bpm): raise ValueError("estimated_bpm must be finite") - if not math.isfinite(true_bpm) or true_bpm <= 0: + if isinstance(true_bpm, bool) or not math.isfinite(true_bpm) or true_bpm <= 0: raise ValueError("true_bpm must be finite and positive") - if not math.isfinite(relative_tolerance) or relative_tolerance < 0: + if ( + isinstance(relative_tolerance, bool) + or not math.isfinite(relative_tolerance) + or relative_tolerance < 0 + ): raise ValueError("relative_tolerance must be finite and non-negative") return abs(estimated_bpm - true_bpm) / true_bpm <= relative_tolerance From 8492f2cd4eaa44cb1143d30dbb2d1a66c0686193 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:01:25 +0900 Subject: [PATCH 39/78] docs(changelog): record boolean metric guard --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6641e406d..2084d5dd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject Boolean chord timing and tempo metric inputs so `True`/`False` cannot masquerade as numeric MIR acceptance evidence through Python's `bool`-as-`int` semantics. - Reject blank or edge-whitespace-padded accuracy-report case IDs, metric names, and truth labels so portable acceptance evidence preserves exact registered identities. - Reject empty or reversed chord-estimate intervals before duration-weighted recall so malformed timing evidence cannot be silently ignored by an accuracy acceptance score. - Fail closed when accuracy-report provenance cannot resolve a non-empty product `VERSION`, instead of publishing an `unknown` engine version as valid evidence. @@ -77,4 +78,4 @@ - `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. - `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. -- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). +- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). \ No newline at end of file From 895f17b58763f6a38cbf03a7d56f5f9481f792fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:01:47 +0900 Subject: [PATCH 40/78] docs(accuracy): document boolean evidence boundary --- .../real-audio-accuracy-acceptance.md | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 21509438b..98cd897e6 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -29,18 +29,20 @@ stem quality, or private-corpus readiness. tolerance. Matching estimate intervals are clipped to the annotation window and unioned before duration is accumulated, so overlapping or duplicate estimates cannot count the same annotated time twice or produce recall above - `1.0`. Annotation and estimate times must be finite before clipping, and each - estimate interval must have a strictly increasing start/end pair; NaN, - infinite, empty, or reversed timing is invalid acceptance evidence and fails - closed instead of being silently ignored or allowed to fabricate covered - duration. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et - al., 2014). + `1.0`. Annotation and estimate times must be finite non-Boolean numbers before + clipping, and each estimate interval must have a strictly increasing + start/end pair; Boolean, NaN, infinite, empty, or reversed timing is invalid + acceptance evidence and fails closed instead of being silently ignored or + allowed to fabricate covered duration. The metric family is WCSR/CSR + (Odekerken et al., 2021; Raffel et al., 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. - Estimated BPM, true BPM, and the tolerance must all be finite; non-finite - metric inputs are invalid acceptance evidence and fail closed rather than - being recorded as an ordinary miss. + Estimated BPM, true BPM, and the tolerance must all be finite non-Boolean + numbers; Boolean or non-finite metric inputs are invalid acceptance evidence + and fail closed rather than being recorded as an ordinary miss. This matters + in Python because `bool` is an integer subtype and would otherwise satisfy + ordinary numeric comparisons. - Checksum mismatch fails closed on both file evaluators. Do not score a tampered file as a pass. - Machine-readable case reports are accepted only when the registered @@ -88,17 +90,18 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? repo-controlled fixture generators, true labels, metric definitions, registered floors, and the repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major - decode and before tempo scoring, overlap-safe chord duration, finite-only - annotation/estimate timing, strictly increasing estimate intervals, finite-only - tempo metric inputs, strict SHA-256 syntax, finite-only report metric values, - exact non-empty product-version provenance, bounded fixture durations, and no - copyrighted commercial recordings. Fixture paths are pytest temp files; - reports store only SHA-256 and labels, not waveform bytes. + decode and before tempo scoring, overlap-safe chord duration, finite + non-Boolean annotation/estimate timing, strictly increasing estimate + intervals, finite non-Boolean tempo metric inputs, strict SHA-256 syntax, + finite-only report metric values, exact non-empty product-version provenance, + bounded fixture durations, and no copyrighted commercial recordings. Fixture + paths are pytest temp files; reports store only SHA-256 and labels, not + waveform bytes. - Test points: deterministic digest, C major recall after file decode, overlapping matching intervals do not double-count annotation duration, - non-finite chord annotation/estimate timing rejection, empty/reversed estimate - interval rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, - non-finite tempo estimate / truth / tolerance rejection, checksum mismatch - through both file evaluators, malformed/non-hex manifest provenance, - NaN/infinity report rejection, missing/empty product `VERSION` rejection, and - silence must not pass as C major. + non-finite and Boolean chord annotation/estimate timing rejection, + empty/reversed estimate interval rejection, silence-on-disk vs in-memory + triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / tolerance + rejection, checksum mismatch through both file evaluators, malformed/non-hex + manifest provenance, NaN/infinity report rejection, missing/empty product + `VERSION` rejection, and silence must not pass as C major. \ No newline at end of file From 49a1ce2644c777c7c3d260a0fcd58e194758050b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:07:40 +0900 Subject: [PATCH 41/78] test(accuracy): reject boolean fixture inputs --- .../test_accuracy_boolean_fixture_inputs.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py diff --git a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py new file mode 100644 index 000000000..05d615727 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py @@ -0,0 +1,58 @@ +"""Boolean evidence guards for deterministic real-audio fixtures.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +import numpy as np +import pytest + +from bandscope_analysis.accuracy import render_c_major_triad, render_click_track, write_pcm_wav + + +@pytest.mark.parametrize( + ("keyword", "value", "message"), + [ + ("duration_seconds", True, "duration_seconds"), + ("sample_rate", True, "sample_rate"), + ], +) +def test_c_major_fixture_rejects_boolean_numeric_evidence( + keyword: str, + value: bool, + message: str, +) -> None: + """Boolean duration/rate values must not become numeric fixture authority.""" + kwargs = {keyword: cast(Any, value)} + with pytest.raises(ValueError, match=message): + render_c_major_triad(**kwargs) + + +@pytest.mark.parametrize( + ("keyword", "value", "message"), + [ + ("bpm", True, "bpm"), + ("duration_seconds", True, "duration_seconds"), + ("sample_rate", True, "sample_rate"), + ], +) +def test_click_fixture_rejects_boolean_numeric_evidence( + keyword: str, + value: bool, + message: str, +) -> None: + """Boolean tempo/duration/rate values must fail before allocation or loops.""" + kwargs = {keyword: cast(Any, value)} + with pytest.raises(ValueError, match=message): + render_click_track(**kwargs) + + +def test_wav_writer_rejects_boolean_sample_rate(tmp_path: Path) -> None: + """A Boolean sample rate must not be serialized as a 1 Hz WAV contract.""" + with pytest.raises(ValueError, match="sample_rate"): + write_pcm_wav( + tmp_path / "boolean-rate.wav", + np.zeros(4, dtype=np.float32), + cast(Any, True), + ) From e83f3ed14105d361e6a33295b432460cefe15f41 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:08:11 +0900 Subject: [PATCH 42/78] fix(accuracy): reject boolean fixture inputs --- .../bandscope_analysis/accuracy/fixtures.py | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index d2644bb54..b76cbf948 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -27,19 +27,23 @@ def render_c_major_triad( """Render a unit-peak C major triad as float32 PCM. Args: - duration_seconds: Finite positive fixture length in seconds. - sample_rate: Finite positive samples-per-second rate. + duration_seconds: Finite positive non-Boolean fixture length in seconds. + sample_rate: Finite positive non-Boolean samples-per-second rate. Returns: Mono float32 samples in ``[-1, 1]``. Raises: - ValueError: If duration or sample rate is non-finite or not positive. + ValueError: If duration or sample rate is Boolean, non-finite, or not positive. """ - if not np.isfinite(duration_seconds) or duration_seconds <= 0: - raise ValueError("duration_seconds must be finite and positive") - if not np.isfinite(sample_rate) or sample_rate <= 0: - raise ValueError("sample_rate must be finite and positive") + if ( + isinstance(duration_seconds, bool) + or not np.isfinite(duration_seconds) + or duration_seconds <= 0 + ): + raise ValueError("duration_seconds must be a finite positive non-Boolean number") + if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be a finite positive non-Boolean number") sample_count = int(duration_seconds * sample_rate) times = np.arange(sample_count, dtype=np.float32) / np.float32(sample_rate) @@ -59,23 +63,27 @@ def render_click_track( """Render a click track at a known tempo. Args: - bpm: Finite positive true tempo in beats per minute. - duration_seconds: Finite positive fixture length in seconds. - sample_rate: Finite positive samples-per-second rate. + bpm: Finite positive non-Boolean true tempo in beats per minute. + duration_seconds: Finite positive non-Boolean fixture length in seconds. + sample_rate: Finite positive non-Boolean samples-per-second rate. Returns: Mono float32 samples with a decaying click on each beat. Raises: - ValueError: If tempo, duration, or sample rate is non-finite or not - positive. + ValueError: If tempo, duration, or sample rate is Boolean, non-finite, + or not positive. """ - if not np.isfinite(bpm) or bpm <= 0: - raise ValueError("bpm must be finite and positive") - if not np.isfinite(duration_seconds) or duration_seconds <= 0: - raise ValueError("duration_seconds must be finite and positive") - if not np.isfinite(sample_rate) or sample_rate <= 0: - raise ValueError("sample_rate must be finite and positive") + if isinstance(bpm, bool) or not np.isfinite(bpm) or bpm <= 0: + raise ValueError("bpm must be a finite positive non-Boolean number") + if ( + isinstance(duration_seconds, bool) + or not np.isfinite(duration_seconds) + or duration_seconds <= 0 + ): + raise ValueError("duration_seconds must be a finite positive non-Boolean number") + if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be a finite positive non-Boolean number") sample_count = int(duration_seconds * sample_rate) audio = np.zeros(sample_count, dtype=np.float32) @@ -108,16 +116,16 @@ def write_pcm_wav(path: Path, audio: NDArray[np.floating], sample_rate: int) -> Args: path: Destination path. Parent directories are created. audio: Mono PCM samples. - sample_rate: Finite positive samples-per-second rate used to write the file. + sample_rate: Finite positive non-Boolean samples-per-second rate used to write the file. Returns: Lowercase hex SHA-256 of the written file. Raises: - ValueError: If the sample rate is non-finite or not positive. + ValueError: If the sample rate is Boolean, non-finite, or not positive. """ - if not np.isfinite(sample_rate) or sample_rate <= 0: - raise ValueError("sample_rate must be finite and positive") + if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be a finite positive non-Boolean number") path.parent.mkdir(parents=True, exist_ok=True) sf.write(path, np.asarray(audio, dtype=np.float32), sample_rate) return hashlib.sha256(path.read_bytes()).hexdigest() From b8707ffd9fa258ecdad90eb1f7dee000aa193429 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:08:44 +0900 Subject: [PATCH 43/78] docs(changelog): record boolean fixture guards --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2084d5dd6..930cf98b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject Boolean fixture duration, tempo, and sample-rate inputs so Python `True`/`False` values cannot become numeric real-audio acceptance authority or a 1 Hz WAV contract. - Reject Boolean chord timing and tempo metric inputs so `True`/`False` cannot masquerade as numeric MIR acceptance evidence through Python's `bool`-as-`int` semantics. - Reject blank or edge-whitespace-padded accuracy-report case IDs, metric names, and truth labels so portable acceptance evidence preserves exact registered identities. - Reject empty or reversed chord-estimate intervals before duration-weighted recall so malformed timing evidence cannot be silently ignored by an accuracy acceptance score. From 7f0c24c42296799190306d2dda8b8f6a6cf68f41 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:09:10 +0900 Subject: [PATCH 44/78] docs(accuracy): document boolean fixture boundary --- .../real-audio-accuracy-acceptance.md | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 98cd897e6..d25369630 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -7,6 +7,7 @@ evidence. ```bash uv run --project services/analysis-engine pytest \ services/analysis-engine/tests/test_accuracy_acceptance.py \ + services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py \ services/analysis-engine/tests/test_accuracy_manifest_version.py \ services/analysis-engine/tests/test_accuracy_metric_interval_contract.py ``` @@ -43,6 +44,12 @@ stem quality, or private-corpus readiness. and fail closed rather than being recorded as an ordinary miss. This matters in Python because `bool` is an integer subtype and would otherwise satisfy ordinary numeric comparisons. +- Fixture generation and WAV serialization accept only finite positive + non-Boolean duration, BPM, and sample-rate evidence. In Python, `True` would + otherwise act as numeric `1`, which could create a one-second fixture, a + one-BPM click contract, or a one-Hz WAV while still looking type-compatible at + runtime. Those values now fail closed before allocation, loop construction, + or file serialization. - Checksum mismatch fails closed on both file evaluators. Do not score a tampered file as a pass. - Machine-readable case reports are accepted only when the registered @@ -83,16 +90,18 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? ## Security Notes -- Attack surface: generated WAV bytes, SHA-256 digests, decoded PCM, recognizer - segment timings, tempo estimates, product-version provenance, and parsed - case-report mappings passed into the accuracy acceptance path. -- Trust boundary: untrusted audio, recognizer output, and manifests; trusted - repo-controlled fixture generators, true labels, metric definitions, - registered floors, and the repository product `VERSION`. +- Attack surface: generated WAV bytes, fixture duration/BPM/sample-rate inputs, + SHA-256 digests, decoded PCM, recognizer segment timings, tempo estimates, + product-version provenance, and parsed case-report mappings passed into the + accuracy acceptance path. +- Trust boundary: untrusted audio, runtime numeric evidence, recognizer output, + and manifests; trusted repo-controlled fixture definitions, true labels, + metric definitions, registered floors, and the repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major decode and before tempo scoring, overlap-safe chord duration, finite non-Boolean annotation/estimate timing, strictly increasing estimate - intervals, finite non-Boolean tempo metric inputs, strict SHA-256 syntax, + intervals, finite non-Boolean tempo metric inputs, finite positive + non-Boolean fixture duration/BPM/sample-rate inputs, strict SHA-256 syntax, finite-only report metric values, exact non-empty product-version provenance, bounded fixture durations, and no copyrighted commercial recordings. Fixture paths are pytest temp files; reports store only SHA-256 and labels, not @@ -102,6 +111,7 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? non-finite and Boolean chord annotation/estimate timing rejection, empty/reversed estimate interval rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / tolerance + rejection, non-finite and Boolean fixture generation/WAV sample-rate rejection, checksum mismatch through both file evaluators, malformed/non-hex manifest provenance, NaN/infinity report rejection, missing/empty product `VERSION` rejection, and silence must not pass as C major. \ No newline at end of file From e178558e8109d622c25f01d225e77d29f166b047 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:13:19 +0900 Subject: [PATCH 45/78] test(accuracy): fail closed on malformed PCM evidence --- .../tests/test_accuracy_pcm_input_contract.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_pcm_input_contract.py diff --git a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py new file mode 100644 index 000000000..64662a533 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py @@ -0,0 +1,27 @@ +"""Fail-closed decoded-PCM evidence contract for accuracy acceptance.""" + +from __future__ import annotations + +from typing import Any, cast + +import numpy as np +import pytest + +from bandscope_analysis.accuracy import evaluate_c_major_pcm, render_c_major_triad + + +@pytest.mark.parametrize("sample_rate", [True, 0, -1, float("nan"), float("inf")]) +def test_c_major_pcm_rejects_invalid_sample_rate(sample_rate: object) -> None: + """Invalid rate evidence must fail before production recognition or division.""" + audio = render_c_major_triad(duration_seconds=0.25) + + with pytest.raises(ValueError, match="sample_rate"): + evaluate_c_major_pcm(audio, cast(Any, sample_rate), "a" * 64) + + +def test_c_major_pcm_rejects_non_mono_audio() -> None: + """Accuracy acceptance must not reinterpret multichannel arrays as mono PCM.""" + stereo = np.zeros((32, 2), dtype=np.float32) + + with pytest.raises(ValueError, match="mono"): + evaluate_c_major_pcm(stereo, 22_050, "a" * 64) From 262adae0798a4c059e8225d86e6859082c24f861 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:13:39 +0900 Subject: [PATCH 46/78] fix(accuracy): validate decoded PCM evidence --- .../bandscope_analysis/accuracy/evaluate.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py index 8df7d414f..c10eca41f 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py @@ -30,17 +30,28 @@ def evaluate_c_major_pcm( """Score a C major triad through the production chord recognizer. Args: - audio: Decoded mono PCM. Do not pass a chroma matrix. - sample_rate: Sample rate of ``audio``. + audio: Decoded one-dimensional mono PCM. Do not pass a chroma matrix or + an unresolved multichannel buffer. + sample_rate: Finite positive non-Boolean sample rate of ``audio``. audio_sha256: Digest of the on-disk fixture that produced ``audio``. Returns: A case report whose metric is duration-weighted recall of ``C``. + + Raises: + ValueError: If PCM is not mono or the sample-rate evidence is Boolean, + non-finite, or non-positive. """ + if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: + raise ValueError("sample_rate must be a finite positive non-Boolean number") + samples = np.asarray(audio, dtype=np.float32) + if samples.ndim != 1: + raise ValueError("audio must be one-dimensional mono PCM") + recognizer = ChordRecognizer() - tracked = recognizer.recognize(np.asarray(audio, dtype=np.float32), sr=sample_rate) + tracked = recognizer.recognize(samples, sr=sample_rate) segments = [(item["start_time"], item["end_time"], item["chord"]) for item in tracked] - duration = float(len(audio) / sample_rate) + duration = float(len(samples) / sample_rate) recall = duration_weighted_chord_recall(segments, C_MAJOR_LABEL, 0.0, duration) return build_case_report( case_id="c-major-triad", From e62af576122036e6399c25298b508a923daaa9e8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:14:11 +0900 Subject: [PATCH 47/78] docs(changelog): record decoded PCM validation --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 930cf98b4..35e27d43d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject malformed decoded-PCM accuracy evidence before chord recognition: sample rates must be finite, positive, and non-Boolean, and the C-major acceptance buffer must be one-dimensional mono PCM. - Reject Boolean fixture duration, tempo, and sample-rate inputs so Python `True`/`False` values cannot become numeric real-audio acceptance authority or a 1 Hz WAV contract. - Reject Boolean chord timing and tempo metric inputs so `True`/`False` cannot masquerade as numeric MIR acceptance evidence through Python's `bool`-as-`int` semantics. - Reject blank or edge-whitespace-padded accuracy-report case IDs, metric names, and truth labels so portable acceptance evidence preserves exact registered identities. From be491335edd7a366be476fb14f23b30ca506f2a5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 06:14:34 +0900 Subject: [PATCH 48/78] docs(accuracy): document decoded PCM trust boundary --- .../real-audio-accuracy-acceptance.md | 56 +++++++++++-------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index d25369630..bb7d20172 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -8,6 +8,7 @@ evidence. uv run --project services/analysis-engine pytest \ services/analysis-engine/tests/test_accuracy_acceptance.py \ services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py \ + services/analysis-engine/tests/test_accuracy_pcm_input_contract.py \ services/analysis-engine/tests/test_accuracy_manifest_version.py \ services/analysis-engine/tests/test_accuracy_metric_interval_contract.py ``` @@ -34,8 +35,12 @@ stem quality, or private-corpus readiness. clipping, and each estimate interval must have a strictly increasing start/end pair; Boolean, NaN, infinite, empty, or reversed timing is invalid acceptance evidence and fails closed instead of being silently ignored or - allowed to fabricate covered duration. The metric family is WCSR/CSR - (Odekerken et al., 2021; Raffel et al., 2014). + allowed to fabricate covered duration. The decoded C-major evaluator also + requires one-dimensional mono PCM and a finite positive non-Boolean sample + rate before the production recognizer runs; malformed multichannel buffers or + invalid rate evidence are rejected rather than reinterpreted as acceptance + input. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et al., + 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. @@ -91,27 +96,30 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? ## Security Notes - Attack surface: generated WAV bytes, fixture duration/BPM/sample-rate inputs, - SHA-256 digests, decoded PCM, recognizer segment timings, tempo estimates, - product-version provenance, and parsed case-report mappings passed into the - accuracy acceptance path. -- Trust boundary: untrusted audio, runtime numeric evidence, recognizer output, - and manifests; trusted repo-controlled fixture definitions, true labels, - metric definitions, registered floors, and the repository product `VERSION`. + decoded PCM shape/sample-rate evidence, SHA-256 digests, recognizer segment + timings, tempo estimates, product-version provenance, and parsed case-report + mappings passed into the accuracy acceptance path. +- Trust boundary: untrusted audio, runtime numeric evidence, decoded buffer + structure, recognizer output, and manifests; trusted repo-controlled fixture + definitions, true labels, metric definitions, registered floors, and the + repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major - decode and before tempo scoring, overlap-safe chord duration, finite - non-Boolean annotation/estimate timing, strictly increasing estimate - intervals, finite non-Boolean tempo metric inputs, finite positive - non-Boolean fixture duration/BPM/sample-rate inputs, strict SHA-256 syntax, - finite-only report metric values, exact non-empty product-version provenance, - bounded fixture durations, and no copyrighted commercial recordings. Fixture - paths are pytest temp files; reports store only SHA-256 and labels, not - waveform bytes. + decode and before tempo scoring, one-dimensional mono PCM admission before + C-major recognition, finite positive non-Boolean decoded sample-rate evidence, + overlap-safe chord duration, finite non-Boolean annotation/estimate timing, + strictly increasing estimate intervals, finite non-Boolean tempo metric + inputs, finite positive non-Boolean fixture duration/BPM/sample-rate inputs, + strict SHA-256 syntax, finite-only report metric values, exact non-empty + product-version provenance, bounded fixture durations, and no copyrighted + commercial recordings. Fixture paths are pytest temp files; reports store + SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, - overlapping matching intervals do not double-count annotation duration, - non-finite and Boolean chord annotation/estimate timing rejection, - empty/reversed estimate interval rejection, silence-on-disk vs in-memory - triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / tolerance - rejection, non-finite and Boolean fixture generation/WAV sample-rate - rejection, checksum mismatch through both file evaluators, malformed/non-hex - manifest provenance, NaN/infinity report rejection, missing/empty product - `VERSION` rejection, and silence must not pass as C major. \ No newline at end of file + decoded-PCM non-mono rejection and invalid sample-rate rejection, overlapping + matching intervals do not double-count annotation duration, non-finite and + Boolean chord annotation/estimate timing rejection, empty/reversed estimate + interval rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, + non-finite and Boolean tempo estimate / truth / tolerance rejection, + non-finite and Boolean fixture generation/WAV sample-rate rejection, checksum + mismatch through both file evaluators, malformed/non-hex manifest provenance, + NaN/infinity report rejection, missing/empty product `VERSION` rejection, and + silence must not pass as C major. \ No newline at end of file From 0af265cb77e50d42a1603099704f494a1ad204a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:04:34 +0900 Subject: [PATCH 49/78] fix(accuracy): preserve positive BPM error contract --- .../analysis-engine/src/bandscope_analysis/accuracy/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index b76cbf948..e35b4d5df 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -75,7 +75,7 @@ def render_click_track( or not positive. """ if isinstance(bpm, bool) or not np.isfinite(bpm) or bpm <= 0: - raise ValueError("bpm must be a finite positive non-Boolean number") + raise ValueError("bpm must be positive, finite, and non-Boolean") if ( isinstance(duration_seconds, bool) or not np.isfinite(duration_seconds) From a2ebf30ba1be25c31a07f552a84597dc3bf88e42 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:05:43 +0900 Subject: [PATCH 50/78] test(accuracy): cover overflowing metric evidence --- .../test_accuracy_metric_overflow_contract.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py diff --git a/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py b/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py new file mode 100644 index 000000000..a6498154e --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py @@ -0,0 +1,23 @@ +"""Regression tests for accuracy-report numeric overflow authority.""" + +from __future__ import annotations + +import pytest + +from bandscope_analysis.accuracy import build_case_report, parse_case_report + + +def test_accuracy_report_rejects_integer_metric_that_overflows_float() -> None: + """Huge integer metrics must fail closed instead of escaping validation.""" + valid = build_case_report( + case_id="c-major-triad", + audio_sha256="a" * 64, + metric_name="duration_weighted_chord_recall", + metric_value=0.9, + passed=True, + true_label="C", + engine_version="0.1.3", + ) + + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": 10**400}) From 85a2b99ca37618573eb1bbdabe93f011c1376e34 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:19:17 +0900 Subject: [PATCH 51/78] test(accuracy): reject malformed decoded PCM evidence --- .../tests/test_accuracy_pcm_input_contract.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py index 64662a533..c234502fa 100644 --- a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py +++ b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py @@ -23,5 +23,20 @@ def test_c_major_pcm_rejects_non_mono_audio() -> None: """Accuracy acceptance must not reinterpret multichannel arrays as mono PCM.""" stereo = np.zeros((32, 2), dtype=np.float32) - with pytest.raises(ValueError, match="mono"): + with pytest.raises(ValueError, match="audio"): evaluate_c_major_pcm(stereo, 22_050, "a" * 64) + + +@pytest.mark.parametrize( + "audio", + [ + np.zeros(0, dtype=np.float32), + np.array([0.0, np.nan], dtype=np.float32), + np.array([0.0, np.inf], dtype=np.float32), + np.zeros(32, dtype=np.int16), + ], +) +def test_c_major_pcm_rejects_malformed_decoded_audio(audio: np.ndarray) -> None: + """Empty, non-finite, or non-floating decoded evidence must fail closed.""" + with pytest.raises(ValueError, match="audio"): + evaluate_c_major_pcm(audio, 22_050, "a" * 64) From 0c639e9ab584d7e48951c30d6c30a30ebf208a86 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:19:53 +0900 Subject: [PATCH 52/78] fix(accuracy): validate decoded PCM evidence --- .../bandscope_analysis/accuracy/evaluate.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py index c10eca41f..a6ee265e9 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py @@ -30,8 +30,9 @@ def evaluate_c_major_pcm( """Score a C major triad through the production chord recognizer. Args: - audio: Decoded one-dimensional mono PCM. Do not pass a chroma matrix or - an unresolved multichannel buffer. + audio: Decoded one-dimensional non-empty finite floating-point mono PCM. + Do not pass a chroma matrix, integer payload, or unresolved + multichannel buffer. sample_rate: Finite positive non-Boolean sample rate of ``audio``. audio_sha256: Digest of the on-disk fixture that produced ``audio``. @@ -39,14 +40,20 @@ def evaluate_c_major_pcm( A case report whose metric is duration-weighted recall of ``C``. Raises: - ValueError: If PCM is not mono or the sample-rate evidence is Boolean, - non-finite, or non-positive. + ValueError: If PCM is empty, non-floating, non-finite, not mono, or the + sample-rate evidence is Boolean, non-finite, or non-positive. """ if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: raise ValueError("sample_rate must be a finite positive non-Boolean number") + if ( + not isinstance(audio, np.ndarray) + or audio.ndim != 1 + or audio.size == 0 + or not np.issubdtype(audio.dtype, np.floating) + or not np.isfinite(audio).all() + ): + raise ValueError("audio must be non-empty finite floating-point mono PCM") samples = np.asarray(audio, dtype=np.float32) - if samples.ndim != 1: - raise ValueError("audio must be one-dimensional mono PCM") recognizer = ChordRecognizer() tracked = recognizer.recognize(samples, sr=sample_rate) From 39b72f6a18745ae7c623f3e03b3e744220e8e396 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:20:38 +0900 Subject: [PATCH 53/78] docs(accuracy): harden decoded evidence contract --- .../real-audio-accuracy-acceptance.md | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index bb7d20172..d511e50d5 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -10,7 +10,8 @@ uv run --project services/analysis-engine pytest \ services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py \ services/analysis-engine/tests/test_accuracy_pcm_input_contract.py \ services/analysis-engine/tests/test_accuracy_manifest_version.py \ - services/analysis-engine/tests/test_accuracy_metric_interval_contract.py + services/analysis-engine/tests/test_accuracy_metric_interval_contract.py \ + services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py ``` ## Why this lock exists @@ -36,11 +37,12 @@ stem quality, or private-corpus readiness. start/end pair; Boolean, NaN, infinite, empty, or reversed timing is invalid acceptance evidence and fails closed instead of being silently ignored or allowed to fabricate covered duration. The decoded C-major evaluator also - requires one-dimensional mono PCM and a finite positive non-Boolean sample - rate before the production recognizer runs; malformed multichannel buffers or - invalid rate evidence are rejected rather than reinterpreted as acceptance - input. The metric family is WCSR/CSR (Odekerken et al., 2021; Raffel et al., - 2014). + requires a non-empty, one-dimensional, finite floating-point mono PCM array + and a finite positive non-Boolean sample rate before the production recognizer + runs. Empty arrays, integer arrays, NaN/infinite samples, malformed + multichannel buffers, or invalid rate evidence are rejected rather than + coerced or reinterpreted as acceptance input. The metric family is WCSR/CSR + (Odekerken et al., 2021; Raffel et al., 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. @@ -96,30 +98,31 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? ## Security Notes - Attack surface: generated WAV bytes, fixture duration/BPM/sample-rate inputs, - decoded PCM shape/sample-rate evidence, SHA-256 digests, recognizer segment - timings, tempo estimates, product-version provenance, and parsed case-report - mappings passed into the accuracy acceptance path. + decoded PCM shape/dtype/finiteness/sample-rate evidence, SHA-256 digests, + recognizer segment timings, tempo estimates, product-version provenance, and + parsed case-report mappings passed into the accuracy acceptance path. - Trust boundary: untrusted audio, runtime numeric evidence, decoded buffer structure, recognizer output, and manifests; trusted repo-controlled fixture definitions, true labels, metric definitions, registered floors, and the repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major - decode and before tempo scoring, one-dimensional mono PCM admission before - C-major recognition, finite positive non-Boolean decoded sample-rate evidence, - overlap-safe chord duration, finite non-Boolean annotation/estimate timing, - strictly increasing estimate intervals, finite non-Boolean tempo metric - inputs, finite positive non-Boolean fixture duration/BPM/sample-rate inputs, - strict SHA-256 syntax, finite-only report metric values, exact non-empty + decode and before tempo scoring, non-empty finite floating-point + one-dimensional mono PCM admission before C-major recognition, finite positive + non-Boolean decoded sample-rate evidence, overlap-safe chord duration, finite + non-Boolean annotation/estimate timing, strictly increasing estimate + intervals, finite non-Boolean tempo metric inputs, finite positive non-Boolean + fixture duration/BPM/sample-rate inputs, strict SHA-256 syntax, finite-only + report metric values including overflow rejection, exact non-empty product-version provenance, bounded fixture durations, and no copyrighted commercial recordings. Fixture paths are pytest temp files; reports store SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, - decoded-PCM non-mono rejection and invalid sample-rate rejection, overlapping - matching intervals do not double-count annotation duration, non-finite and - Boolean chord annotation/estimate timing rejection, empty/reversed estimate - interval rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, - non-finite and Boolean tempo estimate / truth / tolerance rejection, - non-finite and Boolean fixture generation/WAV sample-rate rejection, checksum - mismatch through both file evaluators, malformed/non-hex manifest provenance, - NaN/infinity report rejection, missing/empty product `VERSION` rejection, and - silence must not pass as C major. \ No newline at end of file + decoded-PCM empty/non-floating/non-finite/non-mono rejection and invalid + sample-rate rejection, overlapping matching intervals do not double-count + annotation duration, non-finite and Boolean chord annotation/estimate timing + rejection, empty/reversed estimate interval rejection, silence-on-disk vs + in-memory triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / + tolerance rejection, non-finite and Boolean fixture generation/WAV sample-rate + rejection, checksum mismatch through both file evaluators, + malformed/non-hex manifest provenance, NaN/infinity/overflow report rejection, + missing/empty product `VERSION` rejection, and silence must not pass as C major. From a6227230a97088558e1d85961a9dc3171148f1b2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:21:07 +0900 Subject: [PATCH 54/78] docs(accuracy): record evidence validation repairs --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35e27d43d..d796c0f0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ ### Fixed -- Reject malformed decoded-PCM accuracy evidence before chord recognition: sample rates must be finite, positive, and non-Boolean, and the C-major acceptance buffer must be one-dimensional mono PCM. +- Reject malformed decoded-PCM accuracy evidence before chord recognition: the C-major acceptance buffer must be a non-empty, finite, floating-point, one-dimensional mono array and sample rates must be finite, positive, and non-Boolean. +- Reject accuracy-report metric integers whose conversion overflows floating-point evidence instead of letting an `OverflowError` escape the manifest validation boundary. - Reject Boolean fixture duration, tempo, and sample-rate inputs so Python `True`/`False` values cannot become numeric real-audio acceptance authority or a 1 Hz WAV contract. - Reject Boolean chord timing and tempo metric inputs so `True`/`False` cannot masquerade as numeric MIR acceptance evidence through Python's `bool`-as-`int` semantics. - Reject blank or edge-whitespace-padded accuracy-report case IDs, metric names, and truth labels so portable acceptance evidence preserves exact registered identities. @@ -80,4 +81,4 @@ - `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. - `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. -- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). \ No newline at end of file +- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). From 522fce9b7f6e89205f5d086b08bdd8b876338457 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 08:05:04 +0900 Subject: [PATCH 55/78] test(accuracy): reject boolean metric evidence at report builder --- .../test_accuracy_metric_overflow_contract.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py b/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py index a6498154e..6177df77e 100644 --- a/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py +++ b/services/analysis-engine/tests/test_accuracy_metric_overflow_contract.py @@ -21,3 +21,17 @@ def test_accuracy_report_rejects_integer_metric_that_overflows_float() -> None: with pytest.raises(ValueError, match="metric_value"): parse_case_report({**valid, "metric_value": 10**400}) + + +def test_build_case_report_rejects_boolean_metric_evidence() -> None: + """Builder coercion must not turn Boolean evidence into a numeric score.""" + with pytest.raises(ValueError, match="metric_value"): + build_case_report( + case_id="c-major-triad", + audio_sha256="a" * 64, + metric_name="duration_weighted_chord_recall", + metric_value=True, + passed=True, + true_label="C", + engine_version="0.1.3", + ) From 5a378415fc3149e52d3733f545eb844bfd591dee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 08:05:38 +0900 Subject: [PATCH 56/78] fix(accuracy): validate metric evidence before coercion --- .../src/bandscope_analysis/accuracy/manifest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py index 6fcf42b0f..b1d07634c 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/manifest.py @@ -88,11 +88,11 @@ def build_case_report( ValueError: If required report evidence is invalid or the default product ``VERSION`` cannot be resolved. """ - report: AccuracyCaseReport = { + report: dict[str, object] = { "case_id": case_id, "audio_sha256": audio_sha256, "metric_name": metric_name, - "metric_value": float(metric_value), + "metric_value": metric_value, "passed": passed, "engine_version": engine_version if engine_version is not None else read_product_version(), "true_label": true_label, From ddfdf76fe8cefa7ed9338d91ff5331fdab83e849 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 09:11:37 +0900 Subject: [PATCH 57/78] test(accuracy): bind checksum and scoring to same bytes --- .../test_accuracy_same_byte_provenance.py | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_same_byte_provenance.py diff --git a/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py b/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py new file mode 100644 index 000000000..ef027ce60 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py @@ -0,0 +1,86 @@ +"""Same-byte provenance regressions for real-audio accuracy evidence.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from bandscope_analysis.accuracy import ( + DEFAULT_CLICK_BPM, + DEFAULT_SAMPLE_RATE, + evaluate_c_major_file, + evaluate_click_tempo_file, + render_c_major_triad, + render_click_track, + write_pcm_wav, +) + + +def _replace_after_first_read( + monkeypatch: pytest.MonkeyPatch, + target: Path, + replacement_bytes: bytes, +) -> None: + """Replace ``target`` immediately after its first ``Path.read_bytes`` snapshot.""" + original_read_bytes = Path.read_bytes + replaced = False + + def read_bytes(path: Path) -> bytes: + nonlocal replaced + payload = original_read_bytes(path) + if path == target and not replaced: + replaced = True + path.write_bytes(replacement_bytes) + return payload + + monkeypatch.setattr(Path, "read_bytes", read_bytes) + + +def test_c_major_file_scores_the_bytes_that_satisfied_checksum( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Checksum evidence and chord scoring must bind to one immutable byte snapshot.""" + target = tmp_path / "c-major.wav" + digest = write_pcm_wav(target, render_c_major_triad(), DEFAULT_SAMPLE_RATE) + + replacement = tmp_path / "silence.wav" + write_pcm_wav( + replacement, + np.zeros(DEFAULT_SAMPLE_RATE * 3, dtype=np.float32), + DEFAULT_SAMPLE_RATE, + ) + _replace_after_first_read(monkeypatch, target, replacement.read_bytes()) + + report = evaluate_c_major_file(target, digest) + + assert report["audio_sha256"] == digest + assert report["passed"] is True + + +def test_click_file_scores_the_bytes_that_satisfied_checksum( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Checksum evidence and tempo scoring must bind to one immutable byte snapshot.""" + target = tmp_path / "click.wav" + digest = write_pcm_wav( + target, + render_click_track(bpm=DEFAULT_CLICK_BPM), + DEFAULT_SAMPLE_RATE, + ) + + replacement = tmp_path / "silence.wav" + write_pcm_wav( + replacement, + np.zeros(DEFAULT_SAMPLE_RATE * 8, dtype=np.float32), + DEFAULT_SAMPLE_RATE, + ) + _replace_after_first_read(monkeypatch, target, replacement.read_bytes()) + + report = evaluate_click_tempo_file(target, digest, DEFAULT_CLICK_BPM) + + assert report["audio_sha256"] == digest + assert report["passed"] is True From 85ef78a7adb6e005e1ceefe5415acfac183a38fc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 09:18:15 +0900 Subject: [PATCH 58/78] fix(accuracy): score the checksum-verified byte snapshot --- .../bandscope_analysis/accuracy/evaluate.py | 27 ++++++++++++++----- .../bandscope_analysis/accuracy/fixtures.py | 24 ++++++++++++++--- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py index a6ee265e9..91be30a88 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py @@ -2,7 +2,10 @@ from __future__ import annotations +from contextlib import contextmanager from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Iterator import numpy as np from numpy.typing import NDArray @@ -11,8 +14,8 @@ C_MAJOR_LABEL, DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, - assert_fixture_checksum, read_pcm_wav, + read_verified_fixture_bytes, ) from bandscope_analysis.accuracy.manifest import AccuracyCaseReport, build_case_report from bandscope_analysis.accuracy.metrics import duration_weighted_chord_recall, tempo_acc1 @@ -22,6 +25,16 @@ C_MAJOR_RECALL_FLOOR = 0.70 +@contextmanager +def _verified_fixture_path(audio_path: Path, expected_sha256: str) -> Iterator[Path]: + """Stage the exact checksum-verified bytes at an app-owned temporary path.""" + payload = read_verified_fixture_bytes(audio_path, expected_sha256) + with TemporaryDirectory(prefix="bandscope-accuracy-") as temp_dir: + verified_path = Path(temp_dir) / "verified.wav" + verified_path.write_bytes(payload) + yield verified_path + + def evaluate_c_major_pcm( audio: NDArray[np.floating], sample_rate: int = DEFAULT_SAMPLE_RATE, @@ -71,7 +84,7 @@ def evaluate_c_major_pcm( def evaluate_c_major_file(audio_path: Path, expected_sha256: str) -> AccuracyCaseReport: - """Checksum, decode, and score a C major WAV through ChordRecognizer. + """Checksum, decode, and score one immutable C major WAV snapshot. Args: audio_path: On-disk WAV written by ``write_pcm_wav``. @@ -80,8 +93,8 @@ def evaluate_c_major_file(audio_path: Path, expected_sha256: str) -> AccuracyCas Returns: A case report whose metric is duration-weighted recall of ``C``. """ - assert_fixture_checksum(audio_path, expected_sha256) - audio, sample_rate = read_pcm_wav(audio_path) + with _verified_fixture_path(audio_path, expected_sha256) as verified_path: + audio, sample_rate = read_pcm_wav(verified_path) return evaluate_c_major_pcm(audio, sample_rate, expected_sha256) @@ -90,7 +103,7 @@ def evaluate_click_tempo_file( expected_sha256: str, true_bpm: float = DEFAULT_CLICK_BPM, ) -> AccuracyCaseReport: - """Decode a click-track WAV with TemporalAnalyzer and score Acc1. + """Score one immutable checksum-verified click-track WAV snapshot. Args: audio_path: On-disk WAV written by ``write_pcm_wav``. @@ -100,8 +113,8 @@ def evaluate_click_tempo_file( Returns: A case report whose metric is 1.0 on Acc1 pass and 0.0 on Acc1 fail. """ - assert_fixture_checksum(audio_path, expected_sha256) - features = TemporalAnalyzer().analyze(audio_path) + with _verified_fixture_path(audio_path, expected_sha256) as verified_path: + features = TemporalAnalyzer().analyze(verified_path) passed = tempo_acc1(features["bpm"], true_bpm) return build_case_report( case_id="click-120-bpm", diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index e35b4d5df..bc3bfe668 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -152,6 +152,26 @@ def read_pcm_wav(path: Path) -> tuple[NDArray[np.float32], int]: return samples, int(sample_rate) +def read_verified_fixture_bytes(path: Path, expected_sha256: str) -> bytes: + """Read one immutable fixture snapshot and verify its registered digest. + + Args: + path: Existing WAV path. + expected_sha256: Lowercase hex digest recorded in the case manifest. + + Returns: + The exact bytes whose SHA-256 matched ``expected_sha256``. + + Raises: + ValueError: If the snapshot digest does not match. + """ + payload = path.read_bytes() + actual = hashlib.sha256(payload).hexdigest() + if actual != expected_sha256: + raise ValueError("Accuracy fixture checksum mismatch") + return payload + + def assert_fixture_checksum(path: Path, expected_sha256: str) -> None: """Fail closed when a fixture file does not match its registered digest. @@ -162,6 +182,4 @@ def assert_fixture_checksum(path: Path, expected_sha256: str) -> None: Raises: ValueError: If the file digest does not match. """ - actual = hashlib.sha256(path.read_bytes()).hexdigest() - if actual != expected_sha256: - raise ValueError("Accuracy fixture checksum mismatch") + read_verified_fixture_bytes(path, expected_sha256) From da7a2db8254a9f508b7cf15ec670669919897e0c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 09:23:37 +0900 Subject: [PATCH 59/78] test(accuracy): cover checksum compatibility boundary --- .../tests/test_accuracy_same_byte_provenance.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py b/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py index ef027ce60..771dbad07 100644 --- a/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py +++ b/services/analysis-engine/tests/test_accuracy_same_byte_provenance.py @@ -10,6 +10,7 @@ from bandscope_analysis.accuracy import ( DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, + assert_fixture_checksum, evaluate_c_major_file, evaluate_click_tempo_file, render_c_major_triad, @@ -38,6 +39,14 @@ def read_bytes(path: Path) -> bytes: monkeypatch.setattr(Path, "read_bytes", read_bytes) +def test_assert_fixture_checksum_accepts_matching_snapshot(tmp_path: Path) -> None: + """The compatibility checksum boundary must remain directly executable.""" + target = tmp_path / "matching.wav" + digest = write_pcm_wav(target, render_c_major_triad(), DEFAULT_SAMPLE_RATE) + + assert_fixture_checksum(target, digest) + + def test_c_major_file_scores_the_bytes_that_satisfied_checksum( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From b44af346648e8a331a6774c99c72ccbcdf477c52 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:06:46 +0900 Subject: [PATCH 60/78] test(accuracy): reject overflowed fixture timing evidence --- .../test_accuracy_boolean_fixture_inputs.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py index 05d615727..cd151af94 100644 --- a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py +++ b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py @@ -1,4 +1,4 @@ -"""Boolean evidence guards for deterministic real-audio fixtures.""" +"""Boolean and overflow guards for deterministic real-audio fixtures.""" from __future__ import annotations @@ -48,6 +48,21 @@ def test_click_fixture_rejects_boolean_numeric_evidence( render_click_track(**kwargs) +@pytest.mark.parametrize("factory", [render_c_major_triad, render_click_track]) +def test_fixture_rejects_nonfinite_scaled_sample_count(factory: Any) -> None: + """Finite inputs whose product overflows must fail before allocation authority.""" + with pytest.raises(ValueError, match="sample count"): + factory(duration_seconds=1e308) + + +def test_click_fixture_rejects_tempo_whose_beat_interval_overflows() -> None: + """A finite positive tempo must not become infinite loop timing authority.""" + smallest_positive = np.nextafter(0.0, 1.0) + + with pytest.raises(ValueError, match="bpm"): + render_click_track(bpm=smallest_positive, duration_seconds=1.0) + + def test_wav_writer_rejects_boolean_sample_rate(tmp_path: Path) -> None: """A Boolean sample rate must not be serialized as a 1 Hz WAV contract.""" with pytest.raises(ValueError, match="sample_rate"): From bec94965f00945601d2da4e54bd80d64bf2cc46e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:11:09 +0900 Subject: [PATCH 61/78] fix(accuracy): reject derived fixture timing overflow --- .../bandscope_analysis/accuracy/fixtures.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index bc3bfe668..077866f2f 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -20,6 +20,14 @@ _CLICK_DECAY = 80.0 +def _fixture_sample_count(duration_seconds: float, sample_rate: int) -> int: + """Convert finite fixture timing evidence to a finite integer sample count.""" + scaled_sample_count = duration_seconds * sample_rate + if not np.isfinite(scaled_sample_count): + raise ValueError("fixture sample count must be finite") + return int(scaled_sample_count) + + def render_c_major_triad( duration_seconds: float = 3.0, sample_rate: int = DEFAULT_SAMPLE_RATE, @@ -34,7 +42,8 @@ def render_c_major_triad( Mono float32 samples in ``[-1, 1]``. Raises: - ValueError: If duration or sample rate is Boolean, non-finite, or not positive. + ValueError: If duration or sample rate is Boolean, non-finite, or not positive, + or if their derived sample count is non-finite. """ if ( isinstance(duration_seconds, bool) @@ -45,7 +54,7 @@ def render_c_major_triad( if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: raise ValueError("sample_rate must be a finite positive non-Boolean number") - sample_count = int(duration_seconds * sample_rate) + sample_count = _fixture_sample_count(duration_seconds, sample_rate) times = np.arange(sample_count, dtype=np.float32) / np.float32(sample_rate) mix = ( np.sin(2 * np.pi * np.float32(C4_HZ) * times) @@ -72,7 +81,7 @@ def render_click_track( Raises: ValueError: If tempo, duration, or sample rate is Boolean, non-finite, - or not positive. + or not positive, or if derived sample-count/beat timing is non-finite. """ if isinstance(bpm, bool) or not np.isfinite(bpm) or bpm <= 0: raise ValueError("bpm must be positive, finite, and non-Boolean") @@ -85,9 +94,12 @@ def render_click_track( if isinstance(sample_rate, bool) or not np.isfinite(sample_rate) or sample_rate <= 0: raise ValueError("sample_rate must be a finite positive non-Boolean number") - sample_count = int(duration_seconds * sample_rate) + sample_count = _fixture_sample_count(duration_seconds, sample_rate) audio = np.zeros(sample_count, dtype=np.float32) - interval_seconds = 60.0 / bpm + with np.errstate(over="ignore", divide="ignore", invalid="ignore"): + interval_seconds = float(np.divide(60.0, bpm)) + if not np.isfinite(interval_seconds): + raise ValueError("bpm must produce a finite beat interval") click_length = int(_CLICK_DURATION_SECONDS * sample_rate) if click_length > 0: click_times = np.arange(click_length, dtype=np.float32) / np.float32(sample_rate) From d2625a7b16187d18d02214132128abef28a08f47 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:14:21 +0900 Subject: [PATCH 62/78] test(accuracy): reject zero-sample fixture evidence --- .../tests/test_accuracy_boolean_fixture_inputs.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py index cd151af94..ad7142328 100644 --- a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py +++ b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py @@ -1,4 +1,4 @@ -"""Boolean and overflow guards for deterministic real-audio fixtures.""" +"""Boolean and derived-arithmetic guards for deterministic real-audio fixtures.""" from __future__ import annotations @@ -55,6 +55,15 @@ def test_fixture_rejects_nonfinite_scaled_sample_count(factory: Any) -> None: factory(duration_seconds=1e308) +@pytest.mark.parametrize("factory", [render_c_major_triad, render_click_track]) +def test_fixture_rejects_duration_shorter_than_one_sample(factory: Any) -> None: + """Positive durations that quantize to zero samples must fail as evidence.""" + sub_sample_duration = np.nextafter(0.0, 1.0) + + with pytest.raises(ValueError, match="sample count"): + factory(duration_seconds=sub_sample_duration) + + def test_click_fixture_rejects_tempo_whose_beat_interval_overflows() -> None: """A finite positive tempo must not become infinite loop timing authority.""" smallest_positive = np.nextafter(0.0, 1.0) From c1baa4718b50102f7d96c5afe7a9c3c873f45727 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:17:47 +0900 Subject: [PATCH 63/78] fix(accuracy): reject zero-sample fixture evidence --- .../src/bandscope_analysis/accuracy/fixtures.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index 077866f2f..3dcaa8bb1 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -21,10 +21,10 @@ def _fixture_sample_count(duration_seconds: float, sample_rate: int) -> int: - """Convert finite fixture timing evidence to a finite integer sample count.""" + """Convert fixture timing evidence to at least one finite integer sample.""" scaled_sample_count = duration_seconds * sample_rate - if not np.isfinite(scaled_sample_count): - raise ValueError("fixture sample count must be finite") + if not np.isfinite(scaled_sample_count) or scaled_sample_count < 1: + raise ValueError("fixture sample count must be finite and at least one sample") return int(scaled_sample_count) @@ -43,7 +43,7 @@ def render_c_major_triad( Raises: ValueError: If duration or sample rate is Boolean, non-finite, or not positive, - or if their derived sample count is non-finite. + or if their derived sample count is non-finite or below one sample. """ if ( isinstance(duration_seconds, bool) @@ -81,7 +81,8 @@ def render_click_track( Raises: ValueError: If tempo, duration, or sample rate is Boolean, non-finite, - or not positive, or if derived sample-count/beat timing is non-finite. + or not positive, if the derived sample count is non-finite or below + one sample, or if the derived beat interval is non-finite. """ if isinstance(bpm, bool) or not np.isfinite(bpm) or bpm <= 0: raise ValueError("bpm must be positive, finite, and non-Boolean") From 54756702c561b9de77b9098db968ade464e3e1b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:21:46 +0900 Subject: [PATCH 64/78] docs(accuracy): record derived fixture timing guards --- .../real-audio-accuracy-acceptance.md | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index d511e50d5..2cb3a9106 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -55,8 +55,11 @@ stem quality, or private-corpus readiness. non-Boolean duration, BPM, and sample-rate evidence. In Python, `True` would otherwise act as numeric `1`, which could create a one-second fixture, a one-BPM click contract, or a one-Hz WAV while still looking type-compatible at - runtime. Those values now fail closed before allocation, loop construction, - or file serialization. + runtime. Derived fixture timing must also remain representable: the computed + sample count must be finite and at least one sample, and the click-track beat + interval must be finite. Inputs that overflow those derived quantities or + quantize to a zero-sample fixture fail closed before allocation, loop + construction, or file serialization. - Checksum mismatch fails closed on both file evaluators. Do not score a tampered file as a pass. - Machine-readable case reports are accepted only when the registered @@ -111,11 +114,12 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? non-Boolean decoded sample-rate evidence, overlap-safe chord duration, finite non-Boolean annotation/estimate timing, strictly increasing estimate intervals, finite non-Boolean tempo metric inputs, finite positive non-Boolean - fixture duration/BPM/sample-rate inputs, strict SHA-256 syntax, finite-only - report metric values including overflow rejection, exact non-empty - product-version provenance, bounded fixture durations, and no copyrighted - commercial recordings. Fixture paths are pytest temp files; reports store - SHA-256 and labels, not waveform bytes. + fixture duration/BPM/sample-rate inputs, finite derived sample-count and beat- + interval admission, strict SHA-256 syntax, finite-only report metric values + including overflow rejection, exact non-empty product-version provenance, + bounded fixture durations, and no copyrighted commercial recordings. Fixture + paths are pytest temp files; reports store SHA-256 and labels, not waveform + bytes. - Test points: deterministic digest, C major recall after file decode, decoded-PCM empty/non-floating/non-finite/non-mono rejection and invalid sample-rate rejection, overlapping matching intervals do not double-count @@ -123,6 +127,7 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? rejection, empty/reversed estimate interval rejection, silence-on-disk vs in-memory triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / tolerance rejection, non-finite and Boolean fixture generation/WAV sample-rate - rejection, checksum mismatch through both file evaluators, + rejection, derived sample-count overflow and zero-sample rejection, beat- + interval overflow rejection, checksum mismatch through both file evaluators, malformed/non-hex manifest provenance, NaN/infinity/overflow report rejection, missing/empty product `VERSION` rejection, and silence must not pass as C major. From 77b7cb88af1be9a7857d1aca86900c3a2dcc9219 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:22:04 +0900 Subject: [PATCH 65/78] docs(changelog): record fixture timing evidence guards --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d796c0f0f..22d6a827f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject fixture timings whose derived sample count overflows or quantizes below one sample, and reject finite BPM inputs whose derived beat interval becomes non-finite, before they can drive real-audio accuracy allocation or loop authority. - Reject malformed decoded-PCM accuracy evidence before chord recognition: the C-major acceptance buffer must be a non-empty, finite, floating-point, one-dimensional mono array and sample rates must be finite, positive, and non-Boolean. - Reject accuracy-report metric integers whose conversion overflows floating-point evidence instead of letting an `OverflowError` escape the manifest validation boundary. - Reject Boolean fixture duration, tempo, and sample-rate inputs so Python `True`/`False` values cannot become numeric real-audio acceptance authority or a 1 Hz WAV contract. @@ -81,4 +82,4 @@ - `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. - `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. -- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). +- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). \ No newline at end of file From cf465aa34925e530ea1da57b919819b35731a9ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:33:36 +0900 Subject: [PATCH 66/78] test(accuracy): reject unrepresentable click timing --- .../tests/test_accuracy_boolean_fixture_inputs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py index ad7142328..5735ded11 100644 --- a/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py +++ b/services/analysis-engine/tests/test_accuracy_boolean_fixture_inputs.py @@ -72,6 +72,18 @@ def test_click_fixture_rejects_tempo_whose_beat_interval_overflows() -> None: render_click_track(bpm=smallest_positive, duration_seconds=1.0) +def test_click_fixture_rejects_tempo_shorter_than_one_sample() -> None: + """A click cadence faster than one sample must not alias acceptance evidence.""" + with pytest.raises(ValueError, match="beat interval"): + render_click_track(bpm=6_001.0, duration_seconds=0.02, sample_rate=100) + + +def test_click_fixture_rejects_sample_rate_too_low_for_one_click_sample() -> None: + """A click fixture must contain at least one sample of click evidence.""" + with pytest.raises(ValueError, match="click length"): + render_click_track(bpm=60.0, duration_seconds=0.1, sample_rate=99) + + def test_wav_writer_rejects_boolean_sample_rate(tmp_path: Path) -> None: """A Boolean sample rate must not be serialized as a 1 Hz WAV contract.""" with pytest.raises(ValueError, match="sample_rate"): From b0a0ac1937a8317096ca5cade3597b255d08022f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:39:08 +0900 Subject: [PATCH 67/78] fix(accuracy): reject unrepresentable click timing --- .../bandscope_analysis/accuracy/fixtures.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py index 3dcaa8bb1..ab6750825 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/fixtures.py @@ -82,7 +82,9 @@ def render_click_track( Raises: ValueError: If tempo, duration, or sample rate is Boolean, non-finite, or not positive, if the derived sample count is non-finite or below - one sample, or if the derived beat interval is non-finite. + one sample, if the derived beat interval is non-finite or shorter + than one sample, or if the click pulse itself is shorter than one + sample at the requested rate. """ if isinstance(bpm, bool) or not np.isfinite(bpm) or bpm <= 0: raise ValueError("bpm must be positive, finite, and non-Boolean") @@ -99,23 +101,29 @@ def render_click_track( audio = np.zeros(sample_count, dtype=np.float32) with np.errstate(over="ignore", divide="ignore", invalid="ignore"): interval_seconds = float(np.divide(60.0, bpm)) + interval_samples = float(np.multiply(interval_seconds, sample_rate)) if not np.isfinite(interval_seconds): raise ValueError("bpm must produce a finite beat interval") - click_length = int(_CLICK_DURATION_SECONDS * sample_rate) - if click_length > 0: - click_times = np.arange(click_length, dtype=np.float32) / np.float32(sample_rate) - click = ( - np.sin(2 * np.pi * np.float32(_CLICK_FREQUENCY_HZ) * click_times) - * np.exp(-click_times * np.float32(_CLICK_DECAY)) - ).astype(np.float32) - beat_time = 0.0 - while True: - start = int(beat_time * sample_rate) - if start >= sample_count: - break - end = min(sample_count, start + click_length) - audio[start:end] += click[: end - start] - beat_time += interval_seconds + if not np.isfinite(interval_samples) or interval_samples < 1: + raise ValueError("beat interval must be finite and at least one sample") + + click_sample_count = _CLICK_DURATION_SECONDS * sample_rate + if not np.isfinite(click_sample_count) or click_sample_count < 1: + raise ValueError("click length must be finite and at least one sample") + click_length = int(click_sample_count) + click_times = np.arange(click_length, dtype=np.float32) / np.float32(sample_rate) + click = ( + np.sin(2 * np.pi * np.float32(_CLICK_FREQUENCY_HZ) * click_times) + * np.exp(-click_times * np.float32(_CLICK_DECAY)) + ).astype(np.float32) + beat_time = 0.0 + while True: + start = int(beat_time * sample_rate) + if start >= sample_count: + break + end = min(sample_count, start + click_length) + audio[start:end] += click[: end - start] + beat_time += interval_seconds peak = float(np.max(np.abs(audio))) if peak > 0: From 7c93766413bd8d898fb5909c372d31347b8504d9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:39:54 +0900 Subject: [PATCH 68/78] docs(changelog): record representable click timing guard --- CHANGELOG.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d6a827f..03ec03ef6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Reject click-track accuracy fixtures whose beat interval or click pulse cannot be represented by at least one sample, preventing aliased or all-zero tempo evidence from entering acceptance runs. - Reject fixture timings whose derived sample count overflows or quantizes below one sample, and reject finite BPM inputs whose derived beat interval becomes non-finite, before they can drive real-audio accuracy allocation or loop authority. - Reject malformed decoded-PCM accuracy evidence before chord recognition: the C-major acceptance buffer must be a non-empty, finite, floating-point, one-dimensional mono array and sample rates must be finite, positive, and non-Boolean. - Reject accuracy-report metric integers whose conversion overflows floating-point evidence instead of letting an `OverflowError` escape the manifest validation boundary. @@ -77,9 +78,3 @@ - Issue #26: Finalized roadmap and prepared application for initial release ## [0.1.4] - 2026-05-15 - -### 추가됨 (Added) - -- `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. -- `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. -- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). \ No newline at end of file From 1e8e7cf7fcc53e98c85a1a544c1b29aabe3e6bbf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 10:40:37 +0900 Subject: [PATCH 69/78] docs(accuracy): define sample-representable click evidence --- .../real-audio-accuracy-acceptance.md | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 2cb3a9106..7a205237d 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -56,10 +56,12 @@ stem quality, or private-corpus readiness. otherwise act as numeric `1`, which could create a one-second fixture, a one-BPM click contract, or a one-Hz WAV while still looking type-compatible at runtime. Derived fixture timing must also remain representable: the computed - sample count must be finite and at least one sample, and the click-track beat - interval must be finite. Inputs that overflow those derived quantities or - quantize to a zero-sample fixture fail closed before allocation, loop - construction, or file serialization. + fixture sample count, beat interval, and click-pulse width must each be finite + and must resolve to at least one sample at the requested rate. This prevents a + nominally valid high BPM from placing multiple beats onto the same sample and + prevents a low sample rate from producing an all-zero “click” fixture. Inputs + that overflow or undersample those derived quantities fail closed before + allocation, loop construction, or file serialization. - Checksum mismatch fails closed on both file evaluators. Do not score a tampered file as a pass. - Machine-readable case reports are accepted only when the registered @@ -114,12 +116,12 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? non-Boolean decoded sample-rate evidence, overlap-safe chord duration, finite non-Boolean annotation/estimate timing, strictly increasing estimate intervals, finite non-Boolean tempo metric inputs, finite positive non-Boolean - fixture duration/BPM/sample-rate inputs, finite derived sample-count and beat- - interval admission, strict SHA-256 syntax, finite-only report metric values - including overflow rejection, exact non-empty product-version provenance, - bounded fixture durations, and no copyrighted commercial recordings. Fixture - paths are pytest temp files; reports store SHA-256 and labels, not waveform - bytes. + fixture duration/BPM/sample-rate inputs, finite derived fixture sample count, + beat interval, and click-pulse width with a one-sample minimum, strict SHA-256 + syntax, finite-only report metric values including overflow rejection, exact + non-empty product-version provenance, bounded fixture durations, and no + copyrighted commercial recordings. Fixture paths are pytest temp files; + reports store SHA-256 and labels, not waveform bytes. - Test points: deterministic digest, C major recall after file decode, decoded-PCM empty/non-floating/non-finite/non-mono rejection and invalid sample-rate rejection, overlapping matching intervals do not double-count @@ -128,6 +130,7 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? in-memory triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / tolerance rejection, non-finite and Boolean fixture generation/WAV sample-rate rejection, derived sample-count overflow and zero-sample rejection, beat- - interval overflow rejection, checksum mismatch through both file evaluators, - malformed/non-hex manifest provenance, NaN/infinity/overflow report rejection, - missing/empty product `VERSION` rejection, and silence must not pass as C major. + interval overflow and sub-one-sample rejection, sub-one-sample click-pulse + rejection, checksum mismatch through both file evaluators, malformed/non-hex + manifest provenance, NaN/infinity/overflow report rejection, missing/empty + product `VERSION` rejection, and silence must not pass as C major. From 32fb3831553fe894027212ca57b2fb88fecf43d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 11:05:37 +0900 Subject: [PATCH 70/78] test(accuracy): align zero-length click acceptance --- .../tests/test_accuracy_acceptance.py | 530 ++++++++---------- 1 file changed, 235 insertions(+), 295 deletions(-) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index d397e3d57..6313e0a25 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -1,165 +1,129 @@ -"""Real-audio accuracy acceptance for decoded PCM fixtures. - -These cases prove a buyer-visible claim: a known waveform written to disk, -decoded, and analyzed yields the expected chord or tempo. Mocked chroma -matrices are not acceptance evidence. -""" +"""Tier 1 deterministic real-audio accuracy acceptance tests.""" from __future__ import annotations +import json from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast import numpy as np import pytest -import soundfile as sf +from bandscope_analysis import VERSION from bandscope_analysis.accuracy import ( C_MAJOR_LABEL, DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, - build_case_report, + AccuracyCase, + AccuracyMetric, + AccuracyReport, + AccuracyThreshold, + assert_fixture_checksum, + build_accuracy_report, duration_weighted_chord_recall, - evaluate_c_major_file, - evaluate_c_major_pcm, - evaluate_click_tempo_file, - parse_case_report, - read_pcm_wav, - read_product_version, render_c_major_triad, render_click_track, + run_c_major_acceptance, + run_tempo_acceptance, tempo_acc1, write_pcm_wav, ) -from bandscope_analysis.accuracy.evaluate import C_MAJOR_RECALL_FLOOR -from bandscope_analysis.api import build_demo_rehearsal_song - - -def test_c_major_fixture_is_deterministic(tmp_path: Path) -> None: - """Two writes of the same triad must share one SHA-256 digest.""" - audio = render_c_major_triad() - first = write_pcm_wav(tmp_path / "a.wav", audio, DEFAULT_SAMPLE_RATE) - second = write_pcm_wav(tmp_path / "b.wav", audio, DEFAULT_SAMPLE_RATE) - assert first == second - assert len(first) == 64 +from bandscope_analysis.chords import ChordRecognizer +from bandscope_analysis.temporal import TemporalAnalyzer + + +def test_accuracy_case_requires_sha256_digest() -> None: + """Fixture manifests must pin their source bytes with a SHA-256 digest.""" + with pytest.raises(ValueError, match="sha256"): + AccuracyCase( + case_id="case-c-major", + source_kind="generated", + source_locator="generated://c-major", + expected_label="C", + duration_seconds=3.0, + sha256="", + license_id="CC0-1.0", + ) + + +def test_accuracy_case_rejects_boolean_duration() -> None: + """Boolean duration evidence must not be accepted as a numeric value.""" + with pytest.raises(ValueError, match="duration_seconds"): + AccuracyCase( + case_id="case-c-major", + source_kind="generated", + source_locator="generated://c-major", + expected_label="C", + duration_seconds=cast(Any, True), + sha256="a" * 64, + license_id="CC0-1.0", + ) + + +def test_accuracy_case_requires_canonical_sha256_digest() -> None: + """Fixture manifests must use one exact lowercase SHA-256 representation.""" + common = dict( + case_id="case-c-major", + source_kind="generated", + source_locator="generated://c-major", + expected_label="C", + duration_seconds=3.0, + license_id="CC0-1.0", + ) + for digest in ["g" * 64, "A" * 64, "a" * 63, "a" * 65]: + with pytest.raises(ValueError, match="sha256"): + AccuracyCase(sha256=digest, **common) + + +def test_accuracy_metric_rejects_boolean_values() -> None: + """Boolean values must not satisfy numeric accuracy evidence contracts.""" + with pytest.raises(ValueError, match="value"): + AccuracyMetric(name="metric", value=cast(Any, True), threshold=0.5, passed=True) + with pytest.raises(ValueError, match="threshold"): + AccuracyMetric(name="metric", value=0.5, threshold=cast(Any, False), passed=True) + + +def test_accuracy_metric_rejects_non_finite_values() -> None: + """Non-finite metrics must not become buyer-facing acceptance evidence.""" + for value in [np.nan, np.inf, -np.inf]: + with pytest.raises(ValueError, match="value"): + AccuracyMetric(name="metric", value=float(value), threshold=0.5, passed=False) + for threshold in [np.nan, np.inf, -np.inf]: + with pytest.raises(ValueError, match="threshold"): + AccuracyMetric(name="metric", value=0.5, threshold=float(threshold), passed=False) + + +def test_accuracy_report_requires_exact_product_version() -> None: + """Accuracy reports must carry exact non-empty BandScope version provenance.""" + metric = AccuracyMetric(name="metric", value=1.0, threshold=0.5, passed=True) + with pytest.raises(ValueError, match="product_version"): + AccuracyReport(product_version="", metrics=(metric,)) + with pytest.raises(ValueError, match="product_version"): + AccuracyReport(product_version=" stale ", metrics=(metric,)) + + +def test_build_accuracy_report_preserves_product_version() -> None: + """The report builder must stamp the exact running product version.""" + report = build_accuracy_report( + [AccuracyMetric(name="metric", value=1.0, threshold=0.5, passed=True)] + ) + assert report.product_version == VERSION -def test_c_major_wav_recovers_c_after_file_decode(tmp_path: Path) -> None: - """A decoded C major WAV must recover C for most of the fixture duration.""" - audio = render_c_major_triad(duration_seconds=3.0) +def test_write_and_verify_fixture_checksum(tmp_path: Path) -> None: + """The on-disk bytes scored by acceptance must match the pinned digest.""" + audio = render_c_major_triad(duration_seconds=0.1) path = tmp_path / "c-major.wav" digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) - report = evaluate_c_major_file(path, digest) - assert report["true_label"] == C_MAJOR_LABEL - assert report["metric_name"] == "duration_weighted_chord_recall" - assert report["metric_value"] >= C_MAJOR_RECALL_FLOOR - assert report["passed"] is True - assert report["audio_sha256"] == digest - - -def test_click_wav_recovers_120_bpm_acc1(tmp_path: Path) -> None: - """A decoded 120 BPM click WAV must pass tempo Acc1.""" - audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) - path = tmp_path / "click-120.wav" - digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) - report = evaluate_click_tempo_file(path, digest, DEFAULT_CLICK_BPM) - assert report["passed"] is True - assert report["metric_name"] == "tempo_acc1" - assert report["true_label"] == "120 bpm" - - -def test_silence_does_not_pass_c_major_recall() -> None: - """Silence must not be reported as a passing C major acceptance case.""" - silence = np.zeros(DEFAULT_SAMPLE_RATE, dtype=np.float32) - report = evaluate_c_major_pcm(silence, DEFAULT_SAMPLE_RATE, "b" * 64) - assert report["passed"] is False - assert report["metric_value"] < C_MAJOR_RECALL_FLOOR + assert_fixture_checksum(path, digest) + path.write_bytes(path.read_bytes() + b"tamper") + with pytest.raises(ValueError, match="checksum"): + assert_fixture_checksum(path, digest) -def test_click_tempo_acc1_fails_when_true_tempo_is_wrong(tmp_path: Path) -> None: - """Acc1 must fail when the registered true tempo is not the click tempo.""" - audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) - path = tmp_path / "click-wrong-label.wav" - digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) - report = evaluate_click_tempo_file(path, digest, true_bpm=40.0) - assert report["passed"] is False - assert report["metric_value"] == 0.0 - - -def test_checksum_mismatch_fails_closed(tmp_path: Path) -> None: - """A tampered fixture must not be scored as a passing case.""" - click = render_click_track() - click_path = tmp_path / "click.wav" - write_pcm_wav(click_path, click, DEFAULT_SAMPLE_RATE) - with pytest.raises(ValueError, match="checksum mismatch"): - evaluate_click_tempo_file(click_path, "0" * 64) - - triad = render_c_major_triad() - triad_path = tmp_path / "c-major.wav" - write_pcm_wav(triad_path, triad, DEFAULT_SAMPLE_RATE) - with pytest.raises(ValueError, match="checksum mismatch"): - evaluate_c_major_file(triad_path, "0" * 64) - - -def test_c_major_file_decode_scores_disk_not_memory(tmp_path: Path) -> None: - """Silence on disk must fail even when a C major array exists in memory.""" - triad = render_c_major_triad(duration_seconds=3.0) - silence = np.zeros_like(triad) - path = tmp_path / "silence.wav" - digest = write_pcm_wav(path, silence, DEFAULT_SAMPLE_RATE) - report = evaluate_c_major_file(path, digest) - assert report["passed"] is False - assert report["metric_value"] < C_MAJOR_RECALL_FLOOR - memory_report = evaluate_c_major_pcm(triad, DEFAULT_SAMPLE_RATE, digest) - assert memory_report["passed"] is True - - -def test_read_pcm_wav_mixes_stereo_to_mono(tmp_path: Path) -> None: - """A stereo fixture must collapse to mono before scoring.""" - path = tmp_path / "stereo.wav" - stereo = np.column_stack([np.ones(8, dtype=np.float32), np.zeros(8, dtype=np.float32)]) - sf.write(path, stereo, DEFAULT_SAMPLE_RATE) - audio, sample_rate = read_pcm_wav(path) - assert sample_rate == DEFAULT_SAMPLE_RATE - assert audio.shape == (8,) - assert np.allclose(audio, 0.5, atol=1e-3) - - -def test_read_pcm_wav_rejects_empty_file(tmp_path: Path) -> None: - """An empty WAV must fail closed instead of scoring as a pass.""" - path = tmp_path / "empty.wav" - sf.write(path, np.zeros(0, dtype=np.float32), DEFAULT_SAMPLE_RATE) - with pytest.raises(ValueError, match="no samples"): - read_pcm_wav(path) - - -def test_pipeline_surfaces_c_on_active_lead_vocal() -> None: - """Unmocked assembly must put measured C on lead vocal when that stem is active.""" - audio = render_c_major_triad(duration_seconds=3.0) - silence = np.zeros_like(audio) - song = build_demo_rehearsal_song( - { - "stems": { - "vocals": audio * np.float32(0.35), - "bass": silence, - "drums": silence, - "other": audio, - }, - "sr": DEFAULT_SAMPLE_RATE, - "separation": {"duration_seconds": 3.0, "chunk_count": 1, "notes": "accuracy"}, - } - ) - assert song["id"] == "analyzed-song" - lead_chords = [ - role["harmony"]["chord"] - for section in song["sections"] - for role in section["roles"] - if role["id"] == "lead-vocal" - ] - assert C_MAJOR_LABEL in lead_chords - - -def test_render_helpers_reject_non_positive_inputs() -> None: - """Fixture helpers must refuse empty or reversed generation parameters.""" +def test_render_helpers_reject_invalid_inputs() -> None: + """Fixture generation must fail before invalid values reach DSP allocation.""" with pytest.raises(ValueError, match="duration_seconds"): render_c_major_triad(duration_seconds=0) with pytest.raises(ValueError, match="sample_rate"): @@ -180,11 +144,10 @@ def test_render_helpers_reject_non_finite_inputs() -> None: render_click_track(duration_seconds=np.inf) -def test_click_track_with_zero_length_click_stays_silent() -> None: - """A 1 Hz sample rate makes the click window empty and leaves silence.""" - audio = render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) - assert audio.shape == (1,) - assert float(audio[0]) == 0.0 +def test_click_track_rejects_zero_length_click_evidence() -> None: + """A sample rate that cannot represent one click sample must fail closed.""" + with pytest.raises(ValueError, match="click length"): + render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) def test_duration_weighted_recall_covers_overlap_and_misses() -> None: @@ -196,157 +159,134 @@ def test_duration_weighted_recall_covers_overlap_and_misses() -> None: def test_duration_weighted_recall_unions_overlapping_matching_estimates() -> None: - """Overlapping matching estimates must not count annotation time twice.""" - recall = duration_weighted_chord_recall( - [(0.0, 2.0, "C"), (1.0, 3.0, "C"), (4.0, 5.0, "C")], - "C", - 0.0, - 5.0, - ) - assert recall == pytest.approx(0.8) - + """Overlapping matching intervals must not double-count annotated time.""" + estimates = [(0.0, 2.0, "C"), (1.0, 3.0, "C")] + assert duration_weighted_chord_recall(estimates, "C", 0.0, 3.0) == 1.0 + + +def test_duration_weighted_recall_rejects_boolean_and_non_finite_timing() -> None: + """Malformed interval evidence must fail closed instead of entering arithmetic.""" + malformed_estimates: list[list[Any]] = [ + [True, 1.0, "C"], + [0.0, False, "C"], + [0.0, np.nan, "C"], + [0.0, np.inf, "C"], + [2.0, 1.0, "C"], + ] + for estimate in malformed_estimates: + with pytest.raises(ValueError, match="interval"): + duration_weighted_chord_recall(cast(Any, [estimate]), "C", 0.0, 3.0) -@pytest.mark.parametrize( - ("segments", "start_seconds", "end_seconds"), - [ - ([(np.nan, 2.0, "C")], 0.0, 4.0), - ([(0.0, np.inf, "C")], 0.0, 4.0), - ([(0.0, 2.0, "C")], np.nan, 4.0), - ([(0.0, 2.0, "C")], 0.0, np.inf), - ], -) -def test_duration_weighted_recall_rejects_non_finite_timing( - segments: list[tuple[float, float, str]], - start_seconds: float, - end_seconds: float, -) -> None: - """Non-finite estimate or annotation times must not fabricate recall.""" - with pytest.raises(ValueError, match="finite"): - duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) - - -@pytest.mark.parametrize( - ("segments", "start_seconds", "end_seconds"), - [ - ([(True, 2.0, "C")], 0.0, 4.0), - ([(0.0, True, "C")], 0.0, 4.0), - ([(0.0, 2.0, "C")], False, 4.0), - ([(0.0, 2.0, "C")], 0.0, True), - ], -) -def test_duration_weighted_recall_rejects_boolean_timing_evidence( - segments: list[tuple[float | bool, float | bool, str]], - start_seconds: float | bool, - end_seconds: float | bool, -) -> None: - """Boolean timestamps must not be accepted as numeric MIR timing evidence.""" - with pytest.raises(ValueError, match="times must be finite numbers"): - duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) # type: ignore[arg-type] - - -def test_tempo_acc1_window_and_guards() -> None: - """Acc1 must accept a 4% window and reject octave errors and bad inputs.""" - assert tempo_acc1(120.0, 120.0) is True - assert tempo_acc1(124.8, 120.0) is True - assert tempo_acc1(240.0, 120.0) is False - with pytest.raises(ValueError, match="true_bpm"): - tempo_acc1(120.0, 0.0) - with pytest.raises(ValueError, match="relative_tolerance"): - tempo_acc1(120.0, 120.0, relative_tolerance=-0.01) - - -@pytest.mark.parametrize( - ("estimated_bpm", "true_bpm", "relative_tolerance", "message"), - [ - (np.nan, 120.0, 0.04, "estimated_bpm"), - (np.inf, 120.0, 0.04, "estimated_bpm"), - (120.0, np.nan, 0.04, "true_bpm"), - (120.0, np.inf, 0.04, "true_bpm"), - (120.0, 120.0, np.nan, "relative_tolerance"), - (120.0, 120.0, np.inf, "relative_tolerance"), - ], -) -def test_tempo_acc1_rejects_non_finite_evidence( - estimated_bpm: float, - true_bpm: float, - relative_tolerance: float, - message: str, -) -> None: - """Non-finite estimate, truth, or tolerance must fail closed.""" - with pytest.raises(ValueError, match=message): - tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) - - -@pytest.mark.parametrize( - ("estimated_bpm", "true_bpm", "relative_tolerance", "message"), - [ - (True, 120.0, 0.04, "estimated_bpm"), - (120.0, True, 0.04, "true_bpm"), - (120.0, 120.0, True, "relative_tolerance"), - ], -) -def test_tempo_acc1_rejects_boolean_numeric_evidence( - estimated_bpm: float | bool, - true_bpm: float | bool, - relative_tolerance: float | bool, - message: str, -) -> None: - """Boolean values must not satisfy numeric Acc1 evidence contracts.""" - with pytest.raises(ValueError, match=message): - tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) # type: ignore[arg-type] - - -def test_parse_case_report_rejects_malformed_payloads() -> None: - """Manifest parsing must fail closed on missing or mistyped fields.""" - valid = build_case_report( - case_id="c-major-triad", - audio_sha256="a" * 64, - metric_name="duration_weighted_chord_recall", - metric_value=0.9, - passed=True, - true_label="C", - engine_version="0.1.3", + for start_seconds, end_seconds in [ + (cast(Any, True), 3.0), + (0.0, cast(Any, False)), + (np.nan, 3.0), + (0.0, np.inf), + ]: + with pytest.raises(ValueError, match="seconds"): + duration_weighted_chord_recall([], "C", start_seconds, end_seconds) + + +def test_tempo_acc1_handles_threshold_and_miss() -> None: + """Tempo acceptance uses the conventional percentage tolerance contract.""" + assert tempo_acc1(120.0, 120.0) == 1.0 + assert tempo_acc1(124.0, 120.0, tolerance=0.04) == 1.0 + assert tempo_acc1(126.0, 120.0, tolerance=0.04) == 0.0 + + +def test_tempo_acc1_rejects_boolean_and_non_finite_inputs() -> None: + """Malformed tempo evidence must fail before ratio arithmetic.""" + for estimated_bpm, truth_bpm, tolerance in [ + (cast(Any, True), 120.0, 0.04), + (120.0, cast(Any, False), 0.04), + (120.0, 120.0, cast(Any, True)), + (np.nan, 120.0, 0.04), + (120.0, np.inf, 0.04), + (120.0, 120.0, np.nan), + ]: + with pytest.raises(ValueError): + tempo_acc1(estimated_bpm, truth_bpm, tolerance=tolerance) + + +def test_c_major_acceptance_scores_production_recognizer(tmp_path: Path) -> None: + """Tier 1 C-major acceptance must score decoded fixture bytes via production code.""" + report = run_c_major_acceptance(tmp_path) + assert report.product_version == VERSION + assert report.passed + assert report.metrics[0].name == "c_major_duration_weighted_recall" + assert report.metrics[0].value >= report.metrics[0].threshold + + +def test_tempo_acceptance_scores_production_temporal_analyzer(tmp_path: Path) -> None: + """Tier 1 tempo acceptance must score decoded fixture bytes via production code.""" + report = run_tempo_acceptance(tmp_path) + assert report.product_version == VERSION + assert report.passed + assert report.metrics[0].name == "tempo_acc1" + assert report.metrics[0].value >= report.metrics[0].threshold + + +def test_c_major_file_decode_scores_disk_not_memory(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Acceptance must analyze decoded on-disk bytes rather than the render buffer.""" + from bandscope_analysis.accuracy import evaluate as evaluate_module + + original_read = evaluate_module.read_pcm_wav + + def corrupt_decoded_audio(path: Path) -> tuple[np.ndarray[Any, Any], int]: + decoded, sample_rate = original_read(path) + return np.zeros_like(decoded), sample_rate + + monkeypatch.setattr(evaluate_module, "read_pcm_wav", corrupt_decoded_audio) + report = run_c_major_acceptance(tmp_path) + assert not report.passed + + +def test_acceptance_rejects_manifest_digest_mismatch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Acceptance must fail closed if the registered fixture digest drifts.""" + from bandscope_analysis.accuracy import evaluate as evaluate_module + + original_writer = evaluate_module.write_pcm_wav + + def wrong_digest(path: Path, audio: np.ndarray[Any, Any], sample_rate: int) -> str: + original_writer(path, audio, sample_rate) + return "0" * 64 + + monkeypatch.setattr(evaluate_module, "write_pcm_wav", wrong_digest) + with pytest.raises(ValueError, match="checksum"): + run_c_major_acceptance(tmp_path) + + +def test_acceptance_report_json_is_machine_readable(tmp_path: Path) -> None: + """Tier 1 evidence must serialize deterministically for CI artifact consumers.""" + report = run_tempo_acceptance(tmp_path) + payload = json.loads(report.to_json()) + assert payload["productVersion"] == VERSION + assert payload["passed"] is True + assert payload["metrics"][0]["name"] == "tempo_acc1" + + +def test_metric_threshold_model_rejects_invalid_bounds() -> None: + """Metric thresholds must remain finite and bounded to meaningful ranges.""" + with pytest.raises(ValueError, match="minimum"): + AccuracyThreshold(metric_name="metric", minimum=np.nan) + with pytest.raises(ValueError, match="minimum"): + AccuracyThreshold(metric_name="metric", minimum=cast(Any, True)) + + +def test_accuracy_evaluation_rejects_non_finite_metric_result(monkeypatch: pytest.MonkeyPatch) -> None: + """Acceptance must reject non-finite analyzer outputs before building evidence.""" + from bandscope_analysis.accuracy import evaluate as evaluate_module + + fake_case = AccuracyCase( + case_id="case", + source_kind="generated", + source_locator="generated://fixture", + expected_label=C_MAJOR_LABEL, + duration_seconds=1.0, + sha256="a" * 64, + license_id="CC0-1.0", ) - assert parse_case_report(valid)["passed"] is True - - with pytest.raises(ValueError, match="must be an object"): - parse_case_report(["not", "an", "object"]) - with pytest.raises(ValueError, match="missing"): - parse_case_report({"case_id": "only"}) - with pytest.raises(ValueError, match="case_id"): - parse_case_report({**valid, "case_id": ""}) - with pytest.raises(ValueError, match="audio_sha256"): - parse_case_report({**valid, "audio_sha256": "short"}) - with pytest.raises(ValueError, match="audio_sha256"): - parse_case_report({**valid, "audio_sha256": "g" * 64}) - with pytest.raises(ValueError, match="metric_name"): - parse_case_report({**valid, "metric_name": ""}) - with pytest.raises(ValueError, match="metric_value"): - parse_case_report({**valid, "metric_value": True}) - with pytest.raises(ValueError, match="metric_value"): - parse_case_report({**valid, "metric_value": "0.9"}) - with pytest.raises(ValueError, match="metric_value"): - parse_case_report({**valid, "metric_value": np.nan}) - with pytest.raises(ValueError, match="metric_value"): - parse_case_report({**valid, "metric_value": np.inf}) - with pytest.raises(ValueError, match="passed"): - parse_case_report({**valid, "passed": 1}) - with pytest.raises(ValueError, match="engine_version"): - parse_case_report({**valid, "engine_version": ""}) - with pytest.raises(ValueError, match="true_label"): - parse_case_report({**valid, "true_label": ""}) - - -def test_read_product_version_uses_version_file_and_fails_closed(tmp_path: Path) -> None: - """Version lookup must read VERSION and reject missing provenance.""" - versioned = tmp_path / "versioned" - versioned.mkdir() - (versioned / "VERSION").write_text("9.9.9\n", encoding="utf-8") - assert read_product_version(versioned) == "9.9.9" - assert read_product_version(versioned / "VERSION") == "9.9.9" - empty = tmp_path / "empty-tree" - empty.mkdir() - (empty / "VERSION").write_text(" \n", encoding="utf-8") - with pytest.raises(ValueError, match="VERSION"): - read_product_version(empty) - assert read_product_version() != "unknown" + fake_recognizer = SimpleNamespace(analyze=lambda *_args, **_kwargs: np.nan) + monkeypatch.setattr(evaluate_module, "ChordRecognizer", lambda: fake_recognizer) + with pytest.raises(ValueError): + evaluate_module.evaluate_c_major_case(fake_case, np.zeros(8, dtype=np.float32), 8) From e1a53a250f71a2f6cee1a6ac3ca87cfe3cd0e1ec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 11:06:51 +0900 Subject: [PATCH 71/78] test(accuracy): restore acceptance suite before focused repair --- .../tests/test_accuracy_acceptance.py | 530 ++++++++++-------- 1 file changed, 295 insertions(+), 235 deletions(-) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index 6313e0a25..d397e3d57 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -1,129 +1,165 @@ -"""Tier 1 deterministic real-audio accuracy acceptance tests.""" +"""Real-audio accuracy acceptance for decoded PCM fixtures. + +These cases prove a buyer-visible claim: a known waveform written to disk, +decoded, and analyzed yields the expected chord or tempo. Mocked chroma +matrices are not acceptance evidence. +""" from __future__ import annotations -import json from pathlib import Path -from types import SimpleNamespace -from typing import Any, cast import numpy as np import pytest +import soundfile as sf -from bandscope_analysis import VERSION from bandscope_analysis.accuracy import ( C_MAJOR_LABEL, DEFAULT_CLICK_BPM, DEFAULT_SAMPLE_RATE, - AccuracyCase, - AccuracyMetric, - AccuracyReport, - AccuracyThreshold, - assert_fixture_checksum, - build_accuracy_report, + build_case_report, duration_weighted_chord_recall, + evaluate_c_major_file, + evaluate_c_major_pcm, + evaluate_click_tempo_file, + parse_case_report, + read_pcm_wav, + read_product_version, render_c_major_triad, render_click_track, - run_c_major_acceptance, - run_tempo_acceptance, tempo_acc1, write_pcm_wav, ) -from bandscope_analysis.chords import ChordRecognizer -from bandscope_analysis.temporal import TemporalAnalyzer - - -def test_accuracy_case_requires_sha256_digest() -> None: - """Fixture manifests must pin their source bytes with a SHA-256 digest.""" - with pytest.raises(ValueError, match="sha256"): - AccuracyCase( - case_id="case-c-major", - source_kind="generated", - source_locator="generated://c-major", - expected_label="C", - duration_seconds=3.0, - sha256="", - license_id="CC0-1.0", - ) - - -def test_accuracy_case_rejects_boolean_duration() -> None: - """Boolean duration evidence must not be accepted as a numeric value.""" - with pytest.raises(ValueError, match="duration_seconds"): - AccuracyCase( - case_id="case-c-major", - source_kind="generated", - source_locator="generated://c-major", - expected_label="C", - duration_seconds=cast(Any, True), - sha256="a" * 64, - license_id="CC0-1.0", - ) - - -def test_accuracy_case_requires_canonical_sha256_digest() -> None: - """Fixture manifests must use one exact lowercase SHA-256 representation.""" - common = dict( - case_id="case-c-major", - source_kind="generated", - source_locator="generated://c-major", - expected_label="C", - duration_seconds=3.0, - license_id="CC0-1.0", - ) - for digest in ["g" * 64, "A" * 64, "a" * 63, "a" * 65]: - with pytest.raises(ValueError, match="sha256"): - AccuracyCase(sha256=digest, **common) - - -def test_accuracy_metric_rejects_boolean_values() -> None: - """Boolean values must not satisfy numeric accuracy evidence contracts.""" - with pytest.raises(ValueError, match="value"): - AccuracyMetric(name="metric", value=cast(Any, True), threshold=0.5, passed=True) - with pytest.raises(ValueError, match="threshold"): - AccuracyMetric(name="metric", value=0.5, threshold=cast(Any, False), passed=True) - - -def test_accuracy_metric_rejects_non_finite_values() -> None: - """Non-finite metrics must not become buyer-facing acceptance evidence.""" - for value in [np.nan, np.inf, -np.inf]: - with pytest.raises(ValueError, match="value"): - AccuracyMetric(name="metric", value=float(value), threshold=0.5, passed=False) - for threshold in [np.nan, np.inf, -np.inf]: - with pytest.raises(ValueError, match="threshold"): - AccuracyMetric(name="metric", value=0.5, threshold=float(threshold), passed=False) - - -def test_accuracy_report_requires_exact_product_version() -> None: - """Accuracy reports must carry exact non-empty BandScope version provenance.""" - metric = AccuracyMetric(name="metric", value=1.0, threshold=0.5, passed=True) - with pytest.raises(ValueError, match="product_version"): - AccuracyReport(product_version="", metrics=(metric,)) - with pytest.raises(ValueError, match="product_version"): - AccuracyReport(product_version=" stale ", metrics=(metric,)) - - -def test_build_accuracy_report_preserves_product_version() -> None: - """The report builder must stamp the exact running product version.""" - report = build_accuracy_report( - [AccuracyMetric(name="metric", value=1.0, threshold=0.5, passed=True)] - ) - assert report.product_version == VERSION +from bandscope_analysis.accuracy.evaluate import C_MAJOR_RECALL_FLOOR +from bandscope_analysis.api import build_demo_rehearsal_song + + +def test_c_major_fixture_is_deterministic(tmp_path: Path) -> None: + """Two writes of the same triad must share one SHA-256 digest.""" + audio = render_c_major_triad() + first = write_pcm_wav(tmp_path / "a.wav", audio, DEFAULT_SAMPLE_RATE) + second = write_pcm_wav(tmp_path / "b.wav", audio, DEFAULT_SAMPLE_RATE) + assert first == second + assert len(first) == 64 -def test_write_and_verify_fixture_checksum(tmp_path: Path) -> None: - """The on-disk bytes scored by acceptance must match the pinned digest.""" - audio = render_c_major_triad(duration_seconds=0.1) +def test_c_major_wav_recovers_c_after_file_decode(tmp_path: Path) -> None: + """A decoded C major WAV must recover C for most of the fixture duration.""" + audio = render_c_major_triad(duration_seconds=3.0) path = tmp_path / "c-major.wav" digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) - assert_fixture_checksum(path, digest) - path.write_bytes(path.read_bytes() + b"tamper") - with pytest.raises(ValueError, match="checksum"): - assert_fixture_checksum(path, digest) + report = evaluate_c_major_file(path, digest) + assert report["true_label"] == C_MAJOR_LABEL + assert report["metric_name"] == "duration_weighted_chord_recall" + assert report["metric_value"] >= C_MAJOR_RECALL_FLOOR + assert report["passed"] is True + assert report["audio_sha256"] == digest + + +def test_click_wav_recovers_120_bpm_acc1(tmp_path: Path) -> None: + """A decoded 120 BPM click WAV must pass tempo Acc1.""" + audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) + path = tmp_path / "click-120.wav" + digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + report = evaluate_click_tempo_file(path, digest, DEFAULT_CLICK_BPM) + assert report["passed"] is True + assert report["metric_name"] == "tempo_acc1" + assert report["true_label"] == "120 bpm" + + +def test_silence_does_not_pass_c_major_recall() -> None: + """Silence must not be reported as a passing C major acceptance case.""" + silence = np.zeros(DEFAULT_SAMPLE_RATE, dtype=np.float32) + report = evaluate_c_major_pcm(silence, DEFAULT_SAMPLE_RATE, "b" * 64) + assert report["passed"] is False + assert report["metric_value"] < C_MAJOR_RECALL_FLOOR -def test_render_helpers_reject_invalid_inputs() -> None: - """Fixture generation must fail before invalid values reach DSP allocation.""" +def test_click_tempo_acc1_fails_when_true_tempo_is_wrong(tmp_path: Path) -> None: + """Acc1 must fail when the registered true tempo is not the click tempo.""" + audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) + path = tmp_path / "click-wrong-label.wav" + digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + report = evaluate_click_tempo_file(path, digest, true_bpm=40.0) + assert report["passed"] is False + assert report["metric_value"] == 0.0 + + +def test_checksum_mismatch_fails_closed(tmp_path: Path) -> None: + """A tampered fixture must not be scored as a passing case.""" + click = render_click_track() + click_path = tmp_path / "click.wav" + write_pcm_wav(click_path, click, DEFAULT_SAMPLE_RATE) + with pytest.raises(ValueError, match="checksum mismatch"): + evaluate_click_tempo_file(click_path, "0" * 64) + + triad = render_c_major_triad() + triad_path = tmp_path / "c-major.wav" + write_pcm_wav(triad_path, triad, DEFAULT_SAMPLE_RATE) + with pytest.raises(ValueError, match="checksum mismatch"): + evaluate_c_major_file(triad_path, "0" * 64) + + +def test_c_major_file_decode_scores_disk_not_memory(tmp_path: Path) -> None: + """Silence on disk must fail even when a C major array exists in memory.""" + triad = render_c_major_triad(duration_seconds=3.0) + silence = np.zeros_like(triad) + path = tmp_path / "silence.wav" + digest = write_pcm_wav(path, silence, DEFAULT_SAMPLE_RATE) + report = evaluate_c_major_file(path, digest) + assert report["passed"] is False + assert report["metric_value"] < C_MAJOR_RECALL_FLOOR + memory_report = evaluate_c_major_pcm(triad, DEFAULT_SAMPLE_RATE, digest) + assert memory_report["passed"] is True + + +def test_read_pcm_wav_mixes_stereo_to_mono(tmp_path: Path) -> None: + """A stereo fixture must collapse to mono before scoring.""" + path = tmp_path / "stereo.wav" + stereo = np.column_stack([np.ones(8, dtype=np.float32), np.zeros(8, dtype=np.float32)]) + sf.write(path, stereo, DEFAULT_SAMPLE_RATE) + audio, sample_rate = read_pcm_wav(path) + assert sample_rate == DEFAULT_SAMPLE_RATE + assert audio.shape == (8,) + assert np.allclose(audio, 0.5, atol=1e-3) + + +def test_read_pcm_wav_rejects_empty_file(tmp_path: Path) -> None: + """An empty WAV must fail closed instead of scoring as a pass.""" + path = tmp_path / "empty.wav" + sf.write(path, np.zeros(0, dtype=np.float32), DEFAULT_SAMPLE_RATE) + with pytest.raises(ValueError, match="no samples"): + read_pcm_wav(path) + + +def test_pipeline_surfaces_c_on_active_lead_vocal() -> None: + """Unmocked assembly must put measured C on lead vocal when that stem is active.""" + audio = render_c_major_triad(duration_seconds=3.0) + silence = np.zeros_like(audio) + song = build_demo_rehearsal_song( + { + "stems": { + "vocals": audio * np.float32(0.35), + "bass": silence, + "drums": silence, + "other": audio, + }, + "sr": DEFAULT_SAMPLE_RATE, + "separation": {"duration_seconds": 3.0, "chunk_count": 1, "notes": "accuracy"}, + } + ) + assert song["id"] == "analyzed-song" + lead_chords = [ + role["harmony"]["chord"] + for section in song["sections"] + for role in section["roles"] + if role["id"] == "lead-vocal" + ] + assert C_MAJOR_LABEL in lead_chords + + +def test_render_helpers_reject_non_positive_inputs() -> None: + """Fixture helpers must refuse empty or reversed generation parameters.""" with pytest.raises(ValueError, match="duration_seconds"): render_c_major_triad(duration_seconds=0) with pytest.raises(ValueError, match="sample_rate"): @@ -144,10 +180,11 @@ def test_render_helpers_reject_non_finite_inputs() -> None: render_click_track(duration_seconds=np.inf) -def test_click_track_rejects_zero_length_click_evidence() -> None: - """A sample rate that cannot represent one click sample must fail closed.""" - with pytest.raises(ValueError, match="click length"): - render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) +def test_click_track_with_zero_length_click_stays_silent() -> None: + """A 1 Hz sample rate makes the click window empty and leaves silence.""" + audio = render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) + assert audio.shape == (1,) + assert float(audio[0]) == 0.0 def test_duration_weighted_recall_covers_overlap_and_misses() -> None: @@ -159,134 +196,157 @@ def test_duration_weighted_recall_covers_overlap_and_misses() -> None: def test_duration_weighted_recall_unions_overlapping_matching_estimates() -> None: - """Overlapping matching intervals must not double-count annotated time.""" - estimates = [(0.0, 2.0, "C"), (1.0, 3.0, "C")] - assert duration_weighted_chord_recall(estimates, "C", 0.0, 3.0) == 1.0 - - -def test_duration_weighted_recall_rejects_boolean_and_non_finite_timing() -> None: - """Malformed interval evidence must fail closed instead of entering arithmetic.""" - malformed_estimates: list[list[Any]] = [ - [True, 1.0, "C"], - [0.0, False, "C"], - [0.0, np.nan, "C"], - [0.0, np.inf, "C"], - [2.0, 1.0, "C"], - ] - for estimate in malformed_estimates: - with pytest.raises(ValueError, match="interval"): - duration_weighted_chord_recall(cast(Any, [estimate]), "C", 0.0, 3.0) + """Overlapping matching estimates must not count annotation time twice.""" + recall = duration_weighted_chord_recall( + [(0.0, 2.0, "C"), (1.0, 3.0, "C"), (4.0, 5.0, "C")], + "C", + 0.0, + 5.0, + ) + assert recall == pytest.approx(0.8) - for start_seconds, end_seconds in [ - (cast(Any, True), 3.0), - (0.0, cast(Any, False)), - (np.nan, 3.0), - (0.0, np.inf), - ]: - with pytest.raises(ValueError, match="seconds"): - duration_weighted_chord_recall([], "C", start_seconds, end_seconds) - - -def test_tempo_acc1_handles_threshold_and_miss() -> None: - """Tempo acceptance uses the conventional percentage tolerance contract.""" - assert tempo_acc1(120.0, 120.0) == 1.0 - assert tempo_acc1(124.0, 120.0, tolerance=0.04) == 1.0 - assert tempo_acc1(126.0, 120.0, tolerance=0.04) == 0.0 - - -def test_tempo_acc1_rejects_boolean_and_non_finite_inputs() -> None: - """Malformed tempo evidence must fail before ratio arithmetic.""" - for estimated_bpm, truth_bpm, tolerance in [ - (cast(Any, True), 120.0, 0.04), - (120.0, cast(Any, False), 0.04), - (120.0, 120.0, cast(Any, True)), - (np.nan, 120.0, 0.04), - (120.0, np.inf, 0.04), - (120.0, 120.0, np.nan), - ]: - with pytest.raises(ValueError): - tempo_acc1(estimated_bpm, truth_bpm, tolerance=tolerance) - - -def test_c_major_acceptance_scores_production_recognizer(tmp_path: Path) -> None: - """Tier 1 C-major acceptance must score decoded fixture bytes via production code.""" - report = run_c_major_acceptance(tmp_path) - assert report.product_version == VERSION - assert report.passed - assert report.metrics[0].name == "c_major_duration_weighted_recall" - assert report.metrics[0].value >= report.metrics[0].threshold - - -def test_tempo_acceptance_scores_production_temporal_analyzer(tmp_path: Path) -> None: - """Tier 1 tempo acceptance must score decoded fixture bytes via production code.""" - report = run_tempo_acceptance(tmp_path) - assert report.product_version == VERSION - assert report.passed - assert report.metrics[0].name == "tempo_acc1" - assert report.metrics[0].value >= report.metrics[0].threshold - - -def test_c_major_file_decode_scores_disk_not_memory(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Acceptance must analyze decoded on-disk bytes rather than the render buffer.""" - from bandscope_analysis.accuracy import evaluate as evaluate_module - - original_read = evaluate_module.read_pcm_wav - - def corrupt_decoded_audio(path: Path) -> tuple[np.ndarray[Any, Any], int]: - decoded, sample_rate = original_read(path) - return np.zeros_like(decoded), sample_rate - - monkeypatch.setattr(evaluate_module, "read_pcm_wav", corrupt_decoded_audio) - report = run_c_major_acceptance(tmp_path) - assert not report.passed - - -def test_acceptance_rejects_manifest_digest_mismatch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Acceptance must fail closed if the registered fixture digest drifts.""" - from bandscope_analysis.accuracy import evaluate as evaluate_module - - original_writer = evaluate_module.write_pcm_wav - - def wrong_digest(path: Path, audio: np.ndarray[Any, Any], sample_rate: int) -> str: - original_writer(path, audio, sample_rate) - return "0" * 64 - - monkeypatch.setattr(evaluate_module, "write_pcm_wav", wrong_digest) - with pytest.raises(ValueError, match="checksum"): - run_c_major_acceptance(tmp_path) - - -def test_acceptance_report_json_is_machine_readable(tmp_path: Path) -> None: - """Tier 1 evidence must serialize deterministically for CI artifact consumers.""" - report = run_tempo_acceptance(tmp_path) - payload = json.loads(report.to_json()) - assert payload["productVersion"] == VERSION - assert payload["passed"] is True - assert payload["metrics"][0]["name"] == "tempo_acc1" - - -def test_metric_threshold_model_rejects_invalid_bounds() -> None: - """Metric thresholds must remain finite and bounded to meaningful ranges.""" - with pytest.raises(ValueError, match="minimum"): - AccuracyThreshold(metric_name="metric", minimum=np.nan) - with pytest.raises(ValueError, match="minimum"): - AccuracyThreshold(metric_name="metric", minimum=cast(Any, True)) - - -def test_accuracy_evaluation_rejects_non_finite_metric_result(monkeypatch: pytest.MonkeyPatch) -> None: - """Acceptance must reject non-finite analyzer outputs before building evidence.""" - from bandscope_analysis.accuracy import evaluate as evaluate_module - - fake_case = AccuracyCase( - case_id="case", - source_kind="generated", - source_locator="generated://fixture", - expected_label=C_MAJOR_LABEL, - duration_seconds=1.0, - sha256="a" * 64, - license_id="CC0-1.0", + +@pytest.mark.parametrize( + ("segments", "start_seconds", "end_seconds"), + [ + ([(np.nan, 2.0, "C")], 0.0, 4.0), + ([(0.0, np.inf, "C")], 0.0, 4.0), + ([(0.0, 2.0, "C")], np.nan, 4.0), + ([(0.0, 2.0, "C")], 0.0, np.inf), + ], +) +def test_duration_weighted_recall_rejects_non_finite_timing( + segments: list[tuple[float, float, str]], + start_seconds: float, + end_seconds: float, +) -> None: + """Non-finite estimate or annotation times must not fabricate recall.""" + with pytest.raises(ValueError, match="finite"): + duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) + + +@pytest.mark.parametrize( + ("segments", "start_seconds", "end_seconds"), + [ + ([(True, 2.0, "C")], 0.0, 4.0), + ([(0.0, True, "C")], 0.0, 4.0), + ([(0.0, 2.0, "C")], False, 4.0), + ([(0.0, 2.0, "C")], 0.0, True), + ], +) +def test_duration_weighted_recall_rejects_boolean_timing_evidence( + segments: list[tuple[float | bool, float | bool, str]], + start_seconds: float | bool, + end_seconds: float | bool, +) -> None: + """Boolean timestamps must not be accepted as numeric MIR timing evidence.""" + with pytest.raises(ValueError, match="times must be finite numbers"): + duration_weighted_chord_recall(segments, "C", start_seconds, end_seconds) # type: ignore[arg-type] + + +def test_tempo_acc1_window_and_guards() -> None: + """Acc1 must accept a 4% window and reject octave errors and bad inputs.""" + assert tempo_acc1(120.0, 120.0) is True + assert tempo_acc1(124.8, 120.0) is True + assert tempo_acc1(240.0, 120.0) is False + with pytest.raises(ValueError, match="true_bpm"): + tempo_acc1(120.0, 0.0) + with pytest.raises(ValueError, match="relative_tolerance"): + tempo_acc1(120.0, 120.0, relative_tolerance=-0.01) + + +@pytest.mark.parametrize( + ("estimated_bpm", "true_bpm", "relative_tolerance", "message"), + [ + (np.nan, 120.0, 0.04, "estimated_bpm"), + (np.inf, 120.0, 0.04, "estimated_bpm"), + (120.0, np.nan, 0.04, "true_bpm"), + (120.0, np.inf, 0.04, "true_bpm"), + (120.0, 120.0, np.nan, "relative_tolerance"), + (120.0, 120.0, np.inf, "relative_tolerance"), + ], +) +def test_tempo_acc1_rejects_non_finite_evidence( + estimated_bpm: float, + true_bpm: float, + relative_tolerance: float, + message: str, +) -> None: + """Non-finite estimate, truth, or tolerance must fail closed.""" + with pytest.raises(ValueError, match=message): + tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) + + +@pytest.mark.parametrize( + ("estimated_bpm", "true_bpm", "relative_tolerance", "message"), + [ + (True, 120.0, 0.04, "estimated_bpm"), + (120.0, True, 0.04, "true_bpm"), + (120.0, 120.0, True, "relative_tolerance"), + ], +) +def test_tempo_acc1_rejects_boolean_numeric_evidence( + estimated_bpm: float | bool, + true_bpm: float | bool, + relative_tolerance: float | bool, + message: str, +) -> None: + """Boolean values must not satisfy numeric Acc1 evidence contracts.""" + with pytest.raises(ValueError, match=message): + tempo_acc1(estimated_bpm, true_bpm, relative_tolerance) # type: ignore[arg-type] + + +def test_parse_case_report_rejects_malformed_payloads() -> None: + """Manifest parsing must fail closed on missing or mistyped fields.""" + valid = build_case_report( + case_id="c-major-triad", + audio_sha256="a" * 64, + metric_name="duration_weighted_chord_recall", + metric_value=0.9, + passed=True, + true_label="C", + engine_version="0.1.3", ) - fake_recognizer = SimpleNamespace(analyze=lambda *_args, **_kwargs: np.nan) - monkeypatch.setattr(evaluate_module, "ChordRecognizer", lambda: fake_recognizer) - with pytest.raises(ValueError): - evaluate_module.evaluate_c_major_case(fake_case, np.zeros(8, dtype=np.float32), 8) + assert parse_case_report(valid)["passed"] is True + + with pytest.raises(ValueError, match="must be an object"): + parse_case_report(["not", "an", "object"]) + with pytest.raises(ValueError, match="missing"): + parse_case_report({"case_id": "only"}) + with pytest.raises(ValueError, match="case_id"): + parse_case_report({**valid, "case_id": ""}) + with pytest.raises(ValueError, match="audio_sha256"): + parse_case_report({**valid, "audio_sha256": "short"}) + with pytest.raises(ValueError, match="audio_sha256"): + parse_case_report({**valid, "audio_sha256": "g" * 64}) + with pytest.raises(ValueError, match="metric_name"): + parse_case_report({**valid, "metric_name": ""}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": True}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": "0.9"}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": np.nan}) + with pytest.raises(ValueError, match="metric_value"): + parse_case_report({**valid, "metric_value": np.inf}) + with pytest.raises(ValueError, match="passed"): + parse_case_report({**valid, "passed": 1}) + with pytest.raises(ValueError, match="engine_version"): + parse_case_report({**valid, "engine_version": ""}) + with pytest.raises(ValueError, match="true_label"): + parse_case_report({**valid, "true_label": ""}) + + +def test_read_product_version_uses_version_file_and_fails_closed(tmp_path: Path) -> None: + """Version lookup must read VERSION and reject missing provenance.""" + versioned = tmp_path / "versioned" + versioned.mkdir() + (versioned / "VERSION").write_text("9.9.9\n", encoding="utf-8") + assert read_product_version(versioned) == "9.9.9" + assert read_product_version(versioned / "VERSION") == "9.9.9" + empty = tmp_path / "empty-tree" + empty.mkdir() + (empty / "VERSION").write_text(" \n", encoding="utf-8") + with pytest.raises(ValueError, match="VERSION"): + read_product_version(empty) + assert read_product_version() != "unknown" From 304684a7422288cbe5d32bce1e59cd17a08eb16b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 11:07:34 +0900 Subject: [PATCH 72/78] test(accuracy): fail closed on unrepresentable click pulse --- .../analysis-engine/tests/test_accuracy_acceptance.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/analysis-engine/tests/test_accuracy_acceptance.py b/services/analysis-engine/tests/test_accuracy_acceptance.py index d397e3d57..069653d83 100644 --- a/services/analysis-engine/tests/test_accuracy_acceptance.py +++ b/services/analysis-engine/tests/test_accuracy_acceptance.py @@ -180,11 +180,10 @@ def test_render_helpers_reject_non_finite_inputs() -> None: render_click_track(duration_seconds=np.inf) -def test_click_track_with_zero_length_click_stays_silent() -> None: - """A 1 Hz sample rate makes the click window empty and leaves silence.""" - audio = render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) - assert audio.shape == (1,) - assert float(audio[0]) == 0.0 +def test_click_track_rejects_zero_length_click_evidence() -> None: + """A sample rate that cannot represent one click sample must fail closed.""" + with pytest.raises(ValueError, match="click length"): + render_click_track(bpm=60.0, duration_seconds=1.0, sample_rate=1) def test_duration_weighted_recall_covers_overlap_and_misses() -> None: From 9bd3edb942c5c6bec79d3a350f81a74aadd1ebc0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 02:12:26 -0700 Subject: [PATCH 73/78] fix(changelog): preserve released 0.1.4 history --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03ec03ef6..6911ebcbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,3 +78,9 @@ - Issue #26: Finalized roadmap and prepared application for initial release ## [0.1.4] - 2026-05-15 + +### 추가됨 (Added) + +- `ChordsFeature` (코드 분석) 화면에서 각 파트(Role)의 `transpositionPlan`(이조/조옮김 계획)을 표시하는 기능을 추가했습니다. +- `RangesFeature` (음역대 분석) 화면에서 겹침 경고(Overlap warning) 외에 해당 파트의 채보(Transcription) 가능 노드 수를 요약하여 보여주는 기능을 추가했습니다. +- 신규 UI 요소에 대한 100% 테스트 커버리지를 보장하는 단위 테스트를 추가했습니다 (`apps/desktop/src/features/chords/index.test.tsx`, `apps/desktop/src/features/ranges/index.test.tsx`). From ebaf44436f93dab9722f6dda1e12f2fa4bba5e6a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 02:14:10 -0700 Subject: [PATCH 74/78] test(accuracy): reject multichannel WAV evidence --- .../tests/test_accuracy_pcm_input_contract.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py index c234502fa..933446ebe 100644 --- a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py +++ b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py @@ -2,12 +2,14 @@ from __future__ import annotations +from pathlib import Path from typing import Any, cast import numpy as np import pytest +import soundfile as sf # type: ignore[import-untyped] -from bandscope_analysis.accuracy import evaluate_c_major_pcm, render_c_major_triad +from bandscope_analysis.accuracy import evaluate_c_major_pcm, read_pcm_wav, render_c_major_triad @pytest.mark.parametrize("sample_rate", [True, 0, -1, float("nan"), float("inf")]) @@ -27,6 +29,15 @@ def test_c_major_pcm_rejects_non_mono_audio() -> None: evaluate_c_major_pcm(stereo, 22_050, "a" * 64) +def test_read_pcm_wav_rejects_multichannel_fixture(tmp_path: Path) -> None: + """A checksum-eligible WAV must stay mono instead of being silently downmixed.""" + stereo_path = tmp_path / "stereo.wav" + sf.write(stereo_path, np.zeros((32, 2), dtype=np.float32), 22_050) + + with pytest.raises(ValueError, match="mono"): + read_pcm_wav(stereo_path) + + @pytest.mark.parametrize( "audio", [ From c8e47116b8e0ded5140d533f97e6de486fb65702 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 02:16:57 -0700 Subject: [PATCH 75/78] test(accuracy): preserve stereo decode contract --- .../tests/test_accuracy_pcm_input_contract.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py index 933446ebe..c234502fa 100644 --- a/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py +++ b/services/analysis-engine/tests/test_accuracy_pcm_input_contract.py @@ -2,14 +2,12 @@ from __future__ import annotations -from pathlib import Path from typing import Any, cast import numpy as np import pytest -import soundfile as sf # type: ignore[import-untyped] -from bandscope_analysis.accuracy import evaluate_c_major_pcm, read_pcm_wav, render_c_major_triad +from bandscope_analysis.accuracy import evaluate_c_major_pcm, render_c_major_triad @pytest.mark.parametrize("sample_rate", [True, 0, -1, float("nan"), float("inf")]) @@ -29,15 +27,6 @@ def test_c_major_pcm_rejects_non_mono_audio() -> None: evaluate_c_major_pcm(stereo, 22_050, "a" * 64) -def test_read_pcm_wav_rejects_multichannel_fixture(tmp_path: Path) -> None: - """A checksum-eligible WAV must stay mono instead of being silently downmixed.""" - stereo_path = tmp_path / "stereo.wav" - sf.write(stereo_path, np.zeros((32, 2), dtype=np.float32), 22_050) - - with pytest.raises(ValueError, match="mono"): - read_pcm_wav(stereo_path) - - @pytest.mark.parametrize( "audio", [ From 5df9fb1f7ed7ecbcb4b9544200f33924fca01eff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 02:18:31 -0700 Subject: [PATCH 76/78] docs(accuracy): align stereo decode evidence boundary --- .../real-audio-accuracy-acceptance.md | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/docs/doctoring/real-audio-accuracy-acceptance.md b/docs/doctoring/real-audio-accuracy-acceptance.md index 7a205237d..37e854adf 100644 --- a/docs/doctoring/real-audio-accuracy-acceptance.md +++ b/docs/doctoring/real-audio-accuracy-acceptance.md @@ -36,13 +36,15 @@ stem quality, or private-corpus readiness. clipping, and each estimate interval must have a strictly increasing start/end pair; Boolean, NaN, infinite, empty, or reversed timing is invalid acceptance evidence and fails closed instead of being silently ignored or - allowed to fabricate covered duration. The decoded C-major evaluator also - requires a non-empty, one-dimensional, finite floating-point mono PCM array - and a finite positive non-Boolean sample rate before the production recognizer - runs. Empty arrays, integer arrays, NaN/infinite samples, malformed - multichannel buffers, or invalid rate evidence are rejected rather than - coerced or reinterpreted as acceptance input. The metric family is WCSR/CSR - (Odekerken et al., 2021; Raffel et al., 2014). + allowed to fabricate covered duration. File-backed WAV evaluation accepts an + ordinary mono or multichannel WAV and deterministically downmixes decoded + multichannel samples to mono before the production recognizer runs. The direct + `evaluate_c_major_pcm` boundary is narrower: callers must provide a non-empty, + one-dimensional, finite floating-point mono PCM array and a finite positive + non-Boolean sample rate. Unresolved multichannel arrays passed directly, + integer arrays, NaN/infinite samples, or invalid rate evidence are rejected + rather than coerced at that direct evaluator boundary. The metric family is + WCSR/CSR (Odekerken et al., 2021; Raffel et al., 2014). - `click-120-bpm`: eight seconds of 120 BPM clicks decoded by `TemporalAnalyzer`. Pass when estimated tempo satisfies Acc1 at 4% (Schreiber & Müller, 2020). Acc1 does not credit half-time or double-time. @@ -111,24 +113,26 @@ Schreiber, H., & Müller, M. (2020). Music tempo estimation: Are we done yet? definitions, true labels, metric definitions, registered floors, and the repository product `VERSION`. - Mitigations: no network, no shell, checksum fail-closed before C-major - decode and before tempo scoring, non-empty finite floating-point - one-dimensional mono PCM admission before C-major recognition, finite positive - non-Boolean decoded sample-rate evidence, overlap-safe chord duration, finite - non-Boolean annotation/estimate timing, strictly increasing estimate - intervals, finite non-Boolean tempo metric inputs, finite positive non-Boolean - fixture duration/BPM/sample-rate inputs, finite derived fixture sample count, - beat interval, and click-pulse width with a one-sample minimum, strict SHA-256 - syntax, finite-only report metric values including overflow rejection, exact - non-empty product-version provenance, bounded fixture durations, and no - copyrighted commercial recordings. Fixture paths are pytest temp files; - reports store SHA-256 and labels, not waveform bytes. -- Test points: deterministic digest, C major recall after file decode, - decoded-PCM empty/non-floating/non-finite/non-mono rejection and invalid - sample-rate rejection, overlapping matching intervals do not double-count - annotation duration, non-finite and Boolean chord annotation/estimate timing - rejection, empty/reversed estimate interval rejection, silence-on-disk vs - in-memory triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / - tolerance rejection, non-finite and Boolean fixture generation/WAV sample-rate + decode and before tempo scoring, deterministic file-decoder downmix for + ordinary multichannel WAV input, non-empty finite floating-point + one-dimensional mono PCM admission at the direct C-major evaluator boundary, + finite positive non-Boolean decoded sample-rate evidence, overlap-safe chord + duration, finite non-Boolean annotation/estimate timing, strictly increasing + estimate intervals, finite non-Boolean tempo metric inputs, finite positive + non-Boolean fixture duration/BPM/sample-rate inputs, finite derived fixture + sample count, beat interval, and click-pulse width with a one-sample minimum, + strict SHA-256 syntax, finite-only report metric values including overflow + rejection, exact non-empty product-version provenance, bounded fixture + durations, and no copyrighted commercial recordings. Fixture paths are pytest + temp files; reports store SHA-256 and labels, not waveform bytes. +- Test points: deterministic digest, C major recall after file decode, ordinary + stereo WAV downmix at the file-decoder boundary, direct decoded-PCM + empty/non-floating/non-finite/non-mono rejection and invalid sample-rate + rejection, overlapping matching intervals do not double-count annotation + duration, non-finite and Boolean chord annotation/estimate timing rejection, + empty/reversed estimate interval rejection, silence-on-disk vs in-memory + triad, 120 BPM Acc1, non-finite and Boolean tempo estimate / truth / tolerance + rejection, non-finite and Boolean fixture generation/WAV sample-rate rejection, derived sample-count overflow and zero-sample rejection, beat- interval overflow and sub-one-sample rejection, sub-one-sample click-pulse rejection, checksum mismatch through both file evaluators, malformed/non-hex From 8d32d7f584200e7662f16cdf1ba62d41b348e605 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 02:19:19 -0700 Subject: [PATCH 77/78] test(accuracy): bind tempo case identity to truth --- .../test_accuracy_tempo_case_identity.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 services/analysis-engine/tests/test_accuracy_tempo_case_identity.py diff --git a/services/analysis-engine/tests/test_accuracy_tempo_case_identity.py b/services/analysis-engine/tests/test_accuracy_tempo_case_identity.py new file mode 100644 index 000000000..c39264638 --- /dev/null +++ b/services/analysis-engine/tests/test_accuracy_tempo_case_identity.py @@ -0,0 +1,26 @@ +"""Regression contract for tempo accuracy report identity.""" + +from __future__ import annotations + +from pathlib import Path + +from bandscope_analysis.accuracy import ( + DEFAULT_CLICK_BPM, + DEFAULT_SAMPLE_RATE, + evaluate_click_tempo_file, + render_click_track, + write_pcm_wav, +) + + +def test_click_report_case_id_matches_registered_true_tempo(tmp_path: Path) -> None: + """A non-default truth label must not be published under the 120 BPM case ID.""" + audio = render_click_track(bpm=DEFAULT_CLICK_BPM, duration_seconds=8.0) + path = tmp_path / "click-wrong-label.wav" + digest = write_pcm_wav(path, audio, DEFAULT_SAMPLE_RATE) + + report = evaluate_click_tempo_file(path, digest, true_bpm=40.0) + + assert report["case_id"] == "click-40-bpm" + assert report["true_label"] == "40 bpm" + assert report["passed"] is False From 40f138e00ece3d480b5433a5ff8e1d9026cd7ff2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 02:24:55 -0700 Subject: [PATCH 78/78] fix(accuracy): bind tempo case id to truth --- .../analysis-engine/src/bandscope_analysis/accuracy/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py index 91be30a88..8c5a41eb0 100644 --- a/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py +++ b/services/analysis-engine/src/bandscope_analysis/accuracy/evaluate.py @@ -117,7 +117,7 @@ def evaluate_click_tempo_file( features = TemporalAnalyzer().analyze(verified_path) passed = tempo_acc1(features["bpm"], true_bpm) return build_case_report( - case_id="click-120-bpm", + case_id=f"click-{true_bpm:g}-bpm", audio_sha256=expected_sha256, metric_name="tempo_acc1", metric_value=1.0 if passed else 0.0,