From 5c4f5a4e9a6b060d68b46c695444dd855b135a16 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 04:36:15 +0530 Subject: [PATCH 1/7] feat(benchmark): implement paired drone adaptation protocol and analysis (#265) --- configs/drone_distribution_shift.yaml | 50 ++ docs/research/TREATMENT_CARD.md | 69 +++ docs/research/adaptive_rl_hypothesis.md | 53 +- docs/research/issue-265.md | 92 +++ src/adaptive_rl/algorithms/__init__.py | 10 + src/adaptive_rl/algorithms/adaptation.py | 319 +++++++++++ src/adaptive_rl/benchmarking/__init__.py | 10 + .../benchmarking/adaptation_artifacts.py | 153 +++++ .../benchmarking/adaptation_runner.py | 542 ++++++++++++++++++ .../benchmarking/adaptation_runtime.py | 243 ++++++++ .../benchmarking/adaptation_statistics.py | 139 +++++ src/adaptive_rl/cli.py | 107 +++- src/adaptive_rl/config.py | 27 + src/adaptive_rl/environments/__init__.py | 16 + src/adaptive_rl/environments/drone.py | 60 +- src/adaptive_rl/protocol/__init__.py | 16 + src/adaptive_rl/protocol/adaptation.py | 163 ++++++ src/adaptive_rl/protocol/fork.py | 121 ++++ src/adaptive_rl/training/trainer.py | 29 +- tests/test_adaptation_artifacts.py | 72 +++ tests/test_adaptation_environment.py | 67 +++ tests/test_adaptation_smoke.py | 52 ++ tests/test_adaptation_statistics.py | 55 ++ tests/test_algorithm_adaptation.py | 177 ++++++ tests/test_protocol_adaptation_boundary.py | 86 +++ tests/test_protocol_fork.py | 63 ++ 26 files changed, 2756 insertions(+), 35 deletions(-) create mode 100644 configs/drone_distribution_shift.yaml create mode 100644 docs/research/TREATMENT_CARD.md create mode 100644 docs/research/issue-265.md create mode 100644 src/adaptive_rl/algorithms/adaptation.py create mode 100644 src/adaptive_rl/benchmarking/adaptation_artifacts.py create mode 100644 src/adaptive_rl/benchmarking/adaptation_runner.py create mode 100644 src/adaptive_rl/benchmarking/adaptation_runtime.py create mode 100644 src/adaptive_rl/benchmarking/adaptation_statistics.py create mode 100644 src/adaptive_rl/protocol/adaptation.py create mode 100644 src/adaptive_rl/protocol/fork.py create mode 100644 tests/test_adaptation_artifacts.py create mode 100644 tests/test_adaptation_environment.py create mode 100644 tests/test_adaptation_smoke.py create mode 100644 tests/test_adaptation_statistics.py create mode 100644 tests/test_algorithm_adaptation.py create mode 100644 tests/test_protocol_adaptation_boundary.py create mode 100644 tests/test_protocol_fork.py diff --git a/configs/drone_distribution_shift.yaml b/configs/drone_distribution_shift.yaml new file mode 100644 index 0000000..218b470 --- /dev/null +++ b/configs/drone_distribution_shift.yaml @@ -0,0 +1,50 @@ +# Issue #265: primary moderate TEST-B condition. Training receives only the +# nominal environment.parameters block. shift_parameters are held back until +# after training and the shared pre-shift evaluation. +name: "drone_online_adaptation" +seed: 31001 + +algorithm: + name: "ppo" + learning_rate: 0.0003 + gamma: 0.99 + batch_size: 128 + parameters: + n_steps: 1024 + n_epochs: 10 + clip_range: 0.2 + ent_coef: 0.01 + +environment: + name: "drone_disturbed" + max_steps: 300 + parameters: + bounds: [50.0, 50.0, 25.0] + num_obstacles: 8 + num_lidar_rays: 16 + linear_damping: 0.05 + max_acceleration: 4.0 + wind_speed: 0.5 + gust_sigma: 0.15 + gust_theta: 0.15 + wind_direction: [1.5, 0.5, 0.0] + +training: + total_timesteps: 60000 + checkpoint_freq: 0 + log_interval: 10 + +evaluation: + eval_episodes: 15 + deterministic: true + +adaptation_benchmark: + protocol_version: "2.0" + scenario: "TEST-B" + shift_parameters: + num_obstacles: 12 + wind_speed: 4.0 + gust_sigma: 0.6 + +output_dir: "artifacts/adaptation" +log_dir: "artifacts/adaptation/logs" diff --git a/docs/research/TREATMENT_CARD.md b/docs/research/TREATMENT_CARD.md new file mode 100644 index 0000000..adc5913 --- /dev/null +++ b/docs/research/TREATMENT_CARD.md @@ -0,0 +1,69 @@ +# Issue #265 Treatment Card + +**Status: frozen before benchmark execution.** This card defines the adaptation +treatment. Its SHA-256 is stored in every Issue #265 artifact. No benchmark +outcome may be used to change these choices. + +## Fixed treatment choices + +- **Scope:** adapt the already trained PPO or SAC policy using only transitions + collected in completed post-shift episodes. Base training configuration, + checkpoint, and environment interaction schedule are identical between arms. +- **Mechanism:** one cumulative, episode-bounded batch update at each boundary + B5 through B14. Each batch is the concatenation of all post-shift transitions + from episodes 1 through the boundary episode. The batch contains observation, + action, reward, next observation, terminated, and truncated values. No nominal + data is retained in or passed to the adaptation path. +- **PPO update:** store the behavior-policy log-probability and value estimate + with each transition at collection time, plus the behavior value of a + truncated transition's next observation. At each block, construct a fresh + Stable-Baselines3 `RolloutBuffer` from the permitted cumulative prefix, use + the configured `gamma` and `gae_lambda`, bootstrap truncated transitions + from their recorded next observation, and run the native clipped PPO update + for the config's `n_epochs` and `batch_size`. The configured entropy and + value-function coefficients are unchanged. Previously seen prefix data is + reused at later blocks with its stored behavior quantities. +- **SAC update:** construct a fresh Stable-Baselines3 `ReplayBuffer` for each + block from the permitted cumulative prefix; the training replay buffer is + never reused. Its capacity is `max(prefix_transition_count, configured + batch_size)`. Run native SAC training for the configured `gradient_steps` + and `batch_size`, retaining the configured learning rate, target update, and + entropy mechanism. A sampled minibatch may draw with replacement when it is + larger than the currently available transition count. Time-limit truncations + are marked according to SB3's timeout-mask convention. +- **Randomness:** each block seeds Python, NumPy, and Torch with its one + preregistered `update` seed while sampling/updating, then restores the caller's + RNG states. No environment step occurs in either adapter. +- **Learning rate, batch size, epochs/steps, and regularization:** all values + come from the base cell config; no additional optimizer or regularizer is + added. PPO's configured entropy/value terms and SAC's configured entropy + mechanism remain part of their respective native objectives. +- **Minimum data:** at least one complete episode (the current block's newly + completed episode) and at least one valid transition are required. +- **Buffer construction:** a fresh immutable snapshot is assembled at each + block boundary from the allowed prefix only. No shared/global replay buffer + is used. Transitions are never sampled from a future episode. +- **Update timing:** only after episode termination/truncation and before the + next reset; no updates at B15. The policy is constant during every episode. +- **Failure behavior:** any exception, non-finite loss/parameter/optimizer + value, invalid tensor, or shape change invalidates the replicate. The block is + rolled back before the failed replicate is recorded; execution does not + continue to another evaluation episode. +- **Diagnostics:** record pre/post SHA-256 model fingerprints, update seed, + visible episode indices, transition count, status, and global L2 parameter + delta. Zero delta is recorded as a non-mutating update, not successful + adaptation. + +The Fixed arm is an independent clone of the frozen checkpoint. It performs no +training or optimizer operation after the fork. Both arms use the same derived +post-shift episode seeds. A smoke run validates machinery only and is not +empirical evidence for the research hypothesis. + +## Implementation constraint + +Algorithm-specific recorded-data adapters are implemented for PPO and SAC. +The complete benchmark experiment runner, environment shift integration, +artifacts, and CLI are still required before the treatment may be run as an +Issue #265 experiment. The runner must fail explicitly for any configuration +that cannot provide the required PPO behavior quantities or recorded transition +data without extra environment interaction. diff --git a/docs/research/adaptive_rl_hypothesis.md b/docs/research/adaptive_rl_hypothesis.md index 74f5e15..b5442e7 100644 --- a/docs/research/adaptive_rl_hypothesis.md +++ b/docs/research/adaptive_rl_hypothesis.md @@ -9,23 +9,24 @@ This document is the normative, pre-specified experimental contract for evaluati > [!WARNING] > **Implementation Status** > -> **[VERIFIED-CODE]** Current repository infrastructure trains on a nominal scenario, freezes the resulting policy, fingerprints it, and rejects any mutation during evaluation — see `src/adaptive_rl/experiments/shift_runner.py:180-205` at the pinned commit. +> **[VERIFIED-CODE]** `src/adaptive_rl/benchmarking/adaptation_runner.py` now implements the Issue #265 drone TEST-B lifecycle: train once on nominal parameters, share the pre-shift and shock episodes, fork independent PPO/SAC policies, update only Adaptive between episodes B5–B14, and compute recovery from the paired trajectories. `tests/test_adaptation_smoke.py` exercises the CI-sized PPO path; both PPO and SAC real-environment smoke paths have been executed. > -> **[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** The online Adaptive treatment defined in §4–§5. No online update/adaptation harness exists in the repository. +> **[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Four non-drone primary cells do not have an Issue #265 shift environment/configuration in this checkout. They remain inconclusive; the six-cell family claim cannot be evaluated from the drone cell alone. > -> **[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** The per-episode derived seed schedule of §14 is not yet executed by any runner; the current runner evaluates the config-declared scenario seed sets (`shift_runner.py:198-204`). +> **[NO RESEARCH RESULT]** No full ten-replicate scientific benchmark has been collected. Smoke execution validates software behavior only and does not establish empirical superiority or statistical significance. > -> **PR #168 / Issue #98** is documentation plus this executable protocol mirror only. Nothing in this document should be read as claiming that any Adaptive cell is currently executable, that any result exists, or that the protocol has been validated on data. +> **PR #168 / Issue #98** introduced this protocol and its executable constants/seed/recovery/statistics mirror. Issue #265 adds the drone TEST-B harness. Nothing here claims a full research result or empirical validation. **Status tags used throughout** (a value is what its tag says, nothing more): | Tag | Meaning | |---|---| -| `[VERIFIED-CODE]` | Behavior verified by reading/reviewing repository source at the pinned commit | +| `[VERIFIED-CODE]` | Behavior verified in the current repository source; frozen scientific values remain pinned to the protocol version | +| `[SMOKE-VALIDATED]` | CI-sized or one-off smoke execution verified software behavior only; not research evidence | | `[VERIFIED-CONFIG]` | Value appears literally in the cited config file at the pinned commit | | `[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]` | Frozen requirement for a future harness; not present in the repository today | | `[FUTURE PROTOCOL VALUE]` | Pre-registered target that must be configured before that condition is executable | -| `[FUTURE DESIGN DECISION]` | Must be frozen in a signed-off artifact **before any data is collected** (see §5.3) | +| `[FUTURE DESIGN DECISION]` | Must be frozen in a signed-off artifact before that cell collects data | ## 2. Research Question @@ -65,7 +66,7 @@ The primary test is one-sided, consistent with H1 (§18.2). ### 4.2 Train-once, clone, fork design -**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Each replicate is executed as one shared segment followed by two arm segments: +**[SMOKE-VALIDATED]** The drone TEST-B runner executes each replicate as one shared segment followed by two arm segments: ``` TRAIN (seed = training_seed(i)) # one training run per replicate @@ -92,7 +93,7 @@ No running normalization statistics exist in the repository (no VecNormalize or ### 5.1 Block schedule -**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Exactly `N_update = 10` update blocks exist, executed strictly between the termination of episode $k$ and the reset of episode $k+1$, for $k = 5, 6, \dots, 14$ (block identifiers $B_5, \dots, B_{14}$): +**[SMOKE-VALIDATED]** Exactly `N_update = 10` update blocks execute strictly between the termination of episode $k$ and the reset of episode $k+1$, for $k = 5, 6, \dots, 14$ (block identifiers $B_5, \dots, B_{14}$): | Block | Executes between | Visible data (frozen) | |---|---|---| @@ -115,7 +116,7 @@ An update block completes (or fails) **before** the next episode reset. A failed ### 5.3 What is frozen vs. what must still be frozen before execution * **Frozen now**: schedule of §5.1, data visibility, freeze windows, identical-seed rule, logging requirements, all constants in §24. -* **[FUTURE DESIGN DECISION]** The adaptation algorithm itself (loss, epochs, learning rate, batch construction, buffer capacity) is not specified here because no harness exists to validate a specification against. Before any data is collected, a **Treatment Card** must be written and frozen (commit SHA recorded in the results artifact) that specifies these details under these constraints: (a) base-algorithm hyperparameters unchanged from the cell config except parameters declared in the Card; (b) no access to test/evaluation outcomes from episodes > k when executing $B_k$; (c) no seed other than the derived ones; (d) per-block logging of data used, parameter delta norm, and fingerprint. Executing without a frozen Card is a protocol violation. +* **[SMOKE-VALIDATED]** The adaptation algorithm, buffer construction, native PPO/SAC update details, inherited hyperparameters, and failure behavior are frozen in `docs/research/TREATMENT_CARD.md`. Artifacts record the Card SHA-256. Smoke tests are not evidence of outcome-based tuning or scientific validity. ## 6. Operational Definition of Recovery @@ -205,7 +206,7 @@ Pre-registered training budget and config pinned to §15; final-checkpoint rule ## 9. Baseline -**Fixed arm**: trained on the nominal distribution; policy frozen and fingerprinted immediately after training; evaluated under shift with weights locked at all times (`shift_runner.py:180-205` already enforces fingerprint invariance for the existing benchmark `[VERIFIED-CODE]`). +**Fixed arm**: trained on the nominal distribution; policy frozen and fingerprinted immediately after training; evaluated under shift with weights locked at all times (`src/adaptive_rl/benchmarking/adaptation_runner.py` verifies the Fixed fingerprint against the frozen fingerprint `[SMOKE-VALIDATED]`). ## 10. Evaluation Environments @@ -220,9 +221,9 @@ At the pinned commit `[VERIFIED-CODE src/adaptive_rl/environments/__init__.py:60 ## 11. Training Distribution -All training uses only the nominal configuration and only the TRAIN seed pool `[1000..1014]` `[VERIFIED-CONFIG]`. Test seeds and shift parameters must not leak into training; post-hoc scenario selection is prohibited. The existing runner enforces this via `TrainingDistributionWrapper` and the seed-containment audit (`shift_runner.py:143-177`) `[VERIFIED-CODE]`. +Issue #265 training uses only `environment.parameters` from `configs/drone_distribution_shift.yaml`; the distinct TEST-B `shift_parameters` are merged by the runner only after training and the shared pre-shift evaluation. Derived update/evaluation seeds are validated against the frozen configuration seed pools. The runner's separate configuration fields make the TEST-B values unavailable to the training environment constructor `[SMOKE-VALIDATED]`. -**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Each replicate trains with `config.seed := training_seed(i)` (§14.1), replacing the config default (`42`/`123`). The containment audit must still pass afterwards; this has not been exercised. +**[SMOKE-VALIDATED]** Each replicate trains with the selected preregistered `training_seed`; the trainer receives a per-replicate config copy and the source config is unchanged. ## 12. Distribution Shifts @@ -323,7 +324,7 @@ Step 8 — CAUSAL RECOVERY MEASUREMENT adaptive_rl.protocol.recovery, not ad-hoc scripts. ``` -**Common-random-number property** `[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]`: episodes are reseeded at every reset, and environment exogenous noise is drawn from the seeded episode RNG in a way that does not depend on the action taken (e.g. drone gusts: `disturbed_drone.py:637,649` pass `rng=self.np_random`; `wind.py:85` draws `rng.normal(size=3)` per step) `[VERIFIED-CODE]`. Both arms therefore see the same exogenous noise stream for episode $j$ *up to the point where episode length diverges* (the number of RNG draws per episode is action-dependent). CRN alignment is exact for the shared segment and partial-by-construction for the arm segment; it is a variance-reduction property, not a claim of identical trajectories. +**Common-random-number property** `[SMOKE-VALIDATED]`: episodes are reseeded at every reset, and drone gust noise is drawn from the seeded environment RNG once per step. Both arms therefore receive the same exogenous noise stream for episode $j$ up to the point where episode length diverges (the number of RNG draws per episode is action-dependent). CRN alignment is exact for the shared segment and partial-by-construction for the arm segment; it is a variance-reduction property, not a claim of identical trajectories. ## 14. Seed Protocol @@ -339,7 +340,7 @@ TRAINING_SEEDS = [31001, 31002, 31003, 31004, 31005, 31006, 31007, 31008, 31009, ### 14.2 Derived episode/block seeds — exact specification -**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** There are 400 derived values per schedule (10 seeds × (15 pre + 15 post + 10 update)). The derivation is SHA-256, **not** Python's `hash()` (which is salted per process by `PYTHONHASHSEED`): +**[VERIFIED-CODE]** There are 400 derived values per schedule (10 seeds × (15 pre + 15 post + 10 update)). The derivation is SHA-256, **not** Python's `hash()` (which is salted per process by `PYTHONHASHSEED`): ``` payload = f"{training_seed}|{phase}|{index}".encode("utf-8") @@ -369,11 +370,11 @@ value = int.from_bytes(digest[:4], byteorder="big", signed=False) & 0x7FFFFFF schedule_fingerprint = 65939167572731c99599c382ac50cf3fddbba3cf758305764392f13b2e4efa67 ``` -Both arms record this fingerprint in their artifacts; a mismatch between arms (or against this document) invalidates the replicate. Any change to seeds, indices, or phases changes the fingerprint and therefore fails `tests/test_protocol_doc_sync.py`. +The research artifact and each replicate record this fingerprint, shared by both arms; a mismatch against this document invalidates the replicate. Any change to seeds, indices, or phases changes the fingerprint and therefore fails `tests/test_protocol_doc_sync.py`. ### 14.4 RNG initialization and train/test disjointness -Every episode reset uses `reset(seed = derived_value)` so run order cannot leak state across episodes `[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]`. Training-side seeding remains `random.seed` / `np.random.seed` / `torch.manual_seed` / CUDA seeds (`trainer.py:120-126`) `[VERIFIED-CODE]`. Train/test seed-set disjointness for the **config pools** is audited by `shift_runner.py:162-177` `[VERIFIED-CODE]`; disjointness of the **derived schedule** from those pools is enforced by `validate_schedule()` (§14.2). Determinism is **not** claimed across hardware/library versions (§25). +Every pre/post episode reset uses `reset(seed = derived_value)` and every update block uses its corresponding derived update seed `[SMOKE-VALIDATED]`. Training-side seeding remains `random.seed` / `np.random.seed` / `torch.manual_seed` / CUDA seeds in `trainer.py`. Disjointness of the derived schedule from training seeds and config pools is enforced by `validate_schedule()` (§14.2). Determinism is **not** claimed across hardware/library versions (§25). ## 15. Training Budgets (Pinned) @@ -405,8 +406,8 @@ The deterministic pre-declared rule is the **final training checkpoint** at the |---|---|---| | 1 | `gridworld/ppo` | No — no shift config; moderate is design-reference only | | 2 | `traffic_signal/ppo` | No — moderate `[FUTURE PROTOCOL VALUE]` | -| 3 | `drone_disturbed/ppo` | No — Adaptive harness absent (Fixed benchmark runnable) | -| 4 | `drone_disturbed/sac` | No — Adaptive harness absent (budget/config exist) | +| 3 | `drone_disturbed/ppo` | Yes — runner and smoke path implemented; full ten-replicate data not collected | +| 4 | `drone_disturbed/sac` | Yes — runner and smoke path implemented; full ten-replicate data not collected | | 5 | `navigation_2d/ppo` | No — moderate `[FUTURE PROTOCOL VALUE]`; harness absent | | 6 | `navigation_2d/sac` | No — config absent; harness absent | @@ -491,11 +492,11 @@ Pairwise-complete exclusion in the primary analysis can create attrition bias if ## 20. Reporting Requirements -### 20.1 Existing artifact (Fixed benchmark) `[VERIFIED-CODE src/adaptive_rl/evaluation/shift_benchmark.py:523-557]` +### 20.1 Historical Fixed-benchmark artifact (pinned protocol reference) `ShiftBenchmarkReport` stores (actual field names): `schema_version`, `experiment_name`, `environment_name`, `algorithm_name`, `deterministic`, `total_training_timesteps`, `train_seeds`, `test_seeds`, **`scenarios`** (list of `ScenarioResult`), `recovery_definition`, `training_provenance`, `environment_provenance`, `config_sha256`, `metadata`. -Per scenario: `scenario_name`, `role`, `seeds`, `environment_overrides`, `effective_environment_parameters`, `metrics`, `recovery`, **`episodes`** (list of `EpisodeBenchmarkRecord`), `gaps`, `policy_fingerprint`. +The legacy Fixed-benchmark implementation at the pinned protocol revision stored per scenario: `scenario_name`, `role`, `seeds`, `environment_overrides`, `effective_environment_parameters`, `metrics`, `recovery`, **`episodes`** (list of `EpisodeBenchmarkRecord`), `gaps`, `policy_fingerprint`. That legacy module is not present in the current checkout; Issue #265 writes its own explicit adaptation schema. Per episode record (`shift_benchmark.py:466-490`): `seed`, `reward`, `length`, `success`, `collision`, `terminated`, `truncated`, `recovery_times`, `recovery_events`, `recovery_completed`, `recovery_censored`. @@ -503,9 +504,9 @@ Per episode record (`shift_benchmark.py:466-490`): `seed`, `reward`, `length`, ` ### 20.2 Derived research quantities -$P_{pre}$, $P_0$, $P(t)$, $R(t)$, $\tau$, $T_H$, status, δ, `delta_min`, and the update-block log are **not native fields** of the current report; they must be derived from episode records by `src/adaptive_rl/protocol/recovery.py` (never ad hoc) and stored in a derived artifact. +$P_{pre}$, $P_0$, $P(t)$, $R(t)$, $\tau$, $T_H$, status, δ, `delta_min`, and the update-block log are derived from episode records; the Issue #265 artifact stores them using `src/adaptive_rl/protocol/recovery.py` (never ad hoc equations). -### 20.3 Future Adaptive artifact (required contents) +### 20.3 Issue #265 artifact contents Per replicate × arm: training seed, all three phase-seed lists used, schedule fingerprint (§14.3), `PROTOCOL_VERSION`, pinned commit SHA, config SHA-256, pre-shift/shock/post-shift return vectors, per-episode success flags, $P_{pre}$, $P_0$, δ, `delta_min`, $P(t)$ and $R(t)$ trajectories, predicate vector, $\tau$, $T_H$, status; per block: block id, episode range, derived update seed, data-episode indices, parameter-delta norm, fingerprint before/after; plus environment/library provenance (`environment_provenance` already exists) including numpy version (bootstrap dependency, §18.3). @@ -520,13 +521,13 @@ Per replicate × arm: training seed, all three phase-seed lists used, schedule f | Threat | Why it matters | Mitigation | Remaining limitation | |---|---|---|---| -| Treatment-contaminated $P_0$ | Adaptive updating before $P_0$ is measured breaks normalization | Shared single execution of episodes 1–5 (§4.2); $B_5$ strictly after episode 5 | Depends on future harness honoring the ordering contract (§5.1–5.2) | +| Treatment-contaminated $P_0$ | Adaptive updating before $P_0$ is measured breaks normalization | Shared single execution of episodes 1–5 (§4.2); $B_5$ strictly after episode 5 | Full ten-replicate protocol execution remains unverified | | `no_degradation` convention | $T_H = 0$ anchors the distribution without adaptation evidence | Convention retained, status always labeled; arm-invariant so $D_i = 0$ exactly | Can still shift the *level* of $T_H$ vs other studies; comparisons must match conventions | | Finite-horizon truncation | $T_H = 15$ for non-recovery makes the estimand a truncated mean | Estimand explicitly finite-horizon; right-censoring labeled, never extrapolated | Not a claim about true recovery-time distributions | | Normality of $D_i$ (t-test) | Primary test assumes it; $n = 8..10$ cannot verify it | Sensitivities (§18.3) target median/pseudomedian/resampling; disagreement must be reported | Type I/II error may deviate from nominal if strongly violated | | `MIN_VALID_N = 8` threshold | A cell at exactly 8 has less power than one at 10 | Rule frozen pre-data; per-cell $N_{valid}$ always reported | Power is unquantified; no formal power analysis (§25) | | Failed-run attrition | Excluding crashes can bias the contrast | Pairwise exclusion; per-arm failure counts; two-sided $[0,H]$ imputation bounds | Counterfactual $T_H$ of crashed runs unobservable | -| Adaptive harness absent | Nothing in the treatment sections is executable | Protocol preregistered before implementation; status tags | Actual adaptation behavior entirely untested | +| Partial cell coverage | Only the drone PPO/SAC cells have the Issue #265 runner in this checkout | Missing cells remain explicitly inconclusive; no six-cell family claim | GridWorld, Traffic Signal, and Navigation cells are unimplemented | | Unequal compute | Adaptive adds online compute | Declared part of the intervention; Fixed budget unchanged | Deployment latency/cost unmeasured | | Training non-determinism | No cudnn/deterministic-algorithm flags exist (`trainer.py:120-126`) | Train-once design: one run serves both arms (§4.2) | Cross-machine retraining may not reproduce weights; recorded fingerprint detects it | | CRN partial alignment | Episode lengths are action-dependent | Reseeding per episode; alignment stated as partial-by-construction (§13) | Variance reduction weaker than exact pairing of noise | @@ -541,7 +542,7 @@ Per replicate × arm: training seed, all three phase-seed lists used, schedule f * [ ] Config YAMLs archived at the pinned commit; `config_sha256` recorded. * [ ] All 400 derived seeds logged; schedule fingerprint equals `65939167572731c99599c382ac50cf3fddbba3cf758305764392f13b2e4efa67` in both arms. * [ ] `validate_schedule()` run on the artifact's schedule with zero violations (uniqueness, range, disjointness). -* [ ] Config-pool disjointness verified by the existing runner audit (`shift_runner.py:162-177`) **and** by `validate_schedule()`. +* [ ] Training receives nominal parameters only; the runner withholds TEST-B parameters until after shared pre-shift evaluation, and `validate_schedule()` passes. * [ ] Train-once/fork design confirmed: one training fingerprint per replicate, both arms fork from it; Fixed arm fingerprint unchanged at end. * [ ] $P_0$ measured in the shared segment, pre-$B_5$; block ordering log shows $B_5$ strictly between episodes 5 and 6. * [ ] Per-block log complete for $B_5..B_{14}$ (10 blocks, no block after episode 15). @@ -549,7 +550,7 @@ Per replicate × arm: training seed, all three phase-seed lists used, schedule f * [ ] Primary test, sensitivities, Holm, and `decide_family()` executed from `adaptive_rl.protocol.statistics`. * [ ] Per-cell $N_{valid}$, failure counts by arm, and both imputation directions reported. * [ ] Doc-sync tests pass: `python -m pytest tests/test_protocol_seed_schedule.py tests/test_protocol_recovery.py tests/test_protocol_statistics.py tests/test_protocol_doc_sync.py`. -* [ ] Treatment Card (§5.3) committed with SHA recorded before any data collection. +* [ ] Treatment Card (§5.3) frozen before a scientific run and its SHA recorded in the artifact. ## 24. Executable Protocol Mirror diff --git a/docs/research/issue-265.md b/docs/research/issue-265.md new file mode 100644 index 0000000..28a92c1 --- /dev/null +++ b/docs/research/issue-265.md @@ -0,0 +1,92 @@ +# Issue #265 implementation guide + +## What is implemented + +`adaptive-rl benchmark adaptation` runs the protocol's train-once, shared +pre-shift, shared shock, fork, and paired post-shift lifecycle for drone TEST-B +with PPO or SAC. The base environment receives only the nominal configuration +from `configs/drone_distribution_shift.yaml`; the TEST-B overrides are merged +after training and the pre-shift segment. The run checks the effective shift +values before proceeding. + +The online data boundary is represented by completed post-shift episode +records. Block Bk receives the ordered prefix 1..k only, and the builder rejects +missing, duplicated, out-of-order, wrong-seed, or future episode data. Blocks +B5 through B14 execute between episodes; B15 is rejected. PPO uses stored +behavior values/log-probabilities with its native clipped update. SAC uses a +new replay buffer containing only the allowed post-shift prefix. Both use the +block's derived update seed. Update failures roll model state back and fail the +replicate. + +The fork deep-copies both arms, verifies equal starting policy states, and gives +the Fixed arm a prediction-only interface. The runner verifies its final +fingerprint against the frozen fingerprint. Episode records include outcomes, +derived seeds, policy fingerprints, update association, and the complete +transition data used by adaptation. + +Recovery is computed only by +`adaptive_rl.protocol.recovery.compute_recovery()`. Paired analysis uses the +protocol's paired t-test, interval, dz, sign and Wilcoxon sensitivities, +bootstrap interval, failure-imputation bounds, and Holm correction. Since this +checkout implements the selected drone cell only, the other five primary cells are +reported as inconclusive in each single-cell artifact; the six-cell family +claim is therefore inconclusive until the remaining cells have data. + +## Running it + +Run a reduced, explicitly labeled machinery check: + +```bash +adaptive-rl benchmark adaptation \ + --config configs/drone_distribution_shift.yaml \ + --smoke \ + --output-dir artifacts/issue265_smoke_ppo +``` + +Run the smoke path for SAC: + +```bash +adaptive-rl benchmark adaptation \ + --config configs/drone_distribution_shift.yaml \ + --algorithm sac \ + --smoke \ + --output-dir artifacts/issue265_smoke_sac +``` + +Run the complete ten-seed PPO experiment (the same command accepts `--algorithm +sac` for SAC): + +```bash +adaptive-rl benchmark adaptation \ + --config configs/drone_distribution_shift.yaml \ + --output-dir artifacts/issue265_ppo +``` + +`--training-seeds` may select a comma-separated subset of the frozen training +seeds. Unselected replicates remain missing and count as failures for +pair-complete analysis. `--deterministic` and `--stochastic` override the +configured evaluation action selection. The command writes +`adaptation.json` and `adaptation.csv`; it refuses to overwrite either file or +an existing per-seed training directory. + +The JSON contains the full trajectories, replicate failures and reasons, +protocol and schedule fingerprints, Treatment Card SHA-256, canonical and raw +configuration hashes, repository commit/dirty state, runtime versions, +recovery outputs, and paired analysis. CSV has one row per recorded episode. +Smoke output is labeled `run_type: smoke`, uses a reduced training budget and +episode length, and is never treated as a scientific result. + +## Reproducibility and status + +Adaptation choices are frozen in +[`TREATMENT_CARD.md`](TREATMENT_CARD.md). The JSON records its content hash, +the full derived schedule fingerprint, each replicate's exact phase seeds, +each update seed, and model fingerprints before and after every block. Failed +training, evaluation, or update work is represented in `failure_summary` and +the replicate record; a failed update stops that replicate before the next +episode. + +Both PPO and SAC smoke paths have been executed and checked for block order, +episode counts, seed alignment, Fixed immutability, and recovery output. Those +checks validate the harness only. No full ten-replicate benchmark has been run +and no empirical superiority or statistical significance is claimed. diff --git a/src/adaptive_rl/algorithms/__init__.py b/src/adaptive_rl/algorithms/__init__.py index b727017..15c2919 100644 --- a/src/adaptive_rl/algorithms/__init__.py +++ b/src/adaptive_rl/algorithms/__init__.py @@ -1,5 +1,11 @@ """Algorithm abstraction layer for AdaptiveRL.""" +from adaptive_rl.algorithms.adaptation import ( + AdaptationUpdateLog, + PPOAdaptationAdapter, + SACAdaptationAdapter, + run_adaptation_update, +) from adaptive_rl.algorithms.base import BaseAlgorithm from adaptive_rl.algorithms.ppo import PPOAlgorithm from adaptive_rl.algorithms.random_policy import RandomPolicy @@ -19,13 +25,17 @@ "AlgorithmMetadata", "AlgorithmRegistry", "AlgorithmRegistryError", + "AdaptationUpdateLog", "BaseAlgorithm", "PPOAlgorithm", + "PPOAdaptationAdapter", "RandomPolicy", + "SACAdaptationAdapter", "algorithm_registry", "get_algorithm_factory", "get_algorithm_metadata", "list_algorithms", "list_all_algorithm_metadata", "register_algorithm", + "run_adaptation_update", ] diff --git a/src/adaptive_rl/algorithms/adaptation.py b/src/adaptive_rl/algorithms/adaptation.py new file mode 100644 index 0000000..d32aeba --- /dev/null +++ b/src/adaptive_rl/algorithms/adaptation.py @@ -0,0 +1,319 @@ +"""Recorded-data PPO and SAC update adapters for Issue #265.""" + +from __future__ import annotations + +import random +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from typing import Any, Iterator + +import numpy as np +import torch +from stable_baselines3.common.buffers import ReplayBuffer, RolloutBuffer +from stable_baselines3.common.logger import Logger + +from adaptive_rl.protocol.adaptation import AdaptationAdapter, UpdateBatch, call_update_atomically +from adaptive_rl.protocol.fork import model_fingerprint, policy_state_tensors + + +@contextmanager +def _seeded_update(seed: int) -> Iterator[None]: + """Seed update-side RNGs and restore caller RNG state on exit.""" + py_state = random.getstate() + np_state = np.random.get_state() + torch_state = torch.random.get_rng_state() + cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None + try: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + yield + finally: + random.setstate(py_state) + np.random.set_state(np_state) + torch.random.set_rng_state(torch_state) + if cuda_states is not None: + torch.cuda.set_rng_state_all(cuda_states) + + +def _model(algorithm: Any) -> Any: + model = getattr(algorithm, "model", None) + if model is None: + raise RuntimeError("Cannot adapt an uninitialized algorithm") + return model + + +@contextmanager +def _logger_ready(model: Any) -> Iterator[None]: + """Provide a silent SB3 logger when a model is adapted before its first fit.""" + had_logger = hasattr(model, "_logger") + old_logger = getattr(model, "_logger", None) + if not had_logger: + model.set_logger(Logger(folder=None, output_formats=[])) + try: + yield + finally: + if had_logger: + model._logger = old_logger + else: + delattr(model, "_logger") + + +class PPOAdaptationAdapter: + """Use Stable-Baselines3 PPO's native clipped objective on stored rollouts.""" + + def update(self, algorithm: Any, batch: UpdateBatch) -> None: + model = _model(algorithm) + if not hasattr(model, "rollout_buffer") or not hasattr(model, "n_epochs"): + raise TypeError("PPOAdaptationAdapter requires a Stable-Baselines3 PPO model") + if int(model.n_epochs) < 1 or int(model.batch_size) < 1: + raise ValueError("PPO adaptation requires positive configured epochs and batch size") + if any( + transition.behavior_log_prob is None or transition.behavior_value is None + for transition in batch.transitions + ): + raise ValueError( + "PPO adaptation requires behavior log-probability and value per transition" + ) + if any( + transition.truncated + and not transition.terminated + and transition.behavior_next_value is None + for transition in batch.transitions + ): + raise ValueError("PPO truncated transitions require their recorded behavior next-value") + + count = len(batch.transitions) + model.rollout_buffer = RolloutBuffer( + buffer_size=count, + observation_space=model.observation_space, + action_space=model.action_space, + device=model.device, + gae_lambda=model.gae_lambda, + gamma=model.gamma, + n_envs=1, + ) + model.rollout_buffer.reset() + observations: list[np.ndarray] = [] + + for transition in batch.transitions: + observation = np.asarray(transition.observation) + action = np.asarray(transition.action) + reward = float(transition.reward) + if transition.truncated and not transition.terminated: + reward += model.gamma * float(transition.behavior_next_value) + observations.append(observation) + model.rollout_buffer.add( + obs=observation.reshape((1, *observation.shape)), + action=action.reshape((1, -1)), + reward=np.asarray([reward], dtype=np.float32), + episode_start=np.asarray( + [ + len(observations) == 1 + or _previous_done(batch.transitions, len(observations) - 1) + ], + dtype=np.float32, + ), + value=torch.as_tensor( + [transition.behavior_value], dtype=torch.float32, device=model.device + ), + log_prob=torch.as_tensor( + [transition.behavior_log_prob], dtype=torch.float32, device=model.device + ), + ) + model.rollout_buffer.compute_returns_and_advantage( + last_values=torch.zeros(1, dtype=torch.float32, device=model.device), + dones=np.ones(1, dtype=np.float32), + ) + with _logger_ready(model), _seeded_update(batch.seed): + model.train() + model.policy.set_training_mode(False) + + +def _previous_done(transitions: tuple[Any, ...], previous_index: int) -> bool: + previous = transitions[previous_index - 1] + return bool(previous.terminated or previous.truncated) + + +class SACAdaptationAdapter: + """Train SAC from a fresh replay buffer containing only the visible batch.""" + + def update(self, algorithm: Any, batch: UpdateBatch) -> None: + model = _model(algorithm) + if not hasattr(model, "critic_target") or not hasattr(model, "gradient_steps"): + raise TypeError("SACAdaptationAdapter requires a Stable-Baselines3 SAC model") + if int(model.gradient_steps) < 1 or int(model.batch_size) < 1: + raise ValueError( + "SAC adaptation requires positive configured gradient steps and batch size" + ) + count = len(batch.transitions) + # Never reuse the replay buffer containing nominal training experience. + replay = ReplayBuffer( + buffer_size=max(count, int(model.batch_size)), + observation_space=model.observation_space, + action_space=model.action_space, + device=model.device, + n_envs=1, + optimize_memory_usage=False, + handle_timeout_termination=True, + ) + for transition in batch.transitions: + observation = np.asarray(transition.observation).reshape( + (1, *np.asarray(transition.observation).shape) + ) + next_observation = np.asarray(transition.next_observation).reshape( + (1, *np.asarray(transition.next_observation).shape) + ) + action = np.asarray(transition.action).reshape((1, -1)) + done = bool(transition.terminated or transition.truncated) + replay.add( + observation, + next_observation, + action, + np.asarray([transition.reward], dtype=np.float32), + np.asarray([done], dtype=np.float32), + infos=[ + { + "TimeLimit.truncated": bool( + transition.truncated and not transition.terminated + ) + } + ], + ) + + prior_buffer = model.replay_buffer + model.replay_buffer = replay + try: + with _logger_ready(model), _seeded_update(batch.seed): + model.train( + gradient_steps=int(model.gradient_steps), batch_size=int(model.batch_size) + ) + finally: + model.replay_buffer = prior_buffer + model.policy.set_training_mode(False) + + +@dataclass(frozen=True) +class AdaptationUpdateLog: + block_episode: int + update_seed: int + visible_episode_indices: tuple[int, ...] + transition_count: int + fingerprint_before: str + fingerprint_after: str + parameter_delta_l2: float + status: str + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _parameters(model_or_wrapper: Any) -> dict[str, torch.Tensor]: + return { + name: value.detach().cpu().clone() + for name, value in policy_state_tensors(model_or_wrapper).items() + } + + +def _validate_update(model_or_wrapper: Any, before: dict[str, torch.Tensor]) -> float: + model = getattr(model_or_wrapper, "model", model_or_wrapper) + after = policy_state_tensors(model_or_wrapper) + policy = getattr(model, "policy", model) + parameter_names = {f"policy.{name}" for name, _ in policy.named_parameters()} + if isinstance(getattr(model, "log_ent_coef", None), torch.Tensor): + parameter_names.add("algorithm.log_ent_coef") + if after.keys() != before.keys(): + raise RuntimeError("Update changed the model state structure") + squared_delta = 0.0 + for name, tensor in after.items(): + if tensor.shape != before[name].shape or tensor.dtype != before[name].dtype: + raise RuntimeError(f"Update changed model tensor shape or dtype: {name}") + if not torch.isfinite(tensor).all(): + raise FloatingPointError(f"Update produced non-finite model state: {name}") + if name in parameter_names: + difference = tensor.detach().cpu().to(torch.float64) - before[name].to(torch.float64) + squared_delta += float(torch.sum(difference * difference)) + for name, optimizer in _optimizers(model): + _validate_finite_tree(optimizer.state_dict(), f"optimizer {name}") + for parameter, state in optimizer.state.items(): + for state_name, state_value in state.items(): + if isinstance(state_value, torch.Tensor): + if state_value.numel() > 1 and state_value.shape != parameter.shape: + raise RuntimeError( + f"Update corrupted optimizer state shape: {name}.{state_name}" + ) + return float(np.sqrt(squared_delta)) + + +def _optimizers(model: Any) -> list[tuple[str, Any]]: + result = [] + for name, value in vars(model).items(): + if isinstance(value, torch.optim.Optimizer): + result.append((name, value)) + policy_optimizer = getattr(getattr(model, "policy", None), "optimizer", None) + if isinstance(policy_optimizer, torch.optim.Optimizer): + result.append(("policy.optimizer", policy_optimizer)) + for owner_name in ("actor", "critic"): + owner = getattr(model, owner_name, None) + optimizer = getattr(owner, "optimizer", None) + if isinstance(optimizer, torch.optim.Optimizer): + result.append((f"{owner_name}.optimizer", optimizer)) + ent_optimizer = getattr(model, "ent_coef_optimizer", None) + if isinstance(ent_optimizer, torch.optim.Optimizer): + result.append(("ent_coef_optimizer", ent_optimizer)) + return result + + +def _validate_finite_tree(value: Any, path: str) -> None: + if isinstance(value, torch.Tensor): + if not torch.isfinite(value).all(): + raise FloatingPointError(f"Update produced non-finite {path}") + elif isinstance(value, dict): + for key, nested in value.items(): + _validate_finite_tree(nested, f"{path}.{key}") + elif isinstance(value, (list, tuple)): + for index, nested in enumerate(value): + _validate_finite_tree(nested, f"{path}[{index}]") + elif isinstance(value, (float, int)) and not np.isfinite(value): + raise FloatingPointError(f"Update produced non-finite {path}") + + +class _ValidatedAdapter: + def __init__(self, delegate: AdaptationAdapter, before: dict[str, torch.Tensor]) -> None: + self.delegate = delegate + self.before = before + + def update(self, algorithm: Any, batch: UpdateBatch) -> None: + self.delegate.update(algorithm, batch) + _validate_update(algorithm, self.before) + + +def run_adaptation_update( + algorithm: Any, adapter: AdaptationAdapter, batch: UpdateBatch +) -> AdaptationUpdateLog: + """Execute one seeded block atomically and return its auditable diagnostics.""" + before_state = _parameters(algorithm) + before_fingerprint = model_fingerprint(algorithm) + call_update_atomically(algorithm, _ValidatedAdapter(adapter, before_state), batch) + delta = _validate_update(algorithm, before_state) + after_fingerprint = model_fingerprint(algorithm) + return AdaptationUpdateLog( + block_episode=batch.block_episode, + update_seed=batch.seed, + visible_episode_indices=batch.visible_episode_indices, + transition_count=len(batch.transitions), + fingerprint_before=before_fingerprint, + fingerprint_after=after_fingerprint, + parameter_delta_l2=delta, + status="updated" if delta > 0.0 else "no_parameter_change", + ) + + +__all__ = [ + "AdaptationUpdateLog", + "PPOAdaptationAdapter", + "SACAdaptationAdapter", + "run_adaptation_update", +] diff --git a/src/adaptive_rl/benchmarking/__init__.py b/src/adaptive_rl/benchmarking/__init__.py index 2029d7c..e7aa4ac 100644 --- a/src/adaptive_rl/benchmarking/__init__.py +++ b/src/adaptive_rl/benchmarking/__init__.py @@ -9,6 +9,12 @@ from importlib import import_module from typing import Any +from adaptive_rl.benchmarking.adaptation_artifacts import write_adaptation_artifacts +from adaptive_rl.benchmarking.adaptation_statistics import ( + PairedRecoveryAnalysis, + analyze_paired_recovery, + analyze_primary_cells, +) from adaptive_rl.benchmarking.learning_curve import ( BenchmarkRunError, LearningCurveBenchmarkResult, @@ -46,7 +52,11 @@ def __getattr__(name: str) -> Any: "BenchmarkRunError", "LearningCurveBenchmarkResult", "LearningCurvePoint", + "PairedRecoveryAnalysis", + "analyze_paired_recovery", + "analyze_primary_cells", "plot_learning_curve", "run_learning_curve_benchmark", "validate_budgets", + "write_adaptation_artifacts", ] diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py new file mode 100644 index 0000000..3fa3d03 --- /dev/null +++ b/src/adaptive_rl/benchmarking/adaptation_artifacts.py @@ -0,0 +1,153 @@ +"""JSON and analysis-friendly CSV serialization for Issue #265 runs.""" + +from __future__ import annotations + +import csv +import dataclasses +import json +import os +import tempfile +from pathlib import Path +from typing import Any, Iterable, Mapping + +import numpy as np + +CSV_FIELDS = ( + "replicate_index", + "training_seed", + "arm", + "phase", + "episode_index", + "episode_seed", + "algorithm", + "environment", + "reward", + "length", + "success", + "collision", + "terminated", + "truncated", + "policy_fingerprint_start", + "policy_fingerprint_end", + "update_block", + "update_seed", + "update_status", + "parameter_delta_l2", +) + + +def _plain(value: Any) -> Any: + """Convert supported scientific data values into strict JSON primitives.""" + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return _plain(dataclasses.asdict(value)) + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, np.generic): + return value.item() + if isinstance(value, Path): + return str(value) + if isinstance(value, Mapping): + if not all(isinstance(key, (str, int, float, bool)) for key in value): + raise TypeError("artifact mappings must have primitive keys") + return {str(key): _plain(nested) for key, nested in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(nested) for nested in value] + if value is None or isinstance(value, (str, int, float, bool)): + return value + raise TypeError(f"unsupported artifact value type: {type(value).__name__}") + + +def _episode_rows(artifact: Mapping[str, Any]) -> Iterable[dict[str, Any]]: + for replicate_index, replicate in enumerate(artifact.get("replicates", []), start=1): + common = { + "replicate_index": replicate_index, + "training_seed": replicate.get("training_seed"), + "algorithm": artifact.get("experiment", {}).get("algorithm"), + "environment": artifact.get("experiment", {}).get("environment"), + } + segments = ( + ("shared_pre_shift_episodes", "pre", "shared"), + ("shared_shock_episodes", "post", "shared"), + ("adaptive_episodes", "post", "adaptive"), + ("fixed_episodes", "post", "fixed"), + ) + for key, phase, arm in segments: + for episode in replicate.get(key, []): + row = {field: None for field in CSV_FIELDS} + row.update(common) + row.update( + arm=arm, + phase=phase, + episode_index=episode.get("episode_index"), + episode_seed=episode.get("episode_seed"), + reward=episode.get("reward"), + length=episode.get("length"), + success=episode.get("success"), + collision=episode.get("collision"), + terminated=episode.get("terminated"), + truncated=episode.get("truncated"), + policy_fingerprint_start=episode.get("policy_fingerprint_start"), + policy_fingerprint_end=episode.get("policy_fingerprint_end"), + update_block=episode.get("update_block"), + update_seed=episode.get("update_seed"), + update_status=episode.get("update_status"), + parameter_delta_l2=episode.get("parameter_delta_l2"), + ) + yield row + + +def write_adaptation_artifacts( + artifact: Mapping[str, Any], + output_dir: str | Path, + *, + stem: str = "adaptation", +) -> tuple[Path, Path]: + """Write strict JSON and flattened CSV without overwriting prior results. + + Both payloads are first written to temporary files, then installed with + exclusive hard links. If either target already exists, no target is + overwritten and the call raises ``FileExistsError``. + """ + if not stem or Path(stem).name != stem: + raise ValueError("stem must be a non-empty filename component") + target_dir = Path(output_dir) + target_dir.mkdir(parents=True, exist_ok=True) + json_path = target_dir / f"{stem}.json" + csv_path = target_dir / f"{stem}.csv" + plain = _plain(artifact) + if not isinstance(plain, dict): + raise TypeError("artifact root must be a mapping") + + temp_paths: list[Path] = [] + try: + for suffix, writer in ( + (".json", lambda handle: json.dump(plain, handle, indent=2, allow_nan=False)), + (".csv", lambda handle: _write_csv(handle, plain)), + ): + fd, temp_name = tempfile.mkstemp(prefix=f".{stem}-", suffix=suffix, dir=target_dir) + temp_path = Path(temp_name) + temp_paths.append(temp_path) + with os.fdopen(fd, "w", newline="", encoding="utf-8") as handle: + writer(handle) + handle.flush() + os.fsync(handle.fileno()) + # link() is atomic and fails if the destination exists. + os.link(temp_paths[0], json_path) + try: + os.link(temp_paths[1], csv_path) + except BaseException: + json_path.unlink() + raise + finally: + for temp_path in temp_paths: + temp_path.unlink(missing_ok=True) + return json_path, csv_path + + +def _write_csv(handle: Any, artifact: Mapping[str, Any]) -> None: + writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS, extrasaction="raise") + writer.writeheader() + writer.writerows(_episode_rows(artifact)) + + +__all__ = ["CSV_FIELDS", "write_adaptation_artifacts"] diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py new file mode 100644 index 0000000..621db53 --- /dev/null +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -0,0 +1,542 @@ +"""Protocol-ordered Issue #265 train/freeze/share/fork experiment runner.""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import logging +import platform +import subprocess +import sys +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional, Sequence + +import gymnasium as gym +import numpy as np + +from adaptive_rl.algorithms.adaptation import ( + PPOAdaptationAdapter, + SACAdaptationAdapter, + run_adaptation_update, +) +from adaptive_rl.benchmarking.adaptation_artifacts import write_adaptation_artifacts +from adaptive_rl.benchmarking.adaptation_runtime import EpisodeRecord, evaluate_episode +from adaptive_rl.benchmarking.adaptation_statistics import analyze_primary_cells +from adaptive_rl.config import ExperimentConfig, compute_config_sha256 +from adaptive_rl.environments.registry import make_env +from adaptive_rl.protocol.adaptation import build_update_batch, validate_block_sequence +from adaptive_rl.protocol.constants import K_PRE, N_POST, PRIMARY_CELLS, TRAINING_SEEDS +from adaptive_rl.protocol.fork import fork_adaptive_and_fixed, model_fingerprint +from adaptive_rl.protocol.recovery import compute_recovery +from adaptive_rl.protocol.seeds import frozen_schedule, schedule_fingerprint +from adaptive_rl.protocol.statistics import decide_family +from adaptive_rl.training.trainer import get_trainer + +logger = logging.getLogger(__name__) + +EnvironmentFactory = Callable[..., gym.Env] +TrainerFactory = Callable[..., Any] + + +def _repository_metadata() -> dict[str, Any]: + try: + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() + dirty = bool( + subprocess.check_output( + ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL + ).strip() + ) + except (OSError, subprocess.CalledProcessError): + commit, dirty = None, None + versions = {"python": sys.version.split()[0], "platform": platform.platform()} + for distribution in ("adaptive-rl", "gymnasium", "stable-baselines3", "torch", "numpy"): + try: + versions[distribution] = importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + versions[distribution] = None + return {"repository_commit": commit, "working_tree_dirty": dirty, "runtime_versions": versions} + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _card_path() -> Path: + return Path(__file__).resolve().parents[3] / "docs" / "research" / "TREATMENT_CARD.md" + + +def _new_env( + config: ExperimentConfig, + *, + shifted: bool, + environment_factory: EnvironmentFactory, + max_steps: Optional[int] = None, +) -> gym.Env: + parameters = dict(config.environment.parameters) + parameters["max_steps"] = int(max_steps or config.environment.max_steps) + if shifted: + if config.adaptation_benchmark is None: + raise ValueError("configuration does not declare the Issue #265 adaptation cell") + parameters.update(config.adaptation_benchmark.shift_parameters) + return environment_factory(config.environment.name, **parameters) + + +def _adapter_for(algorithm_name: str): + if algorithm_name == "ppo": + return PPOAdaptationAdapter() + if algorithm_name == "sac": + return SACAdaptationAdapter() + raise ValueError(f"Issue #265 supports PPO and SAC, got {algorithm_name!r}") + + +def _train_once( + config: ExperimentConfig, + training_seed: int, + training_dir: Path, + *, + trainer_factory: TrainerFactory, + environment_factory: EnvironmentFactory, + smoke: bool, +) -> tuple[Any, dict[str, Any]]: + if config.training is None: + raise ValueError("Issue #265 requires a training configuration") + if training_dir.exists(): + raise FileExistsError(f"training output already exists: {training_dir}") + training_dir.mkdir(parents=True) + + effective = config.model_copy(deep=True) + effective.seed = int(training_seed) + effective.name = f"{config.name}_seed_{training_seed}" + effective.output_dir = training_dir + effective.log_dir = training_dir / "logs" + # The training component does not receive the TEST-B scenario payload. + effective.adaptation_benchmark = None + algorithm_config = effective.algorithm.model_copy(deep=True) + algorithm_parameters = dict(algorithm_config.parameters) + algorithm_parameters["seed"] = int(training_seed) + if smoke: + effective.training.total_timesteps = 32 + if algorithm_config.name.lower() == "ppo": + algorithm_parameters.update({"n_steps": 16, "n_epochs": 1}) + algorithm_config.batch_size = min(8, algorithm_config.batch_size) + else: + algorithm_parameters.update( + {"learning_starts": 1, "gradient_steps": 1, "buffer_size": 64} + ) + algorithm_config.batch_size = min(8, algorithm_config.batch_size) + algorithm_config.parameters = algorithm_parameters + effective.algorithm = algorithm_config + training_env = _new_env( + effective, + shifted=False, + environment_factory=environment_factory, + max_steps=8 if smoke else None, + ) + trainer = None + started = time.perf_counter() + try: + trainer = trainer_factory(config=effective, env=training_env) + result = trainer.fit() + algorithm = trainer.algorithm + except BaseException: + if trainer is not None: + trainer.close() + else: + training_env.close() + raise + else: + training_seconds = float(time.perf_counter() - started) + trainer.close() + return algorithm, { + "training_seed": training_seed, + "total_timesteps_requested": effective.training.total_timesteps, + "total_timesteps_completed": int(getattr(algorithm, "num_timesteps", 0)), + "training_time_seconds": training_seconds, + "model_path": str(result.final_model_path), + "training_metadata_path": str(result.metadata_path) if result.metadata_path else None, + "episodes_completed": int(result.episodes_completed), + "mean_reward": float(result.mean_reward), + "smoke_override": bool(smoke), + "effective_config": effective.model_dump(mode="json"), + } + + +def _run_evaluation_segment( + *, + algorithm: Any, + env: gym.Env, + training_seed: int, + algorithm_name: str, + environment_name: str, + phase: str, + indices: Sequence[int], + seeds: Sequence[int], + deterministic: bool, + arm: str, + initial_update_log: Any = None, +) -> list[EpisodeRecord]: + if len(indices) != len(seeds): + raise ValueError("episode indices and seeds must have matching lengths") + records: list[EpisodeRecord] = [] + update_log = initial_update_log + for episode_index, episode_seed in zip(indices, seeds): + records.append( + evaluate_episode( + algorithm=algorithm, + env=env, + training_seed=training_seed, + phase=phase, + episode_index=episode_index, + episode_seed=episode_seed, + algorithm_name=algorithm_name, + environment_name=environment_name, + deterministic=deterministic, + arm=arm, + update_log=update_log, + ) + ) + update_log = None + return records + + +def _recover(pre: Sequence[EpisodeRecord], post: Sequence[EpisodeRecord]) -> dict[str, Any]: + result = compute_recovery( + [episode.reward for episode in pre], [episode.reward for episode in post] + ) + output = asdict(result) + output["T_H"] = result.truncated_recovery_time + return output + + +@dataclass +class ReplicateResult: + training_seed: int + status: str + schedule_fingerprint: Optional[str] = None + training_provenance: dict[str, Any] = field(default_factory=dict) + frozen_fingerprint: Optional[str] = None + fixed_final_fingerprint: Optional[str] = None + pre_shift_performance: Optional[float] = None + shock_performance: Optional[float] = None + shared_pre_shift_episodes: list[EpisodeRecord] = field(default_factory=list) + shared_shock_episodes: list[EpisodeRecord] = field(default_factory=list) + adaptive_episodes: list[EpisodeRecord] = field(default_factory=list) + fixed_episodes: list[EpisodeRecord] = field(default_factory=list) + update_blocks: list[dict[str, Any]] = field(default_factory=list) + adaptive_recovery: Optional[dict[str, Any]] = None + fixed_recovery: Optional[dict[str, Any]] = None + seeds: dict[str, list[int]] = field(default_factory=dict) + effective_nominal_parameters: dict[str, Any] = field(default_factory=dict) + effective_shift_parameters: dict[str, Any] = field(default_factory=dict) + failure_reason: Optional[str] = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _run_replicate( + config: ExperimentConfig, + training_seed: int, + training_dir: Path, + *, + schedule: dict[int, dict[str, list[int]]], + smoke: bool, + trainer_factory: TrainerFactory, + environment_factory: EnvironmentFactory, +) -> ReplicateResult: + result = ReplicateResult(training_seed=training_seed, status="failed") + try: + result.schedule_fingerprint = schedule_fingerprint(schedule) + algorithm, training_provenance = _train_once( + config, + training_seed, + training_dir, + trainer_factory=trainer_factory, + environment_factory=environment_factory, + smoke=smoke, + ) + result.training_provenance = training_provenance + frozen_fingerprint = model_fingerprint(algorithm) + result.frozen_fingerprint = frozen_fingerprint + algorithm.model.policy.set_training_mode(False) + + nominal_params = dict(config.environment.parameters) + nominal_params["max_steps"] = 8 if smoke else config.environment.max_steps + shifted_params = dict(nominal_params) + shifted_params.update(config.adaptation_benchmark.shift_parameters) + result.effective_nominal_parameters = dict(nominal_params) + result.effective_shift_parameters = dict(shifted_params) + result.seeds = {phase: list(values) for phase, values in schedule[training_seed].items()} + + nominal_env = _new_env( + config, + shifted=False, + environment_factory=environment_factory, + max_steps=8 if smoke else None, + ) + try: + result.shared_pre_shift_episodes = _run_evaluation_segment( + algorithm=algorithm, + env=nominal_env, + training_seed=training_seed, + algorithm_name=config.algorithm.name.lower(), + environment_name=config.environment.name, + phase="pre", + indices=range(1, K_PRE + 1), + seeds=schedule[training_seed]["pre"], + deterministic=config.evaluation.deterministic, + arm="shared", + ) + finally: + nominal_env.close() + if model_fingerprint(algorithm) != frozen_fingerprint: + raise RuntimeError("shared pre-shift evaluation mutated the frozen policy") + result.pre_shift_performance = float( + np.mean([episode.reward for episode in result.shared_pre_shift_episodes]) + ) + + # Shift introduction occurs only after training and the one shared pre segment. + shift_before = model_fingerprint(algorithm) + shock_env = _new_env( + config, + shifted=True, + environment_factory=environment_factory, + max_steps=8 if smoke else None, + ) + try: + get_effective = getattr(shock_env, "get_effective_parameters", None) + effective = dict(get_effective()) if callable(get_effective) else shifted_params + for key, expected in config.adaptation_benchmark.shift_parameters.items(): + if effective.get(key) != expected: + raise RuntimeError( + f"TEST-B parameter {key!r} did not apply: expected {expected!r}, " + f"got {effective.get(key)!r}" + ) + result.effective_shift_parameters = effective + result.shared_shock_episodes = _run_evaluation_segment( + algorithm=algorithm, + env=shock_env, + training_seed=training_seed, + algorithm_name=config.algorithm.name.lower(), + environment_name=config.environment.name, + phase="post", + indices=range(1, 6), + seeds=schedule[training_seed]["post"][:5], + deterministic=config.evaluation.deterministic, + arm="shared", + ) + finally: + shock_env.close() + if model_fingerprint(algorithm) != shift_before: + raise RuntimeError("shift introduction or shared shock evaluation mutated the policy") + result.shock_performance = float( + np.mean([episode.reward for episode in result.shared_shock_episodes]) + ) + + adaptive, fixed, fork_fingerprint = fork_adaptive_and_fixed(algorithm) + if fork_fingerprint != frozen_fingerprint: + raise RuntimeError("Adaptive/Fixed forks did not originate at the frozen fingerprint") + adapter = _adapter_for(config.algorithm.name.lower()) + post_history = [episode.post_shift_data() for episode in result.shared_shock_episodes] + adaptive_env = _new_env( + config, + shifted=True, + environment_factory=environment_factory, + max_steps=8 if smoke else None, + ) + try: + for episode_index in range(6, N_POST + 1): + boundary = episode_index - 1 + batch = build_update_batch(training_seed, post_history, block_episode=boundary) + update_log = run_adaptation_update(adaptive, adapter, batch) + result.update_blocks.append(update_log.to_dict()) + record = _run_evaluation_segment( + algorithm=adaptive, + env=adaptive_env, + training_seed=training_seed, + algorithm_name=config.algorithm.name.lower(), + environment_name=config.environment.name, + phase="post", + indices=[episode_index], + seeds=[schedule[training_seed]["post"][episode_index - 1]], + deterministic=config.evaluation.deterministic, + arm="adaptive", + initial_update_log=update_log, + )[0] + result.adaptive_episodes.append(record) + post_history.append(record.post_shift_data()) + finally: + adaptive_env.close() + validate_block_sequence([block["block_episode"] for block in result.update_blocks]) + + fixed_env = _new_env( + config, + shifted=True, + environment_factory=environment_factory, + max_steps=8 if smoke else None, + ) + try: + result.fixed_episodes = _run_evaluation_segment( + algorithm=fixed, + env=fixed_env, + training_seed=training_seed, + algorithm_name=config.algorithm.name.lower(), + environment_name=config.environment.name, + phase="post", + indices=range(6, N_POST + 1), + seeds=schedule[training_seed]["post"][5:], + deterministic=config.evaluation.deterministic, + arm="fixed", + ) + finally: + fixed_env.close() + result.fixed_final_fingerprint = fixed.fingerprint + if result.fixed_final_fingerprint != frozen_fingerprint: + raise RuntimeError("Fixed policy fingerprint changed during its evaluation arm") + + adaptive_post = result.shared_shock_episodes + result.adaptive_episodes + fixed_post = result.shared_shock_episodes + result.fixed_episodes + result.adaptive_recovery = _recover(result.shared_pre_shift_episodes, adaptive_post) + result.fixed_recovery = _recover(result.shared_pre_shift_episodes, fixed_post) + result.status = "completed" + except Exception as exc: + result.failure_reason = f"{type(exc).__name__}: {exc}" + logger.exception("Issue #265 replicate failed for training seed %s", training_seed) + return result + + +def run_adaptation_benchmark( + config: ExperimentConfig, + *, + output_dir: str | Path | None = None, + training_seeds: Optional[Sequence[int]] = None, + smoke: bool = False, + trainer_factory: TrainerFactory = get_trainer, + environment_factory: EnvironmentFactory = make_env, + config_path: str | Path | None = None, +) -> dict[str, Any]: + """Run the selected preregistered replicates and write JSON/CSV artifacts. + + A smoke run is visibly marked and uses one preregistered seed, a tiny + training budget, and short episodes. It validates execution only; its + outputs are not empirical research data. + """ + if config.adaptation_benchmark is None: + raise ValueError("configuration must include the Issue #265 adaptation_benchmark section") + if config.algorithm.name.strip().lower() not in {"ppo", "sac"}: + raise ValueError("Issue #265 supports only PPO and SAC") + if config.training is None: + raise ValueError("Issue #265 requires a training section") + schedule = frozen_schedule() + schedule_fp = schedule_fingerprint(schedule) + if training_seeds is None: + selected_seeds = [TRAINING_SEEDS[0]] if smoke else list(TRAINING_SEEDS) + else: + selected_seeds = [int(seed) for seed in training_seeds] + if not selected_seeds or len(set(selected_seeds)) != len(selected_seeds): + raise ValueError("training_seeds must be non-empty and unique") + if not set(selected_seeds).issubset(TRAINING_SEEDS): + raise ValueError("all selected training seeds must come from TRAINING_SEEDS") + if smoke and len(selected_seeds) != 1: + raise ValueError("smoke mode runs exactly one preregistered training seed") + + card = _card_path() + if not card.is_file(): + raise FileNotFoundError(f"Treatment Card is required before experiment execution: {card}") + card_sha = _sha256_file(card) + config_file = Path(config_path).resolve() if config_path is not None else None + config_file_sha = _sha256_file(config_file) if config_file is not None else None + + target_dir = Path(output_dir) if output_dir is not None else Path(config.output_dir) + target_dir.mkdir(parents=True, exist_ok=True) + for suffix in ("adaptation.json", "adaptation.csv"): + if (target_dir / suffix).exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {target_dir / suffix}") + training_root = target_dir / "training" + for seed in selected_seeds: + if (training_root / f"seed_{seed}").exists(): + raise FileExistsError(f"refusing to overwrite training output for seed {seed}") + + results = [ + _run_replicate( + config, + seed, + training_root / f"seed_{seed}", + schedule=schedule, + smoke=smoke, + trainer_factory=trainer_factory, + environment_factory=environment_factory, + ) + for seed in selected_seeds + ] + + vectors: dict[str, tuple[list[Optional[float]], list[Optional[float]]]] = {} + for cell in PRIMARY_CELLS: + vectors[cell] = ([None] * len(TRAINING_SEEDS), [None] * len(TRAINING_SEEDS)) + cell_name = f"drone_disturbed/{config.algorithm.name.strip().lower()}" + if cell_name not in vectors: + raise ValueError(f"configured Issue #265 cell {cell_name!r} is not preregistered") + fixed_vector, adaptive_vector = vectors[cell_name] + for replicate in results: + if replicate.status != "completed": + continue + position = TRAINING_SEEDS.index(replicate.training_seed) + fixed_vector[position] = float(replicate.fixed_recovery["truncated_recovery_time"]) + adaptive_vector[position] = float(replicate.adaptive_recovery["truncated_recovery_time"]) + paired = analyze_primary_cells(vectors) + family_decision = decide_family( + {cell: analysis.primary_p_value for cell, analysis in paired.items()} + ) + + provenance = _repository_metadata() + artifact = { + "schema_version": "1.0", + "protocol_version": config.adaptation_benchmark.protocol_version, + "issue": "265", + "run_type": "smoke" if smoke else "full_or_selected_research_run", + "treatment_card_sha256": card_sha, + "schedule_fingerprint": schedule_fp, + "experiment": { + "name": config.name, + "algorithm": config.algorithm.name.strip().lower(), + "environment": config.environment.name, + "scenario": config.adaptation_benchmark.scenario, + "planned_replicates": len(TRAINING_SEEDS), + "selected_training_seeds": selected_seeds, + "config_sha256": compute_config_sha256(config), + "config": config.model_dump(mode="json"), + }, + "replicates": [replicate.to_dict() for replicate in results], + "paired_analysis": {cell: result.to_dict() for cell, result in paired.items()}, + "family_decision": family_decision, + "failure_summary": { + "failed_replicates": [ + {"training_seed": replicate.training_seed, "reason": replicate.failure_reason} + for replicate in results + if replicate.status != "completed" + ], + "completed_replicates": sum(rep.status == "completed" for rep in results), + "valid_pairs": {cell: result.valid_n for cell, result in paired.items()}, + }, + "provenance": { + **provenance, + "treatment_card_path": str(card), + "config_path": str(config_file) if config_file else None, + "config_file_sha256": config_file_sha, + }, + "scientific_claim": "Harness execution alone does not establish empirical superiority.", + } + json_path, csv_path = write_adaptation_artifacts(artifact, target_dir) + artifact["artifact_paths"] = {"json": str(json_path), "csv": str(csv_path)} + return artifact + + +__all__ = ["ReplicateResult", "run_adaptation_benchmark"] diff --git a/src/adaptive_rl/benchmarking/adaptation_runtime.py b/src/adaptive_rl/benchmarking/adaptation_runtime.py new file mode 100644 index 0000000..92be7b1 --- /dev/null +++ b/src/adaptive_rl/benchmarking/adaptation_runtime.py @@ -0,0 +1,243 @@ +"""Deterministic episode collection for the Issue #265 fork protocol.""" + +from __future__ import annotations + +import random +from contextlib import contextmanager +from dataclasses import asdict, dataclass +from typing import Any, Iterator, Optional + +import gymnasium as gym +import numpy as np +import torch + +from adaptive_rl.protocol.adaptation import PostShiftEpisode, Transition +from adaptive_rl.protocol.fork import model_fingerprint + + +def _fingerprint(algorithm: Any) -> str: + fixed_fingerprint = getattr(algorithm, "fingerprint", None) + if isinstance(fixed_fingerprint, str): + return fixed_fingerprint + return model_fingerprint(algorithm) + + +@contextmanager +def _seed_episode(seed: int) -> Iterator[None]: + """Seed episode-side stochastic actions and restore the caller RNG states.""" + python_state = random.getstate() + numpy_state = np.random.get_state() + torch_state = torch.random.get_rng_state() + cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None + try: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if cuda_states is not None: + torch.cuda.manual_seed_all(seed) + yield + finally: + random.setstate(python_state) + np.random.set_state(numpy_state) + torch.random.set_rng_state(torch_state) + if cuda_states is not None: + torch.cuda.set_rng_state_all(cuda_states) + + +def _action_and_behavior(algorithm: Any, observation: Any, deterministic: bool): + model = getattr(algorithm, "model", None) + policy = getattr(model, "policy", None) if model is not None else None + if policy is None: + environment_action, _ = algorithm.predict(observation, deterministic=deterministic) + environment_action = np.asarray(environment_action) + return environment_action, environment_action, None, None + if hasattr(model, "rollout_buffer"): + obs_tensor, _ = policy.obs_to_tensor(observation) + with torch.no_grad(): + native_action, value, log_prob = policy(obs_tensor, deterministic=deterministic) + native_array = native_action.detach().cpu().numpy().reshape(-1) + if getattr(policy, "squash_output", False): + environment_action = policy.unscale_action(native_array) + else: + environment_action = np.clip( + native_array, + np.asarray(model.action_space.low), + np.asarray(model.action_space.high), + ) + return ( + np.asarray(native_array), + np.asarray(environment_action), + float(log_prob.reshape(-1)[0].cpu()), + float(value.reshape(-1)[0].cpu()), + ) + + environment_action, _ = algorithm.predict(observation, deterministic=deterministic) + environment_action = np.asarray(environment_action) + if getattr(policy, "squash_output", False): + native_action = policy.scale_action(environment_action) + else: + native_action = environment_action + return np.asarray(native_action), environment_action, None, None + + +def _behavior_value(algorithm: Any, observation: Any) -> float | None: + model = getattr(algorithm, "model", None) + if model is None or not hasattr(model, "rollout_buffer"): + return None + obs_tensor, _ = model.policy.obs_to_tensor(observation) + with torch.no_grad(): + value = model.policy.predict_values(obs_tensor) + return float(value.reshape(-1)[0].cpu()) + + +@dataclass(frozen=True) +class EpisodeRecord: + """Episode metrics plus complete transition data for later adaptation.""" + + training_seed: int + arm: str + algorithm: str + environment: str + phase: str + episode_index: int + episode_seed: int + reward: float + length: int + success: Optional[bool] + collision: Optional[bool] + terminated: bool + truncated: bool + policy_fingerprint_start: str + policy_fingerprint_end: str + update_block: Optional[int] + update_seed: Optional[int] + update_status: Optional[str] + parameter_delta_l2: Optional[float] + final_info: dict[str, Any] + transitions: tuple[Transition, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def post_shift_data(self) -> PostShiftEpisode: + if self.phase != "post": + raise ValueError("only completed post-shift episodes can enter adaptation data") + return PostShiftEpisode( + index=self.episode_index, + seed=self.episode_seed, + transitions=self.transitions, + ) + + +def evaluate_episode( + *, + algorithm: Any, + env: gym.Env, + training_seed: int, + phase: str, + episode_index: int, + episode_seed: int, + algorithm_name: str, + environment_name: str, + deterministic: bool, + arm: str, + update_log: Any = None, +) -> EpisodeRecord: + """Run and record one episode; no model update is reachable in this function.""" + if phase not in {"pre", "post"}: + raise ValueError("phase must be 'pre' or 'post'") + model = getattr(algorithm, "model", None) + if model is None and not isinstance(getattr(algorithm, "fingerprint", None), str): + raise RuntimeError("Cannot evaluate an uninitialized algorithm") + if model is not None: + model.policy.set_training_mode(False) + fingerprint_start = _fingerprint(algorithm) + transitions: list[Transition] = [] + rewards: list[float] = [] + success_values: list[bool] = [] + collisions: list[bool] = [] + observation, info = env.reset(seed=int(episode_seed)) + terminated = False + truncated = False + last_info: dict[str, Any] = dict(info) + + with _seed_episode(int(episode_seed)): + while not (terminated or truncated): + native_action, environment_action, behavior_log_prob, behavior_value = ( + _action_and_behavior(algorithm, observation, deterministic) + ) + next_observation, reward, terminated, truncated, step_info = env.step( + environment_action + ) + behavior_next_value = ( + _behavior_value(algorithm, next_observation) + if truncated and not terminated + else None + ) + transition = Transition( + observation=np.asarray(observation), + action=native_action, + environment_action=np.asarray(environment_action), + reward=float(reward), + next_observation=np.asarray(next_observation), + terminated=bool(terminated), + truncated=bool(truncated), + behavior_log_prob=behavior_log_prob, + behavior_value=behavior_value, + behavior_next_value=behavior_next_value, + ) + transitions.append(transition) + rewards.append(float(reward)) + if isinstance(step_info.get("success"), (bool, np.bool_)): + success_values.append(bool(step_info["success"])) + if isinstance(step_info.get("collision"), (bool, np.bool_)): + collisions.append(bool(step_info["collision"])) + last_info = dict(step_info) + observation = next_observation + + fingerprint_end = _fingerprint(algorithm) + if fingerprint_end != fingerprint_start: + raise RuntimeError("Policy parameters changed during an evaluation episode") + if not rewards: + raise RuntimeError("Environment returned an empty episode") + safe_info: dict[str, Any] = {} + for key, value in last_info.items(): + if isinstance(value, np.ndarray): + safe_info[str(key)] = value if np.isfinite(value).all() else None + elif isinstance(value, np.generic): + native = value.item() + safe_info[str(key)] = ( + native if not isinstance(native, float) or np.isfinite(native) else None + ) + elif value is None or isinstance(value, (str, bool, int)): + safe_info[str(key)] = value + elif isinstance(value, float): + safe_info[str(key)] = value if np.isfinite(value) else None + if any(isinstance(value, float) and not np.isfinite(value) for value in rewards): + raise FloatingPointError("episode produced a non-finite reward") + return EpisodeRecord( + training_seed=training_seed, + arm=arm, + algorithm=algorithm_name, + environment=environment_name, + phase=phase, + episode_index=episode_index, + episode_seed=int(episode_seed), + reward=float(sum(rewards)), + length=len(rewards), + success=success_values[-1] if success_values else None, + collision=any(collisions) if collisions else None, + terminated=bool(terminated), + truncated=bool(truncated), + policy_fingerprint_start=fingerprint_start, + policy_fingerprint_end=fingerprint_end, + update_block=getattr(update_log, "block_episode", None), + update_seed=getattr(update_log, "update_seed", None), + update_status=getattr(update_log, "status", None), + parameter_delta_l2=getattr(update_log, "parameter_delta_l2", None), + final_info=safe_info, + transitions=tuple(transitions), + ) + + +__all__ = ["EpisodeRecord", "evaluate_episode"] diff --git a/src/adaptive_rl/benchmarking/adaptation_statistics.py b/src/adaptive_rl/benchmarking/adaptation_statistics.py new file mode 100644 index 0000000..3ececd3 --- /dev/null +++ b/src/adaptive_rl/benchmarking/adaptation_statistics.py @@ -0,0 +1,139 @@ +"""Paired recovery-time and six-cell multiplicity analysis for Issue #265.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +from typing import Any, Optional, Sequence + +from adaptive_rl.protocol.constants import HORIZON, MIN_VALID_N, PLANNED_N, PRIMARY_CELLS +from adaptive_rl.protocol.statistics import ( + bootstrap_percentile_ci, + cohen_dz, + exact_sign_test, + exact_wilcoxon_signed_rank, + holm_adjust, + impute_differences, + paired_differences, + paired_t_interval, + paired_t_test, +) + + +@dataclass(frozen=True) +class PairedRecoveryAnalysis: + planned_n: int + completed_n: int + failed_n: int + valid_n: int + differences: list[Optional[float]] + mean_difference: Optional[float] + standard_deviation: Optional[float] + primary_p_value: Optional[float] + holm_adjusted_p_value: Optional[float] + interval_95: Optional[list[float]] + cohen_dz: Optional[float] + exact_sign_p_value: Optional[float] + exact_wilcoxon_p_value: Optional[float] + bootstrap_interval_95: Optional[list[float]] + failure_imputation_bounds: Optional[dict[str, dict[str, Any]]] + status: str + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +def analyze_paired_recovery( + fixed_t_h: Sequence[Optional[float]], + adaptive_t_h: Sequence[Optional[float]], + *, + planned_n: int = PLANNED_N, + horizon: float = HORIZON, + min_valid_n: int = MIN_VALID_N, +) -> PairedRecoveryAnalysis: + """Analyze one cell using the frozen paired tests and sensitivity methods.""" + if len(fixed_t_h) != len(adaptive_t_h): + raise ValueError("Fixed and Adaptive T_H vectors must have equal length.") + if len(fixed_t_h) != planned_n: + raise ValueError(f"Expected {planned_n} planned pairs, got {len(fixed_t_h)}.") + + completed_n = sum(f is not None and a is not None for f, a in zip(fixed_t_h, adaptive_t_h)) + failed_n = planned_n - completed_n + paired = paired_differences(fixed_t_h, adaptive_t_h) + differences = [ + None if fixed is None or adaptive is None else float(adaptive) - float(fixed) + for fixed, adaptive in zip(fixed_t_h, adaptive_t_h) + ] + worst = impute_differences(fixed_t_h, adaptive_t_h, horizon, "worst_for_adaptive") + best = impute_differences(fixed_t_h, adaptive_t_h, horizon, "best_for_adaptive") + bounds = { + "worst_for_adaptive": {"mean_difference": sum(worst) / planned_n, "per_replicate": worst}, + "best_for_adaptive": {"mean_difference": sum(best) / planned_n, "per_replicate": best}, + } + if len(paired) < min_valid_n: + return PairedRecoveryAnalysis( + planned_n=planned_n, + completed_n=completed_n, + failed_n=failed_n, + valid_n=len(paired), + differences=differences, + mean_difference=None, + standard_deviation=None, + primary_p_value=None, + holm_adjusted_p_value=None, + interval_95=None, + cohen_dz=None, + exact_sign_p_value=None, + exact_wilcoxon_p_value=None, + bootstrap_interval_95=None, + failure_imputation_bounds=bounds, + status="inconclusive", + ) + + primary = paired_t_test(paired, min_valid_n=min_valid_n) + return PairedRecoveryAnalysis( + planned_n=planned_n, + completed_n=completed_n, + failed_n=failed_n, + valid_n=len(paired), + differences=differences, + mean_difference=primary.mean, + standard_deviation=primary.std_dev, + primary_p_value=primary.p_value, + holm_adjusted_p_value=None, + interval_95=list(paired_t_interval(paired, min_valid_n=min_valid_n)), + cohen_dz=cohen_dz(paired), + exact_sign_p_value=exact_sign_test(paired, min_valid_n=min_valid_n).p_value, + exact_wilcoxon_p_value=exact_wilcoxon_signed_rank(paired, min_valid_n=min_valid_n).p_value, + bootstrap_interval_95=list(bootstrap_percentile_ci(paired, min_valid_n=min_valid_n)), + failure_imputation_bounds=bounds, + status="evaluable", + ) + + +def analyze_primary_cells( + outcomes: dict[str, tuple[Sequence[Optional[float]], Sequence[Optional[float]]]], +) -> dict[str, PairedRecoveryAnalysis]: + """Analyze the six frozen cells and apply Holm across the full cell family. + + Missing or invalid cells remain explicitly inconclusive. Their unavailable + p-values are represented as 1.0 during family correction, and remain null + in their own output record. + """ + if set(outcomes) != set(PRIMARY_CELLS): + raise ValueError("outcomes must contain exactly the preregistered PRIMARY_CELLS") + results = {cell: analyze_paired_recovery(*outcomes[cell]) for cell in PRIMARY_CELLS} + adjusted = holm_adjust( + [ + results[cell].primary_p_value if results[cell].primary_p_value is not None else 1.0 + for cell in PRIMARY_CELLS + ] + ) + return { + cell: replace(results[cell], holm_adjusted_p_value=adjusted[index]) + if results[cell].primary_p_value is not None + else results[cell] + for index, cell in enumerate(PRIMARY_CELLS) + } + + +__all__ = ["PairedRecoveryAnalysis", "analyze_paired_recovery", "analyze_primary_cells"] diff --git a/src/adaptive_rl/cli.py b/src/adaptive_rl/cli.py index 3caffc2..0ce5a40 100644 --- a/src/adaptive_rl/cli.py +++ b/src/adaptive_rl/cli.py @@ -28,7 +28,7 @@ benchmark_app = typer.Typer( name="benchmark", - help="PPO learning-curve benchmark commands (PPO only).", + help="Benchmark commands for learning curves and online adaptation.", no_args_is_help=True, ) app.add_typer(benchmark_app, name="benchmark") @@ -381,6 +381,111 @@ def benchmark_budgets( raise typer.Exit(code=1) +@benchmark_app.command(name="adaptation") +def benchmark_adaptation( + config: Path = typer.Option( + Path("configs/drone_distribution_shift.yaml"), + "--config", + "-c", + help="Issue #265 nominal training and TEST-B configuration", + ), + algorithm: Optional[str] = typer.Option( + None, "--algorithm", help="Algorithm cell: ppo or sac (defaults to config value)" + ), + training_seeds: Optional[str] = typer.Option( + None, + "--training-seeds", + help="Comma-separated preregistered training seeds; defaults to all ten", + ), + output_dir: Optional[Path] = typer.Option( + None, "--output-dir", help="Directory for Issue #265 JSON/CSV and training artifacts" + ), + deterministic: Optional[bool] = typer.Option( + None, "--deterministic/--stochastic", help="Override action selection for all evaluations" + ), + smoke: bool = typer.Option( + False, + "--smoke", + help="Run one explicitly labeled, reduced-size machinery check (not research data)", + ), +) -> None: + """Run the preregistered train-once, forked Adaptive-vs-Fixed experiment.""" + try: + exp_config = load_config(config) + if algorithm is not None: + selected_algorithm = algorithm.strip().lower() + if selected_algorithm not in {"ppo", "sac"}: + raise ValueError("--algorithm must be 'ppo' or 'sac'") + algorithm_config = exp_config.algorithm.model_copy(deep=True) + if selected_algorithm != algorithm_config.name.strip().lower(): + algorithm_config.name = selected_algorithm + if selected_algorithm == "sac": + algorithm_config.parameters = { + "buffer_size": 100_000, + "learning_starts": 100, + "train_freq": 1, + "gradient_steps": 1, + "tau": 0.005, + "ent_coef": "auto", + } + else: + algorithm_config.parameters = { + "n_steps": 1024, + "n_epochs": 10, + "clip_range": 0.2, + "ent_coef": 0.01, + } + exp_config = exp_config.model_copy( + update={"algorithm": algorithm_config}, deep=True + ) + if deterministic is not None: + evaluation_config = exp_config.evaluation.model_copy(deep=True) + evaluation_config.deterministic = deterministic + exp_config = exp_config.model_copy(update={"evaluation": evaluation_config}, deep=True) + + selected_seeds = None + if training_seeds is not None: + tokens = [token.strip() for token in training_seeds.split(",")] + if not tokens or any(not token for token in tokens): + raise ValueError("--training-seeds expects comma-separated integers") + try: + selected_seeds = [int(token) for token in tokens] + except ValueError as exc: + raise ValueError("--training-seeds expects comma-separated integers") from exc + + from adaptive_rl.benchmarking.adaptation_runner import run_adaptation_benchmark + + artifact = run_adaptation_benchmark( + exp_config, + output_dir=output_dir, + training_seeds=selected_seeds, + smoke=smoke, + config_path=config, + ) + except Exception as err: + console.print(f"[bold red]Issue #265 benchmark failed:[/bold red] {err}") + raise typer.Exit(code=1) + + failed = artifact["failure_summary"]["failed_replicates"] + completed = artifact["failure_summary"]["completed_replicates"] + console.print( + Panel.fit( + f"[bold]{'Smoke check' if smoke else 'Issue #265 benchmark'} finished[/bold]\n\n" + f"• [bold]Run type:[/bold] {artifact['run_type']}\n" + f"• [bold]Algorithm:[/bold] {artifact['experiment']['algorithm']}\n" + f"• [bold]Completed replicates:[/bold] {completed}\n" + f"• [bold]Failed replicates:[/bold] {len(failed)}\n" + f"• [bold]JSON:[/bold] {Path(output_dir or exp_config.output_dir) / 'adaptation.json'}\n" + f"• [bold]CSV:[/bold] {Path(output_dir or exp_config.output_dir) / 'adaptation.csv'}\n" + f"• [bold]Scientific result:[/bold] not established by harness execution", + title="Online Adaptation Benchmark", + border_style="yellow" if smoke or failed else "green", + ) + ) + if failed: + raise typer.Exit(code=1) + + @app.command(context_settings={"allow_extra_args": True}) def evaluate( ctx: typer.Context, diff --git a/src/adaptive_rl/config.py b/src/adaptive_rl/config.py index 7eccc6a..7a847fd 100644 --- a/src/adaptive_rl/config.py +++ b/src/adaptive_rl/config.py @@ -143,6 +143,29 @@ def _validate_evaluation_seeds(cls, values: list[int]) -> list[int]: return values +class AdaptationBenchmarkConfig(BaseModel): + """Frozen TEST-B cell declaration for Issue #265.""" + + model_config = ConfigDict(extra="forbid") + + protocol_version: str = "2.0" + scenario: str = "TEST-B" + shift_parameters: Dict[str, Any] = Field( + default_factory=lambda: {"num_obstacles": 12, "wind_speed": 4.0, "gust_sigma": 0.6} + ) + + @model_validator(mode="after") + def _validate_test_b(self) -> "AdaptationBenchmarkConfig": + expected = {"num_obstacles": 12, "wind_speed": 4.0, "gust_sigma": 0.6} + if self.protocol_version != "2.0": + raise ValueError("Issue #265 requires protocol_version '2.0'") + if self.scenario != "TEST-B" or self.shift_parameters != expected: + raise ValueError( + "Issue #265's primary cell is exactly TEST-B (12 obstacles, wind 4.0, gust 0.6)" + ) + return self + + class ExperimentConfig(BaseModel): """Top-level configuration schema for an AdaptiveRL experiment.""" @@ -166,6 +189,10 @@ class ExperimentConfig(BaseModel): default=None, description="Optional benchmark settings for training-budget learning curves.", ) + adaptation_benchmark: Optional[AdaptationBenchmarkConfig] = Field( + default=None, + description="Frozen Issue #265 TEST-B online-adaptation cell.", + ) @model_validator(mode="before") @classmethod diff --git a/src/adaptive_rl/environments/__init__.py b/src/adaptive_rl/environments/__init__.py index 5947138..6bf6393 100644 --- a/src/adaptive_rl/environments/__init__.py +++ b/src/adaptive_rl/environments/__init__.py @@ -58,6 +58,22 @@ def register_default_environments() -> None: ), ) + if "drone_disturbed" not in list_environments(): + register( + "drone_disturbed", + lambda **kwargs: DroneNavigation3DEnv(**kwargs), + metadata=EnvironmentMetadata( + name="drone_disturbed", + description="Drone navigation with deterministic steady-wind and OU gust parameters.", + observation_type="box", + action_type="continuous", + version="0.2.0", + max_episode_steps=200, + reward_range=(-100.0, 100.0), + tags=["continuous", "drone", "distribution-shift", "wind", "gust"], + ), + ) + # Automatically register default environments register_default_environments() diff --git a/src/adaptive_rl/environments/drone.py b/src/adaptive_rl/environments/drone.py index 8705812..b093b62 100644 --- a/src/adaptive_rl/environments/drone.py +++ b/src/adaptive_rl/environments/drone.py @@ -130,7 +130,6 @@ def step(self, action_acceleration: np.ndarray) -> Tuple[np.ndarray, np.ndarray] raw_acc = np.asarray(action_acceleration, dtype=np.float64) if raw_acc.shape != (3,): raise ValueError(f"Acceleration command must have shape (3,), got {raw_acc.shape}") - clamped_acc = np.clip(raw_acc, -self.max_acceleration, self.max_acceleration) self.state.acceleration = clamped_acc @@ -395,6 +394,10 @@ def __init__( terminate_on_collision: bool = True, render_mode: Optional[str] = None, split: Optional[str] = None, + wind_speed: float = 0.0, + gust_sigma: float = 0.0, + gust_theta: float = 0.15, + wind_direction: Tuple[float, float, float] = (1.5, 0.5, 0.0), ) -> None: super().__init__() if any(b <= 0.0 for b in bounds): @@ -405,6 +408,19 @@ def __init__( raise ValueError(f"target_radius must be positive, got {target_radius}") if collision_radius <= 0.0: raise ValueError(f"collision_radius must be positive, got {collision_radius}") + if not np.isfinite(wind_speed) or wind_speed < 0.0: + raise ValueError(f"wind_speed must be finite and non-negative, got {wind_speed}") + if not np.isfinite(gust_sigma) or gust_sigma < 0.0: + raise ValueError(f"gust_sigma must be finite and non-negative, got {gust_sigma}") + if not np.isfinite(gust_theta) or gust_theta <= 0.0: + raise ValueError(f"gust_theta must be finite and positive, got {gust_theta}") + direction = np.asarray(wind_direction, dtype=np.float64) + if ( + direction.shape != (3,) + or not np.isfinite(direction).all() + or np.linalg.norm(direction) == 0 + ): + raise ValueError("wind_direction must be a finite, non-zero 3-vector") self.split: Optional[str] = None if split is not None: @@ -447,6 +463,11 @@ def __init__( self.action_penalty_weight = float(action_penalty_weight) self.terminate_on_collision = terminate_on_collision self.render_mode = render_mode + self.wind_speed = float(wind_speed) + self.gust_sigma = float(gust_sigma) + self.gust_theta = float(gust_theta) + self.wind_direction = direction / np.linalg.norm(direction) + self._gust_velocity = np.zeros(3, dtype=np.float64) self.max_diagonal = float(np.linalg.norm(self.bounds)) @@ -546,6 +567,7 @@ def _get_info(self) -> Dict[str, Any]: "num_obstacles": len(self._obstacles), "min_obstacle_distance": min_obs_dist if self._obstacles else float("inf"), "altitude": float(self._position[2]), + "wind_velocity": self._wind_velocity(self._position).copy(), } current_split = self._active_split if self._active_split is not None else self.split if current_split is not None: @@ -553,6 +575,27 @@ def _get_info(self) -> Dict[str, Any]: info["split_seed"] = self._last_split_seed return info + def _wind_velocity(self, position: np.ndarray) -> np.ndarray: + altitude_factor = 1.0 + 0.02 * max(0.0, float(position[2])) + return self.wind_direction * self.wind_speed * altitude_factor + self._gust_velocity + + def get_effective_parameters(self) -> Dict[str, Any]: + """Return the physical environment parameters used by this instance.""" + return { + "bounds": list(self.bounds), + "num_obstacles": self.num_obstacles, + "obstacle_radius": self.obstacle_radius, + "wind_speed": self.wind_speed, + "gust_sigma": self.gust_sigma, + "gust_theta": self.gust_theta, + "wind_direction": self.wind_direction.tolist(), + "wind_altitude_shear": 0.02, + "max_gust": 4.0, + "linear_damping": self.kinematics.linear_damping, + "max_acceleration": self.kinematics.max_acceleration, + "max_steps": self.max_steps, + } + @property def drone_state(self) -> DroneState3D: """Convenience property exposing the underlying drone state.""" @@ -623,6 +666,7 @@ def reset( self._position = self.default_start.copy() self._velocity = np.zeros(3, dtype=np.float64) self._goal = self.default_goal.copy() + self._gust_velocity = np.zeros(3, dtype=np.float64) self.kinematics.reset(self._position, self._velocity) @@ -654,7 +698,19 @@ def step( self._current_step += 1 - acc_command = act_arr * self.kinematics.max_acceleration + # Exact discrete OU update: stationary per-axis gust standard deviation + # approaches sigma/sqrt(2*theta), as specified by the shift config. + if self.gust_sigma > 0.0: + noise = self.np_random.normal(size=3) + self._gust_velocity += ( + -self.gust_theta * self._gust_velocity * self.kinematics.dt + + self.gust_sigma * np.sqrt(self.kinematics.dt) * noise + ) + gust_magnitude = float(np.linalg.norm(self._gust_velocity)) + if gust_magnitude > 4.0: + self._gust_velocity *= 4.0 / gust_magnitude + wind_acceleration = self.kinematics.linear_damping * self._wind_velocity(self._position) + acc_command = act_arr * self.kinematics.max_acceleration + wind_acceleration new_pos, new_vel = self.kinematics.step(acc_command) self._position = new_pos self._velocity = new_vel diff --git a/src/adaptive_rl/protocol/__init__.py b/src/adaptive_rl/protocol/__init__.py index 964ffa7..f0128e6 100644 --- a/src/adaptive_rl/protocol/__init__.py +++ b/src/adaptive_rl/protocol/__init__.py @@ -9,6 +9,15 @@ ``shift_runner.py``, and do not make any experimental cell executable. """ +from adaptive_rl.protocol.adaptation import ( + AdaptationAdapter, + PostShiftEpisode, + Transition, + UpdateBatch, + build_update_batch, + call_update_atomically, + validate_block_sequence, +) from adaptive_rl.protocol.constants import ( ALPHA, BOOTSTRAP_REPS, @@ -74,6 +83,7 @@ ) __all__ = [ + "AdaptationAdapter", "ALPHA", "BOOTSTRAP_REPS", "BOOTSTRAP_SEED", @@ -95,6 +105,7 @@ "PROTOCOL_VERSION", "PairedTTest", "PERSISTENCE", + "PostShiftEpisode", "RECOVERY_STATUSES", "RECOVERY_THRESHOLD", "RecoveryResult", @@ -105,10 +116,14 @@ "STATUS_RIGHT_CENSORED", "SignTestResult", "TRAINING_SEEDS", + "Transition", + "UpdateBatch", "WINDOW", "WilcoxonResult", "bootstrap_percentile_ci", "build_schedule", + "build_update_batch", + "call_update_atomically", "cohen_dz", "compute_recovery", "decide_family", @@ -128,5 +143,6 @@ "student_t_cdf", "student_t_ppf", "trailing_window_means", + "validate_block_sequence", "validate_schedule", ] diff --git a/src/adaptive_rl/protocol/adaptation.py b/src/adaptive_rl/protocol/adaptation.py new file mode 100644 index 0000000..bbb088f --- /dev/null +++ b/src/adaptive_rl/protocol/adaptation.py @@ -0,0 +1,163 @@ +"""Episode-bounded data boundary for Issue #265 online updates.""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +from typing import Any, Protocol, Sequence + +import numpy as np + +from adaptive_rl.protocol.constants import N_POST, N_UPDATE +from adaptive_rl.protocol.seeds import derive_seed + + +@dataclass(frozen=True) +class Transition: + """One recorded environment transition, detached from live environment state.""" + + observation: np.ndarray + action: np.ndarray + reward: float + next_observation: np.ndarray + terminated: bool + truncated: bool + environment_action: np.ndarray | None = None + behavior_log_prob: float | None = None + behavior_value: float | None = None + behavior_next_value: float | None = None + + def __post_init__(self) -> None: + for name in ("observation", "action", "next_observation"): + value = np.array(getattr(self, name), copy=True) + if value.dtype.kind not in "biuf": + raise ValueError(f"{name} must have a numeric dtype") + if not np.isfinite(value).all(): + raise ValueError(f"{name} must contain only finite values") + value.setflags(write=False) + object.__setattr__(self, name, value) + if self.environment_action is not None: + value = np.array(self.environment_action, copy=True) + if value.dtype.kind not in "biuf" or not np.isfinite(value).all(): + raise ValueError("environment_action must be a finite numeric array") + value.setflags(write=False) + object.__setattr__(self, "environment_action", value) + if not np.isfinite(float(self.reward)): + raise ValueError("reward must be finite") + object.__setattr__(self, "reward", float(self.reward)) + if not isinstance(self.terminated, bool) or not isinstance(self.truncated, bool): + raise ValueError("terminated and truncated must be booleans") + for name in ("behavior_log_prob", "behavior_value", "behavior_next_value"): + value = getattr(self, name) + if value is not None: + if not np.isfinite(float(value)): + raise ValueError(f"{name} must be finite when provided") + object.__setattr__(self, name, float(value)) + + +@dataclass(frozen=True) +class PostShiftEpisode: + """Completed post-shift episode with its protocol index and derived seed.""" + + index: int + seed: int + transitions: tuple[Transition, ...] + + def __post_init__(self) -> None: + if not 1 <= self.index <= N_POST: + raise ValueError(f"post-shift episode index must be in [1, {N_POST}]") + if not self.transitions: + raise ValueError("a completed episode must contain at least one transition") + + +@dataclass(frozen=True) +class UpdateBatch: + """Cumulative post-shift experience visible at exactly one update boundary.""" + + block_episode: int + seed: int + visible_episode_indices: tuple[int, ...] + transitions: tuple[Transition, ...] + + +class AdaptationAdapter(Protocol): + """Algorithm-specific adapter interface; called only at episode boundaries.""" + + def update(self, algorithm: Any, batch: UpdateBatch) -> None: ... + + +def build_update_batch( + training_seed: int, + completed_episodes: Sequence[PostShiftEpisode], + *, + block_episode: int, +) -> UpdateBatch: + """Build Bk from precisely post-shift episodes 1..k. + + The builder requires the full ordered prefix. It has no parameter for + pre-shift or future data, preventing accidental cross-phase exposure. + """ + if not 5 <= block_episode <= 14: + raise ValueError("Update blocks exist only after post-shift episodes 5 through 14") + if len(completed_episodes) != block_episode: + raise ValueError( + f"B{block_episode} requires exactly episodes 1..{block_episode}; " + f"received {len(completed_episodes)} episodes" + ) + expected_indices = tuple(range(1, block_episode + 1)) + actual_indices = tuple(episode.index for episode in completed_episodes) + if actual_indices != expected_indices: + raise ValueError( + f"B{block_episode} can see only ordered completed episodes " + f"1..{block_episode}; got {actual_indices}" + ) + expected_seeds = tuple(derive_seed(training_seed, "post", index) for index in expected_indices) + actual_seeds = tuple(episode.seed for episode in completed_episodes) + if actual_seeds != expected_seeds: + raise ValueError("post-shift episode seeds do not match the preregistered schedule") + + transitions = tuple( + transition for episode in completed_episodes for transition in episode.transitions + ) + return UpdateBatch( + block_episode=block_episode, + seed=derive_seed(training_seed, "update", block_episode - 5), + visible_episode_indices=expected_indices, + transitions=transitions, + ) + + +def validate_block_sequence(block_episodes: Sequence[int]) -> None: + """Require the complete preregistered B5..B14 ordering (never B15).""" + expected = tuple(range(5, 5 + N_UPDATE)) + actual = tuple(block_episodes) + if actual != expected: + raise ValueError(f"update ordering must be exactly {expected}, got {actual}") + + +def call_update_atomically(algorithm: Any, adapter: AdaptationAdapter, batch: UpdateBatch) -> None: + """Run an update with wrapper-state rollback if it raises. + + A failed block remains a failure at the caller; rollback only prevents a + partially-mutated policy from leaking into later evaluation. + """ + if not isinstance(getattr(algorithm, "__dict__", None), dict): + raise TypeError("algorithm adapter target must expose instance state") + snapshot = copy.deepcopy(algorithm.__dict__) + try: + adapter.update(algorithm, batch) + except BaseException: + algorithm.__dict__.clear() + algorithm.__dict__.update(snapshot) + raise + + +__all__ = [ + "AdaptationAdapter", + "PostShiftEpisode", + "Transition", + "UpdateBatch", + "build_update_batch", + "call_update_atomically", + "validate_block_sequence", +] diff --git a/src/adaptive_rl/protocol/fork.py b/src/adaptive_rl/protocol/fork.py new file mode 100644 index 0000000..4d76e7d --- /dev/null +++ b/src/adaptive_rl/protocol/fork.py @@ -0,0 +1,121 @@ +"""Model-state hashing and independent policy forks for paired experiments.""" + +from __future__ import annotations + +import copy +import hashlib +import struct +from typing import Any + +import torch + + +def _state_module(model_or_wrapper: Any) -> Any: + model = getattr(model_or_wrapper, "model", model_or_wrapper) + if callable(getattr(model, "state_dict", None)): + return model + policy = getattr(model, "policy", None) + if policy is not None and callable(getattr(policy, "state_dict", None)): + return policy + raise TypeError("Expected a model or algorithm wrapper with a state_dict().") + + +def policy_state_tensors(model_or_wrapper: Any) -> dict[str, Any]: + model = getattr(model_or_wrapper, "model", model_or_wrapper) + state = { + f"policy.{key}": value + for key, value in _state_module(model_or_wrapper).state_dict().items() + } + # SAC's automatic entropy coefficient is a trainable model parameter stored + # on the algorithm object rather than inside SACPolicy. + log_ent_coef = getattr(model, "log_ent_coef", None) + if isinstance(log_ent_coef, torch.Tensor): + state["algorithm.log_ent_coef"] = log_ent_coef + return state + + +def model_fingerprint(model_or_wrapper: Any) -> str: + """Return a stable SHA-256 fingerprint of a model's complete state dict. + + Stable-Baselines3 wrappers expose their model as ``.model``. Hashing the + state dict includes parameters and persistent buffers, with names, shapes, + and dtypes included to distinguish structurally different models. + """ + digest = hashlib.sha256() + for name, tensor in sorted(policy_state_tensors(model_or_wrapper).items()): + if not hasattr(tensor, "detach"): + raise TypeError(f"Model state entry {name!r} is not a tensor.") + value = tensor.detach().cpu().contiguous() + digest.update(name.encode("utf-8")) + digest.update(b"\0") + digest.update(str(value.dtype).encode("ascii")) + digest.update(b"\0") + digest.update(struct.pack("!I", value.ndim)) + for dimension in value.shape: + digest.update(struct.pack("!Q", int(dimension))) + digest.update(value.view(torch.uint8).numpy().tobytes(order="C")) + return digest.hexdigest() + + +def clone_algorithm(algorithm: Any) -> Any: + """Deep-clone an algorithm wrapper and reject clones with shared tensors. + + ``deepcopy`` is intentional: SB3 algorithms contain mutable optimizers, + buffers, counters, and RNG state in addition to policy parameters. + """ + clone = copy.deepcopy(algorithm) + original_model = getattr(algorithm, "model", None) + clone_model = getattr(clone, "model", None) + if original_model is None or clone_model is None: + raise TypeError("Algorithm and clone must both have initialized models.") + original_state = policy_state_tensors(algorithm) + clone_state = policy_state_tensors(clone) + if original_state.keys() != clone_state.keys(): + raise RuntimeError("Cloned model state has a different structure.") + for name in original_state: + left = original_state[name] + right = clone_state[name] + if left.shape != right.shape or left.dtype != right.dtype: + raise RuntimeError(f"Cloned model state {name!r} has a different shape or dtype.") + if left.data_ptr() == right.data_ptr(): + raise RuntimeError(f"Cloned model state {name!r} shares storage with its source.") + if not left.equal(right): + raise RuntimeError(f"Cloned model state {name!r} differs from its source.") + return clone + + +class FrozenPolicy: + """Prediction-only facade that exposes no training or model mutation API.""" + + __slots__ = ("__algorithm",) + + def __init__(self, algorithm: Any) -> None: + if not callable(getattr(algorithm, "predict", None)): + raise TypeError("Frozen policy requires an algorithm with predict().") + self.__algorithm = algorithm + + def predict(self, observation: Any, deterministic: bool = True) -> Any: + return self.__algorithm.predict(observation, deterministic=deterministic) + + @property + def fingerprint(self) -> str: + return model_fingerprint(self.__algorithm) + + +def fork_adaptive_and_fixed(algorithm: Any) -> tuple[Any, FrozenPolicy, str]: + """Return an independent adaptive clone and prediction-only fixed arm.""" + expected = model_fingerprint(algorithm) + adaptive = clone_algorithm(algorithm) + fixed = FrozenPolicy(clone_algorithm(algorithm)) + if model_fingerprint(adaptive) != expected or fixed.fingerprint != expected: + raise RuntimeError("Forked policies do not match the frozen model fingerprint.") + return adaptive, fixed, expected + + +__all__ = [ + "FrozenPolicy", + "clone_algorithm", + "fork_adaptive_and_fixed", + "model_fingerprint", + "policy_state_tensors", +] diff --git a/src/adaptive_rl/training/trainer.py b/src/adaptive_rl/training/trainer.py index 17ddf0d..c464f26 100644 --- a/src/adaptive_rl/training/trainer.py +++ b/src/adaptive_rl/training/trainer.py @@ -17,6 +17,7 @@ import torch from adaptive_rl.algorithms.ppo import PPOAlgorithm +from adaptive_rl.algorithms.sac import SACAlgorithm from adaptive_rl.config import ExperimentConfig from adaptive_rl.environments.registry import make_env from adaptive_rl.training.callbacks import ( @@ -55,8 +56,8 @@ class TrainingResult: training_time_seconds: float = 0.0 -class PPOTrainer: - """Trainer orchestrating PPO policy learning on the drone navigation environment.""" +class AlgorithmTrainer: + """Trainer orchestrating supported Stable-Baselines3 algorithm learning.""" def __init__( self, @@ -93,7 +94,11 @@ def __init__( gamma = algo_params.pop("gamma", self.config.algorithm.gamma) batch_size = algo_params.pop("batch_size", self.config.algorithm.batch_size) seed = algo_params.pop("seed", self.config.seed) - self.algorithm = PPOAlgorithm( + algorithm_name = self.config.algorithm.name.strip().lower() + algorithm_types = {"ppo": PPOAlgorithm, "sac": SACAlgorithm} + if algorithm_name not in algorithm_types: + raise ValueError(f"Unsupported training algorithm: {self.config.algorithm.name!r}") + self.algorithm = algorithm_types[algorithm_name]( env=self.env, learning_rate=lr, gamma=gamma, @@ -203,10 +208,22 @@ def close(self) -> None: ) +class PPOTrainer(AlgorithmTrainer): + """Trainer for PPO; retained as the explicit PPO-facing public class.""" + + +class SACTrainer(AlgorithmTrainer): + """Trainer for SAC using the same callbacks and artifact lifecycle.""" + + def get_trainer( config: ExperimentConfig, env: Optional[gym.Env] = None, callbacks: Optional[List[BaseCallback]] = None, -) -> PPOTrainer: - """Factory returning the trainer based on configuration.""" - return PPOTrainer(config=config, env=env, callbacks=callbacks) +) -> AlgorithmTrainer: + """Factory for the configured PPO or SAC algorithm trainer.""" + trainer_types = {"ppo": PPOTrainer, "sac": SACTrainer} + algorithm_name = config.algorithm.name.strip().lower() + if algorithm_name not in trainer_types: + raise ValueError(f"Unsupported training algorithm: {config.algorithm.name!r}") + return trainer_types[algorithm_name](config=config, env=env, callbacks=callbacks) diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py new file mode 100644 index 0000000..a261366 --- /dev/null +++ b/tests/test_adaptation_artifacts.py @@ -0,0 +1,72 @@ +"""Structured artifact serialization and no-overwrite guarantees.""" + +from __future__ import annotations + +import csv +import json + +import numpy as np +import pytest + +from adaptive_rl.benchmarking.adaptation_artifacts import write_adaptation_artifacts + + +def _artifact(): + return { + "schema_version": "1.0", + "protocol_version": "2.0", + "issue": "265", + "experiment": {"algorithm": "ppo", "environment": "drone"}, + "replicates": [ + { + "training_seed": 31001, + "shared_pre_shift_episodes": [ + { + "episode_index": 1, + "episode_seed": 123, + "reward": np.float64(2.0), + "length": 15, + "success": True, + "collision": False, + "terminated": True, + "truncated": False, + "transitions": [{"observation": np.asarray([0.1, 0.2])}], + } + ], + "shared_shock_episodes": [], + "adaptive_episodes": [], + "fixed_episodes": [], + } + ], + } + + +def test_json_and_flattened_csv_are_written_without_opaque_objects(tmp_path) -> None: + json_path, csv_path = write_adaptation_artifacts(_artifact(), tmp_path) + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["replicates"][0]["shared_pre_shift_episodes"][0]["transitions"][0][ + "observation" + ] == [0.1, 0.2] + with csv_path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + assert len(rows) == 1 + assert rows[0]["arm"] == "shared" + assert rows[0]["phase"] == "pre" + assert rows[0]["episode_seed"] == "123" + + +def test_existing_artifacts_are_never_overwritten(tmp_path) -> None: + json_path, csv_path = write_adaptation_artifacts(_artifact(), tmp_path) + before_json = json_path.read_bytes() + before_csv = csv_path.read_bytes() + with pytest.raises(FileExistsError): + write_adaptation_artifacts(_artifact(), tmp_path) + assert json_path.read_bytes() == before_json + assert csv_path.read_bytes() == before_csv + + +def test_nonfinite_values_are_rejected_for_strict_json(tmp_path) -> None: + data = _artifact() + data["value"] = float("nan") + with pytest.raises(ValueError, match="JSON compliant"): + write_adaptation_artifacts(data, tmp_path) diff --git a/tests/test_adaptation_environment.py b/tests/test_adaptation_environment.py new file mode 100644 index 0000000..153cdaf --- /dev/null +++ b/tests/test_adaptation_environment.py @@ -0,0 +1,67 @@ +"""Deterministic wind shift support in the existing drone environment.""" + +from __future__ import annotations + +import numpy as np + +from adaptive_rl.config import load_config +from adaptive_rl.environments.drone import DroneNavigation3DEnv + + +def test_issue_265_config_freezes_nominal_and_test_b_parameters() -> None: + config = load_config("configs/drone_distribution_shift.yaml") + assert config.adaptation_benchmark is not None + assert config.adaptation_benchmark.scenario == "TEST-B" + assert config.environment.parameters["num_obstacles"] == 8 + assert config.environment.parameters["wind_speed"] == 0.5 + assert config.adaptation_benchmark.shift_parameters == { + "num_obstacles": 12, + "wind_speed": 4.0, + "gust_sigma": 0.6, + } + + +def test_steady_wind_is_an_external_force_and_keeps_action_limit() -> None: + env = DroneNavigation3DEnv( + bounds=(30.0, 30.0, 15.0), + num_obstacles=0, + max_steps=2, + wind_speed=4.0, + gust_sigma=0.0, + linear_damping=0.05, + ) + try: + env.reset(seed=17) + env.step(np.zeros(3, dtype=np.float32)) + state = env.drone_state + assert state.velocity[0] > 0.0 + assert np.isclose(np.linalg.norm(state.acceleration), 0.22, atol=1e-7) + assert env.action_space.contains(np.zeros(3, dtype=np.float32)) + finally: + env.close() + + +def test_ou_gust_stream_repeats_for_same_episode_seed() -> None: + environments = [ + DroneNavigation3DEnv( + bounds=(30.0, 30.0, 15.0), + num_obstacles=0, + max_steps=3, + wind_speed=4.0, + gust_sigma=0.6, + gust_theta=0.15, + ) + for _ in range(2) + ] + try: + for env in environments: + env.reset(seed=791) + for _ in range(3): + outputs = [env.step(np.zeros(3, dtype=np.float32)) for env in environments] + np.testing.assert_array_equal(outputs[0][0], outputs[1][0]) + np.testing.assert_array_equal( + outputs[0][4]["wind_velocity"], outputs[1][4]["wind_velocity"] + ) + finally: + for env in environments: + env.close() diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py new file mode 100644 index 0000000..9b0bf3a --- /dev/null +++ b/tests/test_adaptation_smoke.py @@ -0,0 +1,52 @@ +"""CI-sized end-to-end Issue #265 protocol smoke test.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from adaptive_rl.cli import app +from adaptive_rl.protocol.constants import TRAINING_SEEDS +from adaptive_rl.protocol.seeds import frozen_schedule, schedule_fingerprint + + +def test_cli_adaptation_smoke_runs_complete_protocol_and_writes_artifacts(tmp_path: Path) -> None: + output_dir = tmp_path / "adaptation-smoke" + result = CliRunner().invoke( + app, + [ + "benchmark", + "adaptation", + "--smoke", + "--output-dir", + str(output_dir), + ], + ) + assert result.exit_code == 0, result.output + artifact = json.loads((output_dir / "adaptation.json").read_text(encoding="utf-8")) + assert artifact["run_type"] == "smoke" + replicate = artifact["replicates"][0] + assert replicate["training_seed"] == TRAINING_SEEDS[0] + assert replicate["status"] == "completed" + train_config = replicate["training_provenance"]["effective_config"] + assert train_config["adaptation_benchmark"] is None + assert train_config["environment"]["parameters"]["num_obstacles"] == 8 + assert train_config["environment"]["parameters"]["wind_speed"] == 0.5 + assert train_config["environment"]["parameters"]["gust_sigma"] == 0.15 + assert replicate["schedule_fingerprint"] == schedule_fingerprint(frozen_schedule()) + assert len(replicate["shared_pre_shift_episodes"]) == 15 + assert len(replicate["shared_shock_episodes"]) == 5 + assert len(replicate["adaptive_episodes"]) == 10 + assert len(replicate["fixed_episodes"]) == 10 + assert [block["block_episode"] for block in replicate["update_blocks"]] == list(range(5, 15)) + assert [block["visible_episode_indices"] for block in replicate["update_blocks"]] == [ + list(range(1, boundary + 1)) for boundary in range(5, 15) + ] + assert [e["episode_seed"] for e in replicate["adaptive_episodes"]] == [ + e["episode_seed"] for e in replicate["fixed_episodes"] + ] + assert replicate["fixed_final_fingerprint"] == replicate["frozen_fingerprint"] + assert artifact["paired_analysis"]["drone_disturbed/ppo"]["status"] == "inconclusive" + assert (output_dir / "adaptation.csv").is_file() diff --git a/tests/test_adaptation_statistics.py b/tests/test_adaptation_statistics.py new file mode 100644 index 0000000..4c1d7c3 --- /dev/null +++ b/tests/test_adaptation_statistics.py @@ -0,0 +1,55 @@ +"""Tests for paired Issue #265 statistical reporting and Holm correction.""" + +from __future__ import annotations + +import pytest + +from adaptive_rl.benchmarking.adaptation_statistics import ( + analyze_paired_recovery, + analyze_primary_cells, +) +from adaptive_rl.protocol.constants import PRIMARY_CELLS + + +def test_inconclusive_cell_suppresses_inference_below_minimum_valid_n() -> None: + result = analyze_paired_recovery( + fixed_t_h=[15.0] * 7 + [None, None, None], + adaptive_t_h=[10.0] * 7 + [None, None, None], + ) + assert result.valid_n == 7 + assert result.failed_n == 3 + assert result.status == "inconclusive" + assert result.primary_p_value is None + assert result.holm_adjusted_p_value is None + assert result.failure_imputation_bounds is not None + + +def test_evaluable_cell_contains_paired_analysis_and_sensitivity_outputs() -> None: + result = analyze_paired_recovery( + fixed_t_h=[15.0] * 10, + adaptive_t_h=[6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 15.0, 15.0], + ) + assert result.status == "evaluable" + assert result.valid_n == 10 + assert result.mean_difference is not None and result.mean_difference < 0.0 + assert result.primary_p_value is not None + assert result.interval_95 is not None + assert result.exact_sign_p_value is not None + assert result.exact_wilcoxon_p_value is not None + assert result.bootstrap_interval_95 is not None + + +def test_holm_adjustment_covers_all_preregistered_cells_and_invalid_is_null() -> None: + outcomes = {cell: ([15.0] * 10, [6.0] * 10) for cell in PRIMARY_CELLS} + outcomes[PRIMARY_CELLS[-1]] = ([15.0] * 7 + [None] * 3, [6.0] * 7 + [None] * 3) + results = analyze_primary_cells(outcomes) + assert tuple(results) == PRIMARY_CELLS + assert results[PRIMARY_CELLS[0]].holm_adjusted_p_value is not None + assert results[PRIMARY_CELLS[-1]].status == "inconclusive" + assert results[PRIMARY_CELLS[-1]].primary_p_value is None + assert results[PRIMARY_CELLS[-1]].holm_adjusted_p_value is None + + +def test_cell_mapping_must_match_the_frozen_family() -> None: + with pytest.raises(ValueError, match="PRIMARY_CELLS"): + analyze_primary_cells({}) diff --git a/tests/test_algorithm_adaptation.py b/tests/test_algorithm_adaptation.py new file mode 100644 index 0000000..1ae16c3 --- /dev/null +++ b/tests/test_algorithm_adaptation.py @@ -0,0 +1,177 @@ +"""Native PPO/SAC adaptation uses only recorded post-shift data.""" + +from __future__ import annotations + +import gymnasium as gym +import numpy as np +import torch +from gymnasium import spaces + +from adaptive_rl.algorithms.adaptation import ( + PPOAdaptationAdapter, + SACAdaptationAdapter, + run_adaptation_update, +) +from adaptive_rl.algorithms.ppo import PPOAlgorithm +from adaptive_rl.algorithms.sac import SACAlgorithm +from adaptive_rl.benchmarking.adaptation_runtime import evaluate_episode +from adaptive_rl.protocol.adaptation import PostShiftEpisode, Transition, build_update_batch +from adaptive_rl.protocol.fork import model_fingerprint +from adaptive_rl.protocol.seeds import derive_seed + + +class _OneStepEnv(gym.Env): + def __init__(self) -> None: + self.observation_space = spaces.Box(-1.0, 1.0, shape=(3,), dtype=np.float32) + self.action_space = spaces.Box(-1.0, 1.0, shape=(1,), dtype=np.float32) + + def reset(self, *, seed=None, options=None): + super().reset(seed=seed) + self.reset_seed = seed + return np.zeros(3, dtype=np.float32), {} + + def step(self, action): + assert self.action_space.contains(np.asarray(action, dtype=np.float32)) + return ( + np.zeros(3, dtype=np.float32), + 1.0, + False, + True, + {"success": True, "collision": False}, + ) + + +def _batch(algorithm, training_seed: int = 31001): + model = algorithm.model + assert model is not None + episodes = [] + for episode_index in range(1, 6): + transitions = [] + for step in range(3): + observation = np.full(model.observation_space.shape, step * 0.1, dtype=np.float32) + next_observation = observation + 0.05 + if isinstance(algorithm, PPOAlgorithm): + obs_tensor, _ = model.policy.obs_to_tensor(observation) + with torch.no_grad(): + action_tensor, value_tensor, log_prob_tensor = model.policy(obs_tensor) + next_obs_tensor, _ = model.policy.obs_to_tensor(next_observation) + next_value_tensor = model.policy.predict_values(next_obs_tensor) + action = action_tensor.cpu().numpy().reshape(-1) + behavior_value = float(value_tensor.reshape(-1)[0].cpu()) + behavior_log_prob = float(log_prob_tensor.reshape(-1)[0].cpu()) + behavior_next_value = float(next_value_tensor.reshape(-1)[0].cpu()) + else: + action = np.zeros(model.action_space.shape, dtype=np.float32) + behavior_value = None + behavior_log_prob = None + behavior_next_value = None + transitions.append( + Transition( + observation=observation, + action=action, + reward=float(episode_index + step), + next_observation=next_observation, + terminated=False, + truncated=step == 2, + behavior_log_prob=behavior_log_prob, + behavior_value=behavior_value, + behavior_next_value=behavior_next_value, + ) + ) + episodes.append( + PostShiftEpisode( + index=episode_index, + seed=derive_seed(training_seed, "post", episode_index), + transitions=tuple(transitions), + ) + ) + return build_update_batch(training_seed, episodes, block_episode=5) + + +def test_ppo_native_update_uses_recorded_rollout_without_environment_steps() -> None: + env = gym.make("Pendulum-v1") + try: + algorithm = PPOAlgorithm( + env=env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" + ) + batch = _batch(algorithm) + model = algorithm.model + assert model is not None + before = model_fingerprint(algorithm) + log = run_adaptation_update(algorithm, PPOAdaptationAdapter(), batch) + assert log.block_episode == 5 + assert log.transition_count == 15 + assert log.visible_episode_indices == (1, 2, 3, 4, 5) + assert log.update_seed == batch.seed + assert log.parameter_delta_l2 > 0.0 + assert log.fingerprint_before == before + assert log.fingerprint_after != before + assert model.num_timesteps == 0 + finally: + env.close() + + +def test_sac_native_update_uses_fresh_post_only_replay_buffer() -> None: + env = gym.make("Pendulum-v1") + try: + algorithm = SACAlgorithm( + env=env, + batch_size=4, + learning_starts=0, + gradient_steps=1, + buffer_size=100, + seed=31001, + device="cpu", + ) + batch = _batch(algorithm) + model = algorithm.model + assert model is not None + original_buffer = model.replay_buffer + before = model_fingerprint(algorithm) + log = run_adaptation_update(algorithm, SACAdaptationAdapter(), batch) + assert log.block_episode == 5 + assert log.transition_count == 15 + assert log.visible_episode_indices == (1, 2, 3, 4, 5) + assert log.update_seed == batch.seed + assert log.parameter_delta_l2 > 0.0 + assert log.fingerprint_before == before + assert log.fingerprint_after != before + assert model.replay_buffer is original_buffer + assert original_buffer is not None and original_buffer.size() == 0 + assert model.num_timesteps == 0 + finally: + env.close() + + +def test_episode_collector_records_transition_and_keeps_policy_constant() -> None: + model_env = gym.make("Pendulum-v1") + try: + algorithm = PPOAlgorithm( + env=model_env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" + ) + env = _OneStepEnv() + before = model_fingerprint(algorithm) + record = evaluate_episode( + algorithm=algorithm, + env=env, + training_seed=31001, + phase="post", + episode_index=1, + episode_seed=derive_seed(31001, "post", 1), + algorithm_name="ppo", + environment_name="fake_drone", + deterministic=True, + arm="shared", + ) + assert env.reset_seed == record.episode_seed + assert record.length == 1 + assert record.success is True + assert record.collision is False + assert len(record.transitions) == 1 + assert record.transitions[0].behavior_log_prob is not None + assert record.transitions[0].behavior_value is not None + assert record.transitions[0].behavior_next_value is not None + assert record.policy_fingerprint_start == record.policy_fingerprint_end == before + assert model_fingerprint(algorithm) == before + finally: + model_env.close() diff --git a/tests/test_protocol_adaptation_boundary.py b/tests/test_protocol_adaptation_boundary.py new file mode 100644 index 0000000..afe0b00 --- /dev/null +++ b/tests/test_protocol_adaptation_boundary.py @@ -0,0 +1,86 @@ +"""Leakage and ordering tests for the Issue #265 update boundary.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from adaptive_rl.protocol.adaptation import ( + PostShiftEpisode, + Transition, + build_update_batch, + call_update_atomically, + validate_block_sequence, +) +from adaptive_rl.protocol.seeds import derive_seed + + +def _episode(training_seed: int, index: int) -> PostShiftEpisode: + transition = Transition( + observation=np.asarray([index], dtype=np.float32), + action=np.asarray([index], dtype=np.float32), + reward=float(index), + next_observation=np.asarray([index + 0.5], dtype=np.float32), + terminated=False, + truncated=True, + ) + return PostShiftEpisode( + index=index, + seed=derive_seed(training_seed, "post", index), + transitions=(transition,), + ) + + +@pytest.mark.parametrize("boundary", range(5, 15)) +def test_each_update_sees_exactly_its_completed_post_prefix(boundary: int) -> None: + training_seed = 31001 + history = tuple(_episode(training_seed, index) for index in range(1, boundary + 1)) + batch = build_update_batch(training_seed, history, block_episode=boundary) + + assert batch.visible_episode_indices == tuple(range(1, boundary + 1)) + assert [int(t.observation[0]) for t in batch.transitions] == list(range(1, boundary + 1)) + assert batch.seed == derive_seed(training_seed, "update", boundary - 5) + + +def test_update_boundary_rejects_future_missing_or_misordered_data() -> None: + history = tuple(_episode(31001, index) for index in range(1, 7)) + with pytest.raises(ValueError, match="exactly episodes 1..5"): + build_update_batch(31001, history, block_episode=5) + with pytest.raises(ValueError, match="ordered completed episodes"): + build_update_batch( + 31001, (history[0], history[2], history[1], *history[3:]), block_episode=6 + ) + with pytest.raises(ValueError, match="only after"): + build_update_batch(31001, history, block_episode=15) + + +def test_block_schedule_has_no_b15() -> None: + validate_block_sequence(tuple(range(5, 15))) + with pytest.raises(ValueError, match="exactly"): + validate_block_sequence(tuple(range(5, 16))) + + +def test_transition_arrays_are_detached_and_read_only() -> None: + observation = np.asarray([1.0], dtype=np.float32) + transition = Transition(observation, observation, 1.0, observation, False, True) + observation[0] = 99.0 + assert transition.observation[0] == 1.0 + with pytest.raises(ValueError): + transition.observation[0] = 2.0 + + +def test_failed_update_rolls_back_adapter_target_state() -> None: + class Target: + def __init__(self) -> None: + self.weight = [1.0] + + class FailingAdapter: + def update(self, algorithm, batch) -> None: + del batch + algorithm.weight[0] = 2.0 + raise RuntimeError("invalid update") + + target = Target() + with pytest.raises(RuntimeError, match="invalid update"): + call_update_atomically(target, FailingAdapter(), object()) # type: ignore[arg-type] + assert target.weight == [1.0] diff --git a/tests/test_protocol_fork.py b/tests/test_protocol_fork.py new file mode 100644 index 0000000..c8f770a --- /dev/null +++ b/tests/test_protocol_fork.py @@ -0,0 +1,63 @@ +"""Regression coverage for paired policy fork isolation.""" + +from __future__ import annotations + +import pytest + +from adaptive_rl.protocol.fork import fork_adaptive_and_fixed, model_fingerprint + +torch = pytest.importorskip("torch") + + +class _Model(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.layer = torch.nn.Linear(3, 2) + + +class _Algorithm: + def __init__(self) -> None: + self.model = _Model() + self.optimizer = torch.optim.Adam(self.model.parameters()) + + def predict(self, observation, deterministic=True): + del deterministic + with torch.no_grad(): + return self.model(torch.as_tensor(observation, dtype=torch.float32)), None + + def train(self, *args, **kwargs): + del args, kwargs + raise AssertionError("fixed facade must not expose training") + + +def test_fork_is_equal_at_start_and_independently_mutable() -> None: + source = _Algorithm() + expected = model_fingerprint(source) + adaptive, fixed, frozen_fingerprint = fork_adaptive_and_fixed(source) + + assert expected == frozen_fingerprint + assert model_fingerprint(adaptive) == fixed.fingerprint == expected + assert adaptive.model.layer.weight.data_ptr() != source.model.layer.weight.data_ptr() + assert ( + adaptive.model.layer.weight.data_ptr() + != fixed._FrozenPolicy__algorithm.model.layer.weight.data_ptr() + ) + assert adaptive.optimizer is not source.optimizer + assert fixed._FrozenPolicy__algorithm.optimizer is not source.optimizer + + with torch.no_grad(): + adaptive.model.layer.weight.add_(1.0) + assert model_fingerprint(adaptive) != expected + assert fixed.fingerprint == expected + + with pytest.raises(AttributeError): + fixed.train(10) + + +def test_model_fingerprint_is_deterministic_and_sensitive_to_buffers() -> None: + model = _Model() + initial = model_fingerprint(model) + assert model_fingerprint(model) == initial + with torch.no_grad(): + model.layer.bias[0].add_(0.25) + assert model_fingerprint(model) != initial From a5250ffa5efaedaf76ad88c398f9087eb5ef49ca Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 05:10:02 +0530 Subject: [PATCH 2/7] feat(experiment): add immutable preregistered study runner --- docs/research/issue-271.md | 91 ++++++ src/adaptive_rl/algorithms/adaptation.py | 26 ++ .../benchmarking/adaptation_artifacts.py | 242 +++++++++++++++- .../benchmarking/adaptation_runner.py | 268 ++++++++++++++++-- .../benchmarking/adaptation_runtime.py | 5 +- .../benchmarking/adaptation_statistics.py | 20 +- src/adaptive_rl/cli.py | 16 +- tests/test_adaptation_artifacts.py | 71 ++++- tests/test_adaptation_invariants.py | 85 ++++++ tests/test_algorithm_adaptation.py | 2 + 10 files changed, 795 insertions(+), 31 deletions(-) create mode 100644 docs/research/issue-271.md create mode 100644 tests/test_adaptation_invariants.py diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md new file mode 100644 index 0000000..24415c2 --- /dev/null +++ b/docs/research/issue-271.md @@ -0,0 +1,91 @@ +# Issue #271: preregistered Adaptive vs Fixed study + +## Scope and preregistration + +This execution targets only `drone_disturbed/ppo` under TEST-B. The normative +protocol remains [`adaptive_rl_hypothesis.md`](adaptive_rl_hypothesis.md); this +document does not change its hypothesis, thresholds, alpha, horizon, seeds, or +analysis plan. The treatment is specified by +[`TREATMENT_CARD.md`](TREATMENT_CARD.md). + +## Before implementation: gap analysis + +| Requirement | Before | Evidence | +|---|---|---| +| Train once on nominal parameters; share pre-shift and shock; fork and adapt only between episodes | PARTIAL | `src/adaptive_rl/benchmarking/adaptation_runner.py` implemented the Issue #265 lifecycle, but had no Issue #271 invariant record | +| Exact ten-seed schedule and per-block provenance | PARTIAL | `src/adaptive_rl/protocol/seeds.py` froze the schedule; `--training-seeds` still allowed a subset | +| Recovery endpoint and registered tests/sensitivities | EXISTS | `src/adaptive_rl/protocol/recovery.py`, `src/adaptive_rl/protocol/statistics.py`, `src/adaptive_rl/benchmarking/adaptation_statistics.py` | +| Immutable JSON/CSV plus complete checksummed manifest and clean-tree execution gate | PARTIAL | `src/adaptive_rl/benchmarking/adaptation_artifacts.py` refused JSON/CSV overwrite, but there was no manifest or clean-tree gate | +| Required mutation, integrity, and tidy study CSV checks | PARTIAL | Existing Issue #265 tests covered smoke execution; no manifest validation or invariant mutation tests | +| Full real ten-replicate execution and report | MISSING | `docs/research/issue-265.md` stated that no full run had been collected | + +## Implementation + +The `adaptive-rl benchmark adaptation --study prereg-v1 --run-id RUN_ID` +entrypoint runs all ten training seeds in preregistered order, rejects subsets +and smoke mode, requires a clean committed tree, and writes into an immutable +run directory. A repeated run ID is refused. The JSON stores the raw trajectories, +protocol analysis, seed schedule, outcomes, runtime invariants, and run status. +The CSV has one row per replicate and arm, with finite-horizon `T_H`, status, +per-episode return vectors, and seed vectors. `manifest.json` checksums every +file in the run directory; `validate_study_manifest()` detects missing or +modified files. + +The fixed arm has a prediction-only interface. Exact equality of its initial, +per-episode, and final policy fingerprints establishes a zero weight delta; the +artifact records this as `fixed_parameter_delta_l2: 0.0` and +`fixed_weight_update_count: 0`. Adaptive block logs contain their update seed, +visible episode prefix, finite loss metrics, state fingerprints, and parameter +delta. The runner recomputes invariant checks before writing a successful +replicate. + +Right-censored recovery remains the preregistered finite endpoint `T_H = 15`. +Failed replicates are kept in the artifact and CSV; they are omitted pairwise +from the primary analysis and retained in both preregistered imputation bounds. +The statistical artifact includes standard error, t statistic, degrees of +freedom, one-sided p-value, 95% interval, Cohen's `d_z`, exact Wilcoxon and sign +tests, bootstrap interval, and failure-imputation bounds. + +## Ambiguity resolutions + +* The literal training seeds 31001–31010 identify the ten replicates. Each uses + the existing SHA-256-derived `pre`, `post`, and `update` phase seeds; the frozen + schedule fingerprint is recorded and rechecked. +* The shared shock window is the single execution of TEST-B post-shift episodes + 1–5, used by both arms before B5. The shared segment is represented once in + the raw artifact and referenced by identical per-arm return-vector hashes. +* B5 through B14 use the exact completed post-shift episode prefix required by + `protocol.adaptation.build_update_batch`. No block runs after episode 15. +* Censoring uses the preregistered finite-horizon value 15, not infinity. This + differs from the issue prompt's parenthetical `T_H = inf`; the preregistration + is the specified source of truth. + +## Dependency and repository status + +No local `ExperimentManifest` or roadmap issue 4/5 implementation was found in +the current checkout, so the study uses the minimal manifest described above. +The checked-out base was `fix/pr-259-review-hardening`, one commit ahead and +seven behind `origin/main`; implementation proceeds on +`feat/issue-271-preregistered-study`. GitHub issue/PR pages were unavailable to +the browsing environment, so the live status of PR #266 and roadmap issues 4/5 +could not be independently verified. No dependency on unmerged code is used. + +## Execution record + +Execution status: **PENDING**. Do not interpret smoke tests as study results. + +The intended single execution command is: + +```bash +.venv/bin/adaptive-rl benchmark adaptation \ + --config configs/drone_distribution_shift.yaml \ + --output-dir artifacts/issue271 \ + --study prereg-v1 \ + --run-id issue271-prereg-v1-20260929-01 +``` + +The runner enables Torch deterministic algorithms in warn-only mode and cuDNN +deterministic settings for the study, while recording that cross-hardware and +cross-library bitwise reproducibility is not claimed. Full run timing, host +details, commit SHA, status, artifact paths, and artifact digests will be added +after the execution attempt. diff --git a/src/adaptive_rl/algorithms/adaptation.py b/src/adaptive_rl/algorithms/adaptation.py index d32aeba..d414cb1 100644 --- a/src/adaptive_rl/algorithms/adaptation.py +++ b/src/adaptive_rl/algorithms/adaptation.py @@ -64,6 +64,9 @@ def _logger_ready(model: Any) -> Iterator[None]: class PPOAdaptationAdapter: """Use Stable-Baselines3 PPO's native clipped objective on stored rollouts.""" + def __init__(self) -> None: + self.last_loss_metrics: dict[str, float] = {} + def update(self, algorithm: Any, batch: UpdateBatch) -> None: model = _model(algorithm) if not hasattr(model, "rollout_buffer") or not hasattr(model, "n_epochs"): @@ -103,6 +106,7 @@ def update(self, algorithm: Any, batch: UpdateBatch) -> None: action = np.asarray(transition.action) reward = float(transition.reward) if transition.truncated and not transition.terminated: + assert transition.behavior_next_value is not None reward += model.gamma * float(transition.behavior_next_value) observations.append(observation) model.rollout_buffer.add( @@ -129,6 +133,7 @@ def update(self, algorithm: Any, batch: UpdateBatch) -> None: ) with _logger_ready(model), _seeded_update(batch.seed): model.train() + self.last_loss_metrics = _loss_metrics(model) model.policy.set_training_mode(False) @@ -140,6 +145,9 @@ def _previous_done(transitions: tuple[Any, ...], previous_index: int) -> bool: class SACAdaptationAdapter: """Train SAC from a fresh replay buffer containing only the visible batch.""" + def __init__(self) -> None: + self.last_loss_metrics: dict[str, float] = {} + def update(self, algorithm: Any, batch: UpdateBatch) -> None: model = _model(algorithm) if not hasattr(model, "critic_target") or not hasattr(model, "gradient_steps"): @@ -190,6 +198,7 @@ def update(self, algorithm: Any, batch: UpdateBatch) -> None: model.train( gradient_steps=int(model.gradient_steps), batch_size=int(model.batch_size) ) + self.last_loss_metrics = _loss_metrics(model) finally: model.replay_buffer = prior_buffer model.policy.set_training_mode(False) @@ -205,11 +214,27 @@ class AdaptationUpdateLog: fingerprint_after: str parameter_delta_l2: float status: str + loss_metrics: dict[str, float] def to_dict(self) -> dict[str, Any]: return asdict(self) +def _loss_metrics(model: Any) -> dict[str, float]: + logger = getattr(model, "logger", None) + recorded = getattr(logger, "name_to_value", {}) if logger is not None else {} + metrics = { + str(name): float(value) + for name, value in recorded.items() + if "loss" in str(name).lower() + and isinstance(value, (int, float, np.number)) + and np.isfinite(value) + } + if not metrics: + raise RuntimeError("adaptation update produced no finite loss metrics") + return metrics + + def _parameters(model_or_wrapper: Any) -> dict[str, torch.Tensor]: return { name: value.detach().cpu().clone() @@ -308,6 +333,7 @@ def run_adaptation_update( fingerprint_after=after_fingerprint, parameter_delta_l2=delta, status="updated" if delta > 0.0 else "no_parameter_change", + loss_metrics=dict(getattr(adapter, "last_loss_metrics", {})), ) diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py index 3fa3d03..60ecb09 100644 --- a/src/adaptive_rl/benchmarking/adaptation_artifacts.py +++ b/src/adaptive_rl/benchmarking/adaptation_artifacts.py @@ -4,8 +4,12 @@ import csv import dataclasses +import hashlib +import importlib.metadata import json import os +import platform +import subprocess import tempfile from pathlib import Path from typing import Any, Iterable, Mapping @@ -35,6 +39,23 @@ "parameter_delta_l2", ) +STUDY_CSV_FIELDS = ( + "training_seed", + "arm", + "replicate_status", + "recovery_status", + "T_H", + "pre_returns", + "shock_returns", + "post_returns", + "pre_seeds", + "shock_seeds", + "post_seeds", + "update_seeds", + "failure_reason", + "json_trajectory_reference", +) + def _plain(value: Any) -> Any: """Convert supported scientific data values into strict JSON primitives.""" @@ -52,6 +73,12 @@ def _plain(value: Any) -> Any: return {str(key): _plain(nested) for key, nested in value.items()} if isinstance(value, (list, tuple)): return [_plain(nested) for nested in value] + if isinstance(value, str) and value.startswith("/"): + candidate = Path(value) + try: + return candidate.resolve().relative_to(Path.cwd().resolve()).as_posix() + except (OSError, ValueError): + return candidate.name if value is None or isinstance(value, (str, int, float, bool)): return value raise TypeError(f"unsupported artifact value type: {type(value).__name__}") @@ -73,7 +100,7 @@ def _episode_rows(artifact: Mapping[str, Any]) -> Iterable[dict[str, Any]]: ) for key, phase, arm in segments: for episode in replicate.get(key, []): - row = {field: None for field in CSV_FIELDS} + row: dict[str, Any] = {field: None for field in CSV_FIELDS} row.update(common) row.update( arm=arm, @@ -144,10 +171,221 @@ def write_adaptation_artifacts( return json_path, csv_path +def write_adaptive_vs_fixed_artifacts( + artifact: Mapping[str, Any], output_dir: str | Path +) -> tuple[Path, Path]: + """Write study JSON plus the preregistered one-row-per-arm CSV.""" + target_dir = Path(output_dir) + target_dir.mkdir(parents=True, exist_ok=True) + json_path = target_dir / "adaptive_vs_fixed.json" + csv_path = target_dir / "adaptive_vs_fixed.csv" + plain = _plain(artifact) + temp_paths: list[Path] = [] + try: + for suffix, writer in ( + (".json", lambda handle: json.dump(plain, handle, indent=2, allow_nan=False)), + (".csv", lambda handle: _write_study_csv(handle, plain)), + ): + fd, temp_name = tempfile.mkstemp( + prefix=".adaptive-vs-fixed-", suffix=suffix, dir=target_dir + ) + temp_path = Path(temp_name) + temp_paths.append(temp_path) + with os.fdopen(fd, "w", newline="", encoding="utf-8") as handle: + writer(handle) + handle.flush() + os.fsync(handle.fileno()) + os.link(temp_paths[0], json_path) + try: + os.link(temp_paths[1], csv_path) + except BaseException: + json_path.unlink() + raise + finally: + for temp_path in temp_paths: + temp_path.unlink(missing_ok=True) + return json_path, csv_path + + +def _study_rows(artifact: Mapping[str, Any]) -> Iterable[dict[str, Any]]: + for replicate in artifact.get("replicates", []): + for arm in ("adaptive", "fixed"): + failed = replicate.get("status") != "completed" + recovery = replicate.get(f"{arm}_recovery") or {} + post = replicate.get("shared_shock_episodes", []) + replicate.get(f"{arm}_episodes", []) + seed_map = replicate.get("seeds", {}) + values = { + "training_seed": replicate.get("training_seed"), + "arm": arm, + "replicate_status": replicate.get("status"), + "recovery_status": None if failed else recovery.get("status"), + "T_H": None if failed else recovery.get("T_H"), + "pre_returns": _json_cell( + [item.get("reward") for item in replicate.get("shared_pre_shift_episodes", [])] + ), + "shock_returns": _json_cell( + [item.get("reward") for item in replicate.get("shared_shock_episodes", [])] + ), + "post_returns": _json_cell([item.get("reward") for item in post]), + "pre_seeds": _json_cell(seed_map.get("pre", [])), + "shock_seeds": _json_cell(seed_map.get("post", [])[:5]), + "post_seeds": _json_cell(seed_map.get("post", [])), + "update_seeds": _json_cell(seed_map.get("update", [])), + "failure_reason": replicate.get("failure_reason"), + "json_trajectory_reference": ( + f"replicates[training_seed={replicate.get('training_seed')}].{arm}_episodes" + ), + } + yield values + + +def _json_cell(value: Any) -> str: + return json.dumps(_plain(value), separators=(",", ":"), allow_nan=False) + + +def _write_study_csv(handle: Any, artifact: Mapping[str, Any]) -> None: + writer = csv.DictWriter(handle, fieldnames=STUDY_CSV_FIELDS, extrasaction="raise") + writer.writeheader() + writer.writerows(_study_rows(artifact)) + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def write_study_manifest( + artifact_path: str | Path, + csv_path: str | Path, + manifest_path: str | Path, + *, + run_id: str, + command: str, +) -> dict[str, Any]: + """Write an immutable provenance manifest for a completed study attempt.""" + artifact_path = Path(artifact_path) + csv_path = Path(csv_path) + manifest_path = Path(manifest_path) + if manifest_path.exists(): + raise FileExistsError(f"refusing to overwrite manifest: {manifest_path}") + try: + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() + dirty = bool( + subprocess.check_output( + ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL + ).strip() + ) + diff_hash = None + if dirty: + diff = subprocess.check_output(["git", "diff", "HEAD", "--binary"]) + diff_hash = hashlib.sha256(diff).hexdigest() + except (OSError, subprocess.CalledProcessError): + commit, dirty, diff_hash = None, None, None + manifest = { + "schema_version": "1.0", + "study": "adaptive-vs-fixed/prereg-v1", + "run_id": run_id, + "commit_sha": commit, + "working_tree_dirty": dirty, + "dirty_diff_sha256": diff_hash, + "execution_command": command, + "hardware": { + "python": platform.python_version(), + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor() or None, + "cpu_count": os.cpu_count(), + }, + "package_versions": { + name: _distribution_version(name) + for name in ("adaptive-rl", "gymnasium", "stable-baselines3", "torch", "numpy") + }, + "determinism": { + "pythonhashseed_env_recorded": False, + "torch_deterministic_algorithms": _torch_deterministic_algorithms(), + "torch_cudnn_deterministic": _torch_cudnn_deterministic(), + "protocol_seed_schedule": "SHA-256 derived seeds; see adaptive_vs_fixed.json", + }, + "artifacts": { + str(path.relative_to(manifest_path.parent).as_posix()): sha256_file(path) + for path in sorted(manifest_path.parent.rglob("*")) + if path.is_file() and path != manifest_path + }, + } + plain = _plain(manifest) + encoded = json.dumps(plain, indent=2, allow_nan=False) + "\n" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=".manifest-", dir=manifest_path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.link(temp_path, manifest_path) + finally: + temp_path.unlink(missing_ok=True) + return dict(plain) + + +def validate_study_manifest(manifest_path: str | Path) -> None: + """Raise when a listed immutable run artifact is missing or has changed.""" + manifest_path = Path(manifest_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, dict) or not artifacts: + raise ValueError("manifest must list at least one artifact checksum") + for relative_path, expected in artifacts.items(): + path = manifest_path.parent / relative_path + if not path.is_file(): + raise ValueError(f"manifest artifact is missing: {relative_path}") + actual = sha256_file(path) + if actual != expected: + raise ValueError(f"manifest checksum mismatch: {relative_path}") + + +def _distribution_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _torch_deterministic_algorithms() -> bool | None: + try: + import torch + + return bool(torch.are_deterministic_algorithms_enabled()) + except ImportError: + return None + + +def _torch_cudnn_deterministic() -> bool | None: + try: + import torch + + return bool(torch.backends.cudnn.deterministic) + except ImportError: + return None + + def _write_csv(handle: Any, artifact: Mapping[str, Any]) -> None: writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS, extrasaction="raise") writer.writeheader() writer.writerows(_episode_rows(artifact)) -__all__ = ["CSV_FIELDS", "write_adaptation_artifacts"] +__all__ = [ + "CSV_FIELDS", + "STUDY_CSV_FIELDS", + "sha256_file", + "validate_study_manifest", + "write_adaptation_artifacts", + "write_adaptive_vs_fixed_artifacts", + "write_study_manifest", +] diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py index 621db53..f737fbd 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -4,6 +4,7 @@ import hashlib import importlib.metadata +import json import logging import platform import subprocess @@ -15,18 +16,27 @@ import gymnasium as gym import numpy as np +import torch from adaptive_rl.algorithms.adaptation import ( PPOAdaptationAdapter, SACAdaptationAdapter, run_adaptation_update, ) -from adaptive_rl.benchmarking.adaptation_artifacts import write_adaptation_artifacts +from adaptive_rl.benchmarking.adaptation_artifacts import ( + write_adaptation_artifacts, + write_adaptive_vs_fixed_artifacts, + write_study_manifest, +) from adaptive_rl.benchmarking.adaptation_runtime import EpisodeRecord, evaluate_episode from adaptive_rl.benchmarking.adaptation_statistics import analyze_primary_cells from adaptive_rl.config import ExperimentConfig, compute_config_sha256 from adaptive_rl.environments.registry import make_env -from adaptive_rl.protocol.adaptation import build_update_batch, validate_block_sequence +from adaptive_rl.protocol.adaptation import ( + AdaptationAdapter, + build_update_batch, + validate_block_sequence, +) from adaptive_rl.protocol.constants import K_PRE, N_POST, PRIMARY_CELLS, TRAINING_SEEDS from adaptive_rl.protocol.fork import fork_adaptive_and_fixed, model_fingerprint from adaptive_rl.protocol.recovery import compute_recovery @@ -41,6 +51,8 @@ def _repository_metadata() -> dict[str, Any]: + commit: str | None + dirty: bool | None try: commit = subprocess.check_output( ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL @@ -52,7 +64,10 @@ def _repository_metadata() -> dict[str, Any]: ) except (OSError, subprocess.CalledProcessError): commit, dirty = None, None - versions = {"python": sys.version.split()[0], "platform": platform.platform()} + versions: dict[str, str | None] = { + "python": sys.version.split()[0], + "platform": platform.platform(), + } for distribution in ("adaptive-rl", "gymnasium", "stable-baselines3", "torch", "numpy"): try: versions[distribution] = importlib.metadata.version(distribution) @@ -83,13 +98,14 @@ def _new_env( parameters = dict(config.environment.parameters) parameters["max_steps"] = int(max_steps or config.environment.max_steps) if shifted: - if config.adaptation_benchmark is None: + benchmark = config.adaptation_benchmark + if benchmark is None: raise ValueError("configuration does not declare the Issue #265 adaptation cell") - parameters.update(config.adaptation_benchmark.shift_parameters) + parameters.update(benchmark.shift_parameters) return environment_factory(config.environment.name, **parameters) -def _adapter_for(algorithm_name: str): +def _adapter_for(algorithm_name: str) -> AdaptationAdapter: if algorithm_name == "ppo": return PPOAdaptationAdapter() if algorithm_name == "sac": @@ -113,6 +129,9 @@ def _train_once( training_dir.mkdir(parents=True) effective = config.model_copy(deep=True) + training = effective.training + if training is None: + raise ValueError("Issue #265 requires a training configuration") effective.seed = int(training_seed) effective.name = f"{config.name}_seed_{training_seed}" effective.output_dir = training_dir @@ -123,7 +142,7 @@ def _train_once( algorithm_parameters = dict(algorithm_config.parameters) algorithm_parameters["seed"] = int(training_seed) if smoke: - effective.training.total_timesteps = 32 + training.total_timesteps = 32 if algorithm_config.name.lower() == "ppo": algorithm_parameters.update({"n_steps": 16, "n_epochs": 1}) algorithm_config.batch_size = min(8, algorithm_config.batch_size) @@ -157,7 +176,7 @@ def _train_once( trainer.close() return algorithm, { "training_seed": training_seed, - "total_timesteps_requested": effective.training.total_timesteps, + "total_timesteps_requested": training.total_timesteps, "total_timesteps_completed": int(getattr(algorithm, "num_timesteps", 0)), "training_time_seconds": training_seconds, "model_path": str(result.final_model_path), @@ -216,6 +235,77 @@ def _recover(pre: Sequence[EpisodeRecord], post: Sequence[EpisodeRecord]) -> dic return output +def _return_vector_fingerprint(episodes: Sequence[dict[str, Any]]) -> str: + payload = json.dumps([float(item["reward"]) for item in episodes], separators=(",", ":")) + return hashlib.sha256(payload.encode("ascii")).hexdigest() + + +def _enable_study_determinism() -> dict[str, Any]: + """Enable deterministic Torch behavior where supported and report limits.""" + torch.use_deterministic_algorithms(True, warn_only=True) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + return { + "torch_deterministic_algorithms": torch.are_deterministic_algorithms_enabled(), + "torch_deterministic_warn_only": True, + "cudnn_deterministic": bool(torch.backends.cudnn.deterministic), + "cudnn_benchmark": bool(torch.backends.cudnn.benchmark), + "limitations": [ + "warn-only Torch operations may remain nondeterministic", + "cross-hardware and cross-library-version bitwise identity is not claimed", + "training and update RNGs are reseeded from the recorded protocol seeds", + ], + } + + +def _audit_replicate_invariants( + replicate: dict[str, Any], schedule: dict[int, dict[str, list[int]]] +) -> dict[str, Any]: + """Recompute the execution invariants from one serialized replicate record.""" + blocks = replicate["update_blocks"] + arms = replicate["arm_input_fingerprints"] + pre_hash = _return_vector_fingerprint(replicate["shared_pre_shift_episodes"]) + shock_hash = _return_vector_fingerprint(replicate["shared_shock_episodes"]) + checks = { + "pre_shift_shared_once": len(replicate["shared_pre_shift_episodes"]) == K_PRE, + "pre_shift_identical_across_arms": ( + arms["adaptive"]["pre"] == pre_hash == arms["fixed"]["pre"] + ), + "shock_shared_once": len(replicate["shared_shock_episodes"]) == 5, + "shock_returns_identical_across_arms": ( + arms["adaptive"]["shock"] == shock_hash == arms["fixed"]["shock"] + ), + "fixed_zero_updates": replicate["fixed_weight_update_count"] == 0, + "fixed_l2_delta_zero": replicate["fixed_parameter_delta_l2"] == 0.0, + "fixed_fingerprint_unchanged": ( + replicate["fixed_final_fingerprint"] == replicate["frozen_fingerprint"] + and all( + episode["policy_fingerprint_start"] == replicate["frozen_fingerprint"] + and episode["policy_fingerprint_end"] == replicate["frozen_fingerprint"] + for episode in replicate["fixed_episodes"] + ) + ), + "adaptive_exactly_ten_blocks": len(blocks) == 10, + "adaptive_block_schedule": [b["block_episode"] for b in blocks] == list(range(5, 15)), + "no_future_data": all( + list(b["visible_episode_indices"]) == list(range(1, b["block_episode"] + 1)) + for b in blocks + ), + "adaptive_update_seeds_match_schedule": [b["update_seed"] for b in blocks] + == replicate["seeds"]["update"], + "adaptive_blocks_have_loss_metrics": all(b["loss_metrics"] for b in blocks), + "adaptive_updates_between_episodes": [ + episode["update_block"] for episode in replicate["adaptive_episodes"] + ] + == list(range(5, 15)), + "fork_fingerprint_identical": replicate["fork_fingerprint"] + == replicate["frozen_fingerprint"], + "all_seeds_match_schedule": replicate["seeds"] + == {phase: list(values) for phase, values in schedule[replicate["training_seed"]].items()}, + } + return {"all_passed": all(checks.values()), **checks} + + @dataclass class ReplicateResult: training_seed: int @@ -223,6 +313,7 @@ class ReplicateResult: schedule_fingerprint: Optional[str] = None training_provenance: dict[str, Any] = field(default_factory=dict) frozen_fingerprint: Optional[str] = None + fork_fingerprint: Optional[str] = None fixed_final_fingerprint: Optional[str] = None pre_shift_performance: Optional[float] = None shock_performance: Optional[float] = None @@ -234,6 +325,9 @@ class ReplicateResult: adaptive_recovery: Optional[dict[str, Any]] = None fixed_recovery: Optional[dict[str, Any]] = None seeds: dict[str, list[int]] = field(default_factory=dict) + arm_input_fingerprints: dict[str, dict[str, str]] = field(default_factory=dict) + fixed_weight_update_count: int = 0 + fixed_parameter_delta_l2: float = 0.0 effective_nominal_parameters: dict[str, Any] = field(default_factory=dict) effective_shift_parameters: dict[str, Any] = field(default_factory=dict) failure_reason: Optional[str] = None @@ -254,6 +348,9 @@ def _run_replicate( ) -> ReplicateResult: result = ReplicateResult(training_seed=training_seed, status="failed") try: + benchmark = config.adaptation_benchmark + if benchmark is None: + raise ValueError("configuration lacks an Issue #265 adaptation cell") result.schedule_fingerprint = schedule_fingerprint(schedule) algorithm, training_provenance = _train_once( config, @@ -271,7 +368,7 @@ def _run_replicate( nominal_params = dict(config.environment.parameters) nominal_params["max_steps"] = 8 if smoke else config.environment.max_steps shifted_params = dict(nominal_params) - shifted_params.update(config.adaptation_benchmark.shift_parameters) + shifted_params.update(benchmark.shift_parameters) result.effective_nominal_parameters = dict(nominal_params) result.effective_shift_parameters = dict(shifted_params) result.seeds = {phase: list(values) for phase, values in schedule[training_seed].items()} @@ -314,7 +411,7 @@ def _run_replicate( try: get_effective = getattr(shock_env, "get_effective_parameters", None) effective = dict(get_effective()) if callable(get_effective) else shifted_params - for key, expected in config.adaptation_benchmark.shift_parameters.items(): + for key, expected in benchmark.shift_parameters.items(): if effective.get(key) != expected: raise RuntimeError( f"TEST-B parameter {key!r} did not apply: expected {expected!r}, " @@ -342,6 +439,7 @@ def _run_replicate( ) adaptive, fixed, fork_fingerprint = fork_adaptive_and_fixed(algorithm) + result.fork_fingerprint = fork_fingerprint if fork_fingerprint != frozen_fingerprint: raise RuntimeError("Adaptive/Fixed forks did not originate at the frozen fingerprint") adapter = _adapter_for(config.algorithm.name.lower()) @@ -406,6 +504,15 @@ def _run_replicate( fixed_post = result.shared_shock_episodes + result.fixed_episodes result.adaptive_recovery = _recover(result.shared_pre_shift_episodes, adaptive_post) result.fixed_recovery = _recover(result.shared_pre_shift_episodes, fixed_post) + pre_hash = _return_vector_fingerprint( + [{"reward": episode.reward} for episode in result.shared_pre_shift_episodes] + ) + shock_hash = _return_vector_fingerprint( + [{"reward": episode.reward} for episode in result.shared_shock_episodes] + ) + result.arm_input_fingerprints = { + arm: {"pre": pre_hash, "shock": shock_hash} for arm in ("adaptive", "fixed") + } result.status = "completed" except Exception as exc: result.failure_reason = f"{type(exc).__name__}: {exc}" @@ -422,6 +529,7 @@ def run_adaptation_benchmark( trainer_factory: TrainerFactory = get_trainer, environment_factory: EnvironmentFactory = make_env, config_path: str | Path | None = None, + study_run_id: str | None = None, ) -> dict[str, Any]: """Run the selected preregistered replicates and write JSON/CSV artifacts. @@ -431,6 +539,7 @@ def run_adaptation_benchmark( """ if config.adaptation_benchmark is None: raise ValueError("configuration must include the Issue #265 adaptation_benchmark section") + benchmark = config.adaptation_benchmark if config.algorithm.name.strip().lower() not in {"ppo", "sac"}: raise ValueError("Issue #265 supports only PPO and SAC") if config.training is None: @@ -447,6 +556,26 @@ def run_adaptation_benchmark( raise ValueError("all selected training seeds must come from TRAINING_SEEDS") if smoke and len(selected_seeds) != 1: raise ValueError("smoke mode runs exactly one preregistered training seed") + if study_run_id is not None: + if not study_run_id or Path(study_run_id).name != study_run_id: + raise ValueError("study_run_id must be a non-empty filename-safe component") + if smoke or selected_seeds != list(TRAINING_SEEDS): + raise ValueError("prereg-v1 requires one non-smoke attempt of all ten seeds in order") + if output_dir is not None and Path(output_dir).is_absolute(): + raise ValueError("prereg-v1 artifact output_dir must be relative to the repository") + try: + dirty = subprocess.check_output( + ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL + ).strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise RuntimeError( + "cannot verify clean working tree before prereg-v1 execution" + ) from exc + if dirty: + raise RuntimeError("prereg-v1 execution requires a clean, committed working tree") + determinism = _enable_study_determinism() + else: + determinism = None card = _card_path() if not card.is_file(): @@ -456,8 +585,12 @@ def run_adaptation_benchmark( config_file_sha = _sha256_file(config_file) if config_file is not None else None target_dir = Path(output_dir) if output_dir is not None else Path(config.output_dir) + if study_run_id is not None: + target_dir = target_dir / study_run_id target_dir.mkdir(parents=True, exist_ok=True) - for suffix in ("adaptation.json", "adaptation.csv"): + stem = "adaptive_vs_fixed" if study_run_id is not None else "adaptation" + suffixes = (f"{stem}.json", f"{stem}.csv") + for suffix in (*suffixes, "manifest.json"): if (target_dir / suffix).exists(): raise FileExistsError(f"refusing to overwrite existing artifact: {target_dir / suffix}") training_root = target_dir / "training" @@ -489,26 +622,71 @@ def run_adaptation_benchmark( if replicate.status != "completed": continue position = TRAINING_SEEDS.index(replicate.training_seed) + assert replicate.fixed_recovery is not None + assert replicate.adaptive_recovery is not None fixed_vector[position] = float(replicate.fixed_recovery["truncated_recovery_time"]) adaptive_vector[position] = float(replicate.adaptive_recovery["truncated_recovery_time"]) - paired = analyze_primary_cells(vectors) + analysis_vectors: dict[str, tuple[Sequence[Optional[float]], Sequence[Optional[float]]]] = { + cell: (fixed, adaptive) for cell, (fixed, adaptive) in vectors.items() + } + paired = analyze_primary_cells(analysis_vectors) family_decision = decide_family( {cell: analysis.primary_p_value for cell, analysis in paired.items()} ) provenance = _repository_metadata() - artifact = { + run_status = "PARTIAL" if any(rep.status != "completed" for rep in results) else "COMPLETE" + + def outcome_count(arm: str, status: str) -> int: + count = 0 + for replicate_result in results: + if replicate_result.status != "completed": + continue + recovery = ( + replicate_result.adaptive_recovery + if arm == "adaptive" + else replicate_result.fixed_recovery + ) + if recovery is not None and recovery.get("status") == status: + count += 1 + return count + + outcome_summary = { + arm: { + **{ + status: outcome_count(arm, status) + for status in ( + "recovered", + "right_censored", + "no_degradation", + "degradation_below_resolution", + ) + }, + "failed_replicates": sum(rep.status != "completed" for rep in results), + } + for arm in ("adaptive", "fixed") + } + artifact: dict[str, Any] = { "schema_version": "1.0", - "protocol_version": config.adaptation_benchmark.protocol_version, - "issue": "265", - "run_type": "smoke" if smoke else "full_or_selected_research_run", + "run_id": study_run_id, + "run_status": run_status if study_run_id is not None else None, + "protocol_version": benchmark.protocol_version, + "issue": "271" if study_run_id is not None else "265", + "run_type": ( + "smoke" + if smoke + else "prereg-v1" + if study_run_id is not None + else "full_or_selected_research_run" + ), + "determinism": determinism, "treatment_card_sha256": card_sha, "schedule_fingerprint": schedule_fp, "experiment": { "name": config.name, "algorithm": config.algorithm.name.strip().lower(), "environment": config.environment.name, - "scenario": config.adaptation_benchmark.scenario, + "scenario": benchmark.scenario, "planned_replicates": len(TRAINING_SEEDS), "selected_training_seeds": selected_seeds, "config_sha256": compute_config_sha256(config), @@ -526,6 +704,7 @@ def run_adaptation_benchmark( "completed_replicates": sum(rep.status == "completed" for rep in results), "valid_pairs": {cell: result.valid_n for cell, result in paired.items()}, }, + "outcome_summary": outcome_summary, "provenance": { **provenance, "treatment_card_path": str(card), @@ -534,8 +713,59 @@ def run_adaptation_benchmark( }, "scientific_claim": "Harness execution alone does not establish empirical superiority.", } - json_path, csv_path = write_adaptation_artifacts(artifact, target_dir) - artifact["artifact_paths"] = {"json": str(json_path), "csv": str(csv_path)} + for replicate in artifact["replicates"]: + if replicate["status"] != "completed": + replicate["invariants"] = {"all_passed": False, "failed_replicate": True} + continue + replicate["invariants"] = _audit_replicate_invariants(replicate, schedule) + if not replicate["invariants"]["all_passed"]: + raise RuntimeError( + f"runtime invariant failed for seed {replicate['training_seed']}: " + f"{replicate['invariants']}" + ) + artifact["run_status"] = ( + "PARTIAL" + if study_run_id is not None + and any(rep["status"] != "completed" for rep in artifact["replicates"]) + else run_status + ) + if study_run_id is not None: + artifact["artifact_paths"] = { + "json": f"{stem}.json", + "csv": f"{stem}.csv", + "manifest": "manifest.json", + } + if study_run_id is not None: + json_path, csv_path = write_adaptive_vs_fixed_artifacts(artifact, target_dir) + else: + json_path, csv_path = write_adaptation_artifacts(artifact, target_dir, stem=stem) + artifact["artifact_paths"] = { + "json": str(json_path), + "csv": str(csv_path), + **({"manifest": str(target_dir / "manifest.json")} if study_run_id else {}), + } + if study_run_id is not None: + config_arg = ( + Path(config_path).as_posix() + if config_path is not None + else "configs/drone_distribution_shift.yaml" + ) + if Path(config_arg).is_absolute(): + raise ValueError("prereg-v1 config path must be repository-relative") + output_arg = f"--output-dir {Path(output_dir)} " if output_dir is not None else "" + command = ( + "adaptive-rl benchmark adaptation " + f"--config {config_arg} {output_arg}" + f"--study prereg-v1 --run-id {study_run_id}" + ) + manifest = write_study_manifest( + json_path, + csv_path, + target_dir / "manifest.json", + run_id=study_run_id, + command=command, + ) + artifact["manifest"] = manifest return artifact diff --git a/src/adaptive_rl/benchmarking/adaptation_runtime.py b/src/adaptive_rl/benchmarking/adaptation_runtime.py index 92be7b1..21c768e 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runtime.py +++ b/src/adaptive_rl/benchmarking/adaptation_runtime.py @@ -44,7 +44,9 @@ def _seed_episode(seed: int) -> Iterator[None]: torch.cuda.set_rng_state_all(cuda_states) -def _action_and_behavior(algorithm: Any, observation: Any, deterministic: bool): +def _action_and_behavior( + algorithm: Any, observation: Any, deterministic: bool +) -> tuple[np.ndarray, np.ndarray, float | None, float | None]: model = getattr(algorithm, "model", None) policy = getattr(model, "policy", None) if model is not None else None if policy is None: @@ -52,6 +54,7 @@ def _action_and_behavior(algorithm: Any, observation: Any, deterministic: bool): environment_action = np.asarray(environment_action) return environment_action, environment_action, None, None if hasattr(model, "rollout_buffer"): + assert model is not None obs_tensor, _ = policy.obs_to_tensor(observation) with torch.no_grad(): native_action, value, log_prob = policy(obs_tensor, deterministic=deterministic) diff --git a/src/adaptive_rl/benchmarking/adaptation_statistics.py b/src/adaptive_rl/benchmarking/adaptation_statistics.py index 3ececd3..d8ab715 100644 --- a/src/adaptive_rl/benchmarking/adaptation_statistics.py +++ b/src/adaptive_rl/benchmarking/adaptation_statistics.py @@ -28,6 +28,9 @@ class PairedRecoveryAnalysis: differences: list[Optional[float]] mean_difference: Optional[float] standard_deviation: Optional[float] + standard_error: Optional[float] + t_statistic: Optional[float] + degrees_of_freedom: Optional[int] primary_p_value: Optional[float] holm_adjusted_p_value: Optional[float] interval_95: Optional[list[float]] @@ -78,6 +81,9 @@ def analyze_paired_recovery( differences=differences, mean_difference=None, standard_deviation=None, + standard_error=None, + t_statistic=None, + degrees_of_freedom=None, primary_p_value=None, holm_adjusted_p_value=None, interval_95=None, @@ -98,6 +104,9 @@ def analyze_paired_recovery( differences=differences, mean_difference=primary.mean, standard_deviation=primary.std_dev, + standard_error=primary.standard_error, + t_statistic=primary.t_statistic, + degrees_of_freedom=primary.degrees_of_freedom, primary_p_value=primary.p_value, holm_adjusted_p_value=None, interval_95=list(paired_t_interval(paired, min_valid_n=min_valid_n)), @@ -122,12 +131,11 @@ def analyze_primary_cells( if set(outcomes) != set(PRIMARY_CELLS): raise ValueError("outcomes must contain exactly the preregistered PRIMARY_CELLS") results = {cell: analyze_paired_recovery(*outcomes[cell]) for cell in PRIMARY_CELLS} - adjusted = holm_adjust( - [ - results[cell].primary_p_value if results[cell].primary_p_value is not None else 1.0 - for cell in PRIMARY_CELLS - ] - ) + p_values: list[float] = [] + for cell in PRIMARY_CELLS: + p_value = results[cell].primary_p_value + p_values.append(p_value if p_value is not None else 1.0) + adjusted = holm_adjust(p_values) return { cell: replace(results[cell], holm_adjusted_p_value=adjusted[index]) if results[cell].primary_p_value is not None diff --git a/src/adaptive_rl/cli.py b/src/adaptive_rl/cli.py index 0ce5a40..cfc237f 100644 --- a/src/adaptive_rl/cli.py +++ b/src/adaptive_rl/cli.py @@ -400,6 +400,12 @@ def benchmark_adaptation( output_dir: Optional[Path] = typer.Option( None, "--output-dir", help="Directory for Issue #265 JSON/CSV and training artifacts" ), + study: Optional[str] = typer.Option( + None, "--study", help="Run the immutable full protocol study (currently prereg-v1)" + ), + run_id: Optional[str] = typer.Option( + None, "--run-id", help="Unique immutable output directory name required with --study" + ), deterministic: Optional[bool] = typer.Option( None, "--deterministic/--stochastic", help="Override action selection for all evaluations" ), @@ -455,12 +461,18 @@ def benchmark_adaptation( from adaptive_rl.benchmarking.adaptation_runner import run_adaptation_benchmark + if study not in {None, "prereg-v1"}: + raise ValueError("--study currently supports only prereg-v1") + if (study is None) != (run_id is None): + raise ValueError("--study and --run-id must be supplied together") + artifact = run_adaptation_benchmark( exp_config, output_dir=output_dir, training_seeds=selected_seeds, smoke=smoke, config_path=config, + study_run_id=run_id if study is not None else None, ) except Exception as err: console.print(f"[bold red]Issue #265 benchmark failed:[/bold red] {err}") @@ -475,8 +487,8 @@ def benchmark_adaptation( f"• [bold]Algorithm:[/bold] {artifact['experiment']['algorithm']}\n" f"• [bold]Completed replicates:[/bold] {completed}\n" f"• [bold]Failed replicates:[/bold] {len(failed)}\n" - f"• [bold]JSON:[/bold] {Path(output_dir or exp_config.output_dir) / 'adaptation.json'}\n" - f"• [bold]CSV:[/bold] {Path(output_dir or exp_config.output_dir) / 'adaptation.csv'}\n" + f"• [bold]JSON:[/bold] {Path(output_dir or exp_config.output_dir) / ('adaptive_vs_fixed.json' if study else 'adaptation.json')}\n" + f"• [bold]CSV:[/bold] {Path(output_dir or exp_config.output_dir) / ('adaptive_vs_fixed.csv' if study else 'adaptation.csv')}\n" f"• [bold]Scientific result:[/bold] not established by harness execution", title="Online Adaptation Benchmark", border_style="yellow" if smoke or failed else "green", diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py index a261366..85e15d6 100644 --- a/tests/test_adaptation_artifacts.py +++ b/tests/test_adaptation_artifacts.py @@ -8,7 +8,13 @@ import numpy as np import pytest -from adaptive_rl.benchmarking.adaptation_artifacts import write_adaptation_artifacts +from adaptive_rl.benchmarking.adaptation_artifacts import ( + sha256_file, + validate_study_manifest, + write_adaptation_artifacts, + write_adaptive_vs_fixed_artifacts, + write_study_manifest, +) def _artifact(): @@ -70,3 +76,66 @@ def test_nonfinite_values_are_rejected_for_strict_json(tmp_path) -> None: data["value"] = float("nan") with pytest.raises(ValueError, match="JSON compliant"): write_adaptation_artifacts(data, tmp_path) + + +def test_manifest_hashes_all_artifacts_and_detects_tampering(tmp_path) -> None: + json_path, csv_path = write_adaptation_artifacts( + _artifact(), tmp_path, stem="adaptive_vs_fixed" + ) + training_artifact = tmp_path / "training" / "seed_31001" / "weights.zip" + training_artifact.parent.mkdir(parents=True) + training_artifact.write_bytes(b"real-artifact-bytes") + manifest_path = tmp_path / "manifest.json" + manifest = write_study_manifest( + json_path, + csv_path, + manifest_path, + run_id="test-run", + command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id test-run", + ) + assert manifest["artifacts"][json_path.name] == sha256_file(json_path) + assert manifest["artifacts"][csv_path.name] == sha256_file(csv_path) + assert manifest["artifacts"]["training/seed_31001/weights.zip"] == sha256_file( + training_artifact + ) + validate_study_manifest(manifest_path) + with pytest.raises(FileExistsError): + write_study_manifest( + json_path, + csv_path, + manifest_path, + run_id="test-run", + command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id test-run", + ) + training_artifact.write_bytes(b"tampered") + assert manifest["artifacts"]["training/seed_31001/weights.zip"] != sha256_file( + training_artifact + ) + with pytest.raises(ValueError, match="checksum mismatch"): + validate_study_manifest(manifest_path) + + +def test_study_csv_has_one_row_per_arm_and_preserves_finite_censoring(tmp_path) -> None: + data = { + "replicates": [ + { + "training_seed": 31001, + "status": "completed", + "shared_pre_shift_episodes": [{"reward": 10.0}], + "shared_shock_episodes": [{"reward": 2.0}], + "adaptive_episodes": [{"reward": 3.0}], + "fixed_episodes": [{"reward": 2.5}], + "adaptive_recovery": {"status": "right_censored", "T_H": 15}, + "fixed_recovery": {"status": "right_censored", "T_H": 15}, + "seeds": {"pre": [11], "post": [12, 13], "update": [14]}, + } + ] + } + _, csv_path = write_adaptive_vs_fixed_artifacts(data, tmp_path) + with csv_path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + assert len(rows) == 2 + assert {row["arm"] for row in rows} == {"adaptive", "fixed"} + assert [row["T_H"] for row in rows] == ["15", "15"] + assert [row["recovery_status"] for row in rows] == ["right_censored"] * 2 + assert json.loads(rows[0]["post_returns"]) == [2.0, 3.0] diff --git a/tests/test_adaptation_invariants.py b/tests/test_adaptation_invariants.py new file mode 100644 index 0000000..483ad2a --- /dev/null +++ b/tests/test_adaptation_invariants.py @@ -0,0 +1,85 @@ +"""Mutation checks for prereg-v1 runtime invariant auditing.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from adaptive_rl.benchmarking.adaptation_runner import ( + _audit_replicate_invariants, + _return_vector_fingerprint, +) +from adaptive_rl.protocol.constants import TRAINING_SEEDS +from adaptive_rl.protocol.seeds import frozen_schedule + + +@pytest.fixture +def valid_replicate(): + schedule = frozen_schedule() + seed = TRAINING_SEEDS[0] + fingerprint = "frozen" + pre = [{"reward": 2.0} for _ in range(15)] + shock = [{"reward": 0.0} for _ in range(5)] + pre_hash = _return_vector_fingerprint(pre) + shock_hash = _return_vector_fingerprint(shock) + replicate = { + "training_seed": seed, + "shared_pre_shift_episodes": pre, + "shared_shock_episodes": shock, + "adaptive_episodes": [{"update_block": block} for block in range(5, 15)], + "fixed_episodes": [ + {"policy_fingerprint_start": fingerprint, "policy_fingerprint_end": fingerprint} + for _ in range(10) + ], + "update_blocks": [ + { + "block_episode": block, + "visible_episode_indices": list(range(1, block + 1)), + "update_seed": schedule[seed]["update"][block - 5], + "loss_metrics": {"train/loss": 1.0}, + } + for block in range(5, 15) + ], + "seeds": schedule[seed], + "arm_input_fingerprints": { + arm: {"pre": pre_hash, "shock": shock_hash} for arm in ("adaptive", "fixed") + }, + "fixed_weight_update_count": 0, + "fixed_parameter_delta_l2": 0.0, + "fixed_final_fingerprint": fingerprint, + "frozen_fingerprint": fingerprint, + "fork_fingerprint": fingerprint, + } + return replicate, schedule + + +@pytest.mark.parametrize( + ("mutation", "failed_check"), + [ + ("pre", "pre_shift_identical_across_arms"), + ("shock", "shock_returns_identical_across_arms"), + ("fixed", "fixed_zero_updates"), + ("leak", "no_future_data"), + ("fork", "fork_fingerprint_identical"), + ], +) +def test_each_preregistered_invariant_violation_fails_audit( + valid_replicate, mutation, failed_check +): + replicate, schedule = valid_replicate + replicate = deepcopy(replicate) + if mutation == "pre": + replicate["arm_input_fingerprints"]["fixed"]["pre"] = "different" + elif mutation == "shock": + replicate["arm_input_fingerprints"]["adaptive"]["shock"] = "different" + elif mutation == "fixed": + replicate["fixed_weight_update_count"] = 1 + elif mutation == "leak": + replicate["update_blocks"][0]["visible_episode_indices"].append(6) + else: + replicate["fork_fingerprint"] = "different" + + audit = _audit_replicate_invariants(replicate, schedule) + assert audit[failed_check] is False + assert audit["all_passed"] is False diff --git a/tests/test_algorithm_adaptation.py b/tests/test_algorithm_adaptation.py index 1ae16c3..1369af5 100644 --- a/tests/test_algorithm_adaptation.py +++ b/tests/test_algorithm_adaptation.py @@ -99,6 +99,7 @@ def test_ppo_native_update_uses_recorded_rollout_without_environment_steps() -> assert model is not None before = model_fingerprint(algorithm) log = run_adaptation_update(algorithm, PPOAdaptationAdapter(), batch) + assert log.loss_metrics assert log.block_episode == 5 assert log.transition_count == 15 assert log.visible_episode_indices == (1, 2, 3, 4, 5) @@ -129,6 +130,7 @@ def test_sac_native_update_uses_fresh_post_only_replay_buffer() -> None: original_buffer = model.replay_buffer before = model_fingerprint(algorithm) log = run_adaptation_update(algorithm, SACAdaptationAdapter(), batch) + assert log.loss_metrics assert log.block_episode == 5 assert log.transition_count == 15 assert log.visible_episode_indices == (1, 2, 3, 4, 5) From 3fbde2c8d8e106c0a939630c209ba4e399b04ece Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 05:38:54 +0530 Subject: [PATCH 3/7] feat(experiment): safely resume hashed study replicates --- docs/research/issue-271.md | 73 +++++++++++++++-- .../benchmarking/adaptation_artifacts.py | 61 +++++++++++++- .../benchmarking/adaptation_runner.py | 82 +++++++++++++++++-- src/adaptive_rl/cli.py | 12 ++- tests/test_adaptation_artifacts.py | 21 +++++ tests/test_adaptation_invariants.py | 14 ++++ tests/test_adaptation_smoke.py | 20 +++++ 7 files changed, 265 insertions(+), 18 deletions(-) diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md index 24415c2..f56a1cb 100644 --- a/docs/research/issue-271.md +++ b/docs/research/issue-271.md @@ -24,8 +24,12 @@ analysis plan. The treatment is specified by The `adaptive-rl benchmark adaptation --study prereg-v1 --run-id RUN_ID` entrypoint runs all ten training seeds in preregistered order, rejects subsets and smoke mode, requires a clean committed tree, and writes into an immutable -run directory. A repeated run ID is refused. The JSON stores the raw trajectories, -protocol analysis, seed schedule, outcomes, runtime invariants, and run status. +run directory. A repeated completed run ID is refused. `--resume` accepts only +terminal per-replicate records whose SHA-256 sidecar verifies; it never trusts +partial training directories. An interrupted seed without a complete checkpoint +is recorded as failed, and remaining unstarted seeds continue. The JSON stores +the raw trajectories, protocol analysis, seed schedule, outcomes, runtime +invariants, and run status. The CSV has one row per replicate and arm, with finite-horizon `T_H`, status, per-episode return vectors, and seed vectors. `manifest.json` checksums every file in the run directory; `validate_study_manifest()` detects missing or @@ -72,7 +76,9 @@ could not be independently verified. No dependency on unmerged code is used. ## Execution record -Execution status: **PENDING**. Do not interpret smoke tests as study results. +Execution status: **COMPLETE** for this one-cell, ten-replicate study. The +preregistered six-cell family remains **INCONCLUSIVE** because five cells are +not executable in this checkout. These data do not support H1. The intended single execution command is: @@ -86,6 +92,61 @@ The intended single execution command is: The runner enables Torch deterministic algorithms in warn-only mode and cuDNN deterministic settings for the study, while recording that cross-hardware and -cross-library bitwise reproducibility is not claimed. Full run timing, host -details, commit SHA, status, artifact paths, and artifact digests will be added -after the execution attempt. +cross-library bitwise reproducibility is not claimed. + +* Run ID: `issue271-prereg-v1-20260929-01` +* Run status: `COMPLETE`; 10 completed, 0 failed, ordered seeds 31001–31010. +* Executed commit: `a5250ffa5efaedaf76ad88c398f9087eb5ef49ca`; clean tree. +* Wall time: approximately 17 minutes 50 seconds (run-directory creation to + manifest creation); summed PPO training time was 939.38 seconds. +* Hardware/runtime: Linux x86_64, 4 logical CPUs, Python 3.14.7, gymnasium + 1.3.0, stable-baselines3 2.9.0, Torch 2.14.0, NumPy 2.5.3. The platform + reported no processor model. +* Artifacts: `artifacts/issue271/issue271-prereg-v1-20260929-01/` + * `adaptive_vs_fixed.json` SHA-256: + `e7b681d875e8fdfdcc56acada44a34062ac05c92e8881b16c8a3354633b79265` + * `adaptive_vs_fixed.csv` SHA-256: + `4d2879f08cf6caa67585576bc67c2cf0d699b617ee8260849c10b7ae7e7ce098` + * `manifest.json` SHA-256: + `bb5c3e6f5150a329aecca114dfd47f50366d99ddb1c3e84c515f6fa7d5d480ec` +* Config SHA-256: `0039c298b5048254b2d211cc66967e9d3e1d71275575fcfe28736485c65937b5`. + Treatment Card SHA-256: `8383e736f02ff31393474b241b06d2dc92b036910c93c26081c1673c92a317e3`. +* Independent audit verified the exact seed list and frozen schedule fingerprint, + ten passing invariant records, all 100 B5–B14 blocks, 20 tidy CSV rows, + all manifest-listed checksums, and no `/home/aryan` path in the JSON. + +The run predates the follow-up safe-resume addition; it executed from the clean +source commit recorded by the manifest and was not resumed. The artifact's +`execution_command` is normalized to `adaptive-rl` because the entrypoint +records the command's CLI form; the exact shell invocation above includes the +virtual-environment path used to select the installed executable. + +## Result from the artifact + +For every seed, both arms had `T_H = 0`: 4 replicates were `no_degradation` and +6 were `degradation_below_resolution` in each arm. There were no recovered or +right-censored replicates and no failures. Thus the censoring count was zero; +the finite right-censor endpoint remains `T_H = 15` by protocol. + +The paired differences were ten zeros. The artifact reports `N_valid = 10`, +mean difference 0, standard error 0, one-sided paired t statistic 0 on 9 df, +primary p = 0.5, 95% t interval [0, 0], Cohen's `d_z = 0`, exact sign p = 1, +exact Wilcoxon p = 1, and bootstrap interval [0, 0]. The cell is evaluable but +not significant. The six-cell family decision is `INCONCLUSIVE`; no family +claim is made. + +## Final acceptance status + +| Acceptance area | Status | Evidence | +|---|---|---| +| Ten preregistered PPO replicates and seeds | PASS | JSON seed list, schedule fingerprint, 10 completed records | +| Shared segments, identical fork, fixed lock, ten causal update blocks | PASS | Every replicate's `invariants.all_passed`; mutation coverage in `tests/test_adaptation_invariants.py` | +| Recovery and preregistered paired analyses | PASS | Per-replicate recovery plus `paired_analysis` in JSON | +| Immutable JSON/CSV and validated checksums | PASS | 20-row CSV; `manifest.json`; `validate_study_manifest()` | +| Safe continuation after interruption | PASS in follow-up code | Digest-verified terminal replicate checkpoints; incomplete directories become recorded failures | +| Statistical superiority of Adaptive | FAIL | All observed paired differences are zero; p = 0.5 | +| Six-cell family claim / power ≥ 0.80 | NOT MET | Five cells unavailable; preregistration says power is unquantified at N = 10 | + +PR #266 and roadmap issues 4/5 remain unverified live because GitHub was not +available. The run does not depend on them. This is a one-cell study, not +evidence for the broader multi-environment claim. diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py index 60ecb09..ae3af2a 100644 --- a/src/adaptive_rl/benchmarking/adaptation_artifacts.py +++ b/src/adaptive_rl/benchmarking/adaptation_artifacts.py @@ -12,7 +12,7 @@ import subprocess import tempfile from pathlib import Path -from typing import Any, Iterable, Mapping +from typing import Any, Iterable, Mapping, cast import numpy as np @@ -349,6 +349,63 @@ def validate_study_manifest(manifest_path: str | Path) -> None: raise ValueError(f"manifest checksum mismatch: {relative_path}") +def write_replicate_checkpoint(replicate: Mapping[str, Any], checkpoint_path: str | Path) -> None: + """Persist one terminal replicate result and a digest without overwriting.""" + checkpoint_path = Path(checkpoint_path) + if replicate.get("status") not in {"completed", "failed"}: + raise ValueError("only terminal replicate states may be checkpointed") + plain = _plain(dict(replicate)) + encoded = (json.dumps(plain, indent=2, allow_nan=False) + "\n").encode("utf-8") + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=".replicate-", dir=checkpoint_path.parent) + temp_path = Path(temp_name) + digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") + installed_checkpoint = False + try: + with os.fdopen(fd, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.link(temp_path, checkpoint_path) + installed_checkpoint = True + digest = hashlib.sha256(encoded).hexdigest() + digest_temp_fd, digest_temp_name = tempfile.mkstemp( + prefix=".replicate-digest-", dir=checkpoint_path.parent + ) + digest_temp_path = Path(digest_temp_name) + try: + with os.fdopen(digest_temp_fd, "w", encoding="ascii") as handle: + handle.write(f"{digest}\n") + handle.flush() + os.fsync(handle.fileno()) + os.link(digest_temp_path, digest_path) + finally: + digest_temp_path.unlink(missing_ok=True) + except BaseException: + if installed_checkpoint: + checkpoint_path.unlink(missing_ok=True) + raise + finally: + temp_path.unlink(missing_ok=True) + + +def read_replicate_checkpoint(checkpoint_path: str | Path) -> dict[str, Any]: + """Read only a terminal checkpoint whose adjacent digest verifies.""" + checkpoint_path = Path(checkpoint_path) + digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") + payload = checkpoint_path.read_bytes() + if not digest_path.is_file(): + raise ValueError(f"replicate checkpoint digest is missing: {checkpoint_path.name}") + expected = digest_path.read_text(encoding="ascii").strip() + actual = hashlib.sha256(payload).hexdigest() + if actual != expected: + raise ValueError(f"replicate checkpoint checksum mismatch: {checkpoint_path.name}") + result = json.loads(payload) + if result.get("status") not in {"completed", "failed"}: + raise ValueError("replicate checkpoint is not terminal") + return cast(dict[str, Any], result) + + def _distribution_version(name: str) -> str | None: try: return importlib.metadata.version(name) @@ -384,8 +441,10 @@ def _write_csv(handle: Any, artifact: Mapping[str, Any]) -> None: "CSV_FIELDS", "STUDY_CSV_FIELDS", "sha256_file", + "read_replicate_checkpoint", "validate_study_manifest", "write_adaptation_artifacts", "write_adaptive_vs_fixed_artifacts", + "write_replicate_checkpoint", "write_study_manifest", ] diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py index f737fbd..05b088e 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -24,8 +24,10 @@ run_adaptation_update, ) from adaptive_rl.benchmarking.adaptation_artifacts import ( + read_replicate_checkpoint, write_adaptation_artifacts, write_adaptive_vs_fixed_artifacts, + write_replicate_checkpoint, write_study_manifest, ) from adaptive_rl.benchmarking.adaptation_runtime import EpisodeRecord, evaluate_episode @@ -34,6 +36,7 @@ from adaptive_rl.environments.registry import make_env from adaptive_rl.protocol.adaptation import ( AdaptationAdapter, + Transition, build_update_batch, validate_block_sequence, ) @@ -336,6 +339,27 @@ def to_dict(self) -> dict[str, Any]: return asdict(self) +def _restore_replicate(data: dict[str, Any]) -> ReplicateResult: + """Restore only records already accepted by the checkpoint digest check.""" + episode_fields = ( + "shared_pre_shift_episodes", + "shared_shock_episodes", + "adaptive_episodes", + "fixed_episodes", + ) + restored = dict(data) + for field_name in episode_fields: + episodes = [] + for episode_data in restored.get(field_name, []): + episode = dict(episode_data) + episode["transitions"] = tuple( + Transition(**transition) for transition in episode.get("transitions", []) + ) + episodes.append(EpisodeRecord(**episode)) + restored[field_name] = episodes + return ReplicateResult(**restored) + + def _run_replicate( config: ExperimentConfig, training_seed: int, @@ -530,6 +554,7 @@ def run_adaptation_benchmark( environment_factory: EnvironmentFactory = make_env, config_path: str | Path | None = None, study_run_id: str | None = None, + resume: bool = False, ) -> dict[str, Any]: """Run the selected preregistered replicates and write JSON/CSV artifacts. @@ -594,22 +619,56 @@ def run_adaptation_benchmark( if (target_dir / suffix).exists(): raise FileExistsError(f"refusing to overwrite existing artifact: {target_dir / suffix}") training_root = target_dir / "training" + state_root = target_dir / "replicate_state" + if study_run_id is not None and state_root.exists() and not resume: + raise FileExistsError("run state already exists; pass --resume to use hashed replicates") + if study_run_id is None: + for seed in selected_seeds: + if (training_root / f"seed_{seed}").exists(): + raise FileExistsError(f"refusing to overwrite training output for seed {seed}") + + results: list[ReplicateResult] = [] for seed in selected_seeds: - if (training_root / f"seed_{seed}").exists(): - raise FileExistsError(f"refusing to overwrite training output for seed {seed}") - - results = [ - _run_replicate( + training_dir = training_root / f"seed_{seed}" + checkpoint_path = state_root / f"seed_{seed}.json" + if study_run_id is not None and resume and checkpoint_path.exists(): + try: + checkpoint = read_replicate_checkpoint(checkpoint_path) + if checkpoint.get("training_seed") != seed: + raise ValueError("replicate checkpoint seed does not match schedule position") + results.append(_restore_replicate(checkpoint)) + except (OSError, TypeError, ValueError, KeyError) as exc: + failed = ReplicateResult( + training_seed=seed, + status="failed", + failure_reason=f"untrusted replicate checkpoint: {type(exc).__name__}: {exc}", + ) + results.append(failed) + continue + if study_run_id is not None and resume and training_dir.exists(): + results.append( + ReplicateResult( + training_seed=seed, + status="failed", + failure_reason=( + "interrupted replicate has no complete hashed checkpoint; " + "partial training state was not trusted" + ), + ) + ) + continue + replicate = _run_replicate( config, seed, - training_root / f"seed_{seed}", + training_dir, schedule=schedule, smoke=smoke, trainer_factory=trainer_factory, environment_factory=environment_factory, ) - for seed in selected_seeds - ] + results.append(replicate) + if study_run_id is not None: + write_replicate_checkpoint(replicate.to_dict(), state_root / f"seed_{seed}.json") vectors: dict[str, tuple[list[Optional[float]], list[Optional[float]]]] = {} for cell in PRIMARY_CELLS: @@ -753,8 +812,13 @@ def outcome_count(arm: str, status: str) -> int: if Path(config_arg).is_absolute(): raise ValueError("prereg-v1 config path must be repository-relative") output_arg = f"--output-dir {Path(output_dir)} " if output_dir is not None else "" + executable = Path(sys.argv[0]) + try: + executable_arg = executable.resolve().relative_to(Path.cwd().resolve()).as_posix() + except (OSError, ValueError): + executable_arg = executable.name command = ( - "adaptive-rl benchmark adaptation " + f"{executable_arg} benchmark adaptation " f"--config {config_arg} {output_arg}" f"--study prereg-v1 --run-id {study_run_id}" ) diff --git a/src/adaptive_rl/cli.py b/src/adaptive_rl/cli.py index cfc237f..fe08900 100644 --- a/src/adaptive_rl/cli.py +++ b/src/adaptive_rl/cli.py @@ -406,6 +406,11 @@ def benchmark_adaptation( run_id: Optional[str] = typer.Option( None, "--run-id", help="Unique immutable output directory name required with --study" ), + resume: bool = typer.Option( + False, + "--resume", + help="Reuse only complete hashed replicate checkpoints for an unfinished study run", + ), deterministic: Optional[bool] = typer.Option( None, "--deterministic/--stochastic", help="Override action selection for all evaluations" ), @@ -465,6 +470,8 @@ def benchmark_adaptation( raise ValueError("--study currently supports only prereg-v1") if (study is None) != (run_id is None): raise ValueError("--study and --run-id must be supplied together") + if resume and study is None: + raise ValueError("--resume requires --study prereg-v1 and --run-id") artifact = run_adaptation_benchmark( exp_config, @@ -473,6 +480,7 @@ def benchmark_adaptation( smoke=smoke, config_path=config, study_run_id=run_id if study is not None else None, + resume=resume, ) except Exception as err: console.print(f"[bold red]Issue #265 benchmark failed:[/bold red] {err}") @@ -487,8 +495,8 @@ def benchmark_adaptation( f"• [bold]Algorithm:[/bold] {artifact['experiment']['algorithm']}\n" f"• [bold]Completed replicates:[/bold] {completed}\n" f"• [bold]Failed replicates:[/bold] {len(failed)}\n" - f"• [bold]JSON:[/bold] {Path(output_dir or exp_config.output_dir) / ('adaptive_vs_fixed.json' if study else 'adaptation.json')}\n" - f"• [bold]CSV:[/bold] {Path(output_dir or exp_config.output_dir) / ('adaptive_vs_fixed.csv' if study else 'adaptation.csv')}\n" + f"• [bold]JSON:[/bold] {artifact['artifact_paths']['json']}\n" + f"• [bold]CSV:[/bold] {artifact['artifact_paths']['csv']}\n" f"• [bold]Scientific result:[/bold] not established by harness execution", title="Online Adaptation Benchmark", border_style="yellow" if smoke or failed else "green", diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py index 85e15d6..f5331e7 100644 --- a/tests/test_adaptation_artifacts.py +++ b/tests/test_adaptation_artifacts.py @@ -9,10 +9,12 @@ import pytest from adaptive_rl.benchmarking.adaptation_artifacts import ( + read_replicate_checkpoint, sha256_file, validate_study_manifest, write_adaptation_artifacts, write_adaptive_vs_fixed_artifacts, + write_replicate_checkpoint, write_study_manifest, ) @@ -139,3 +141,22 @@ def test_study_csv_has_one_row_per_arm_and_preserves_finite_censoring(tmp_path) assert [row["T_H"] for row in rows] == ["15", "15"] assert [row["recovery_status"] for row in rows] == ["right_censored"] * 2 assert json.loads(rows[0]["post_returns"]) == [2.0, 3.0] + + +def test_replicate_checkpoint_is_terminal_hashed_and_tamper_evident(tmp_path) -> None: + checkpoint = tmp_path / "replicate_state" / "seed_31001.json" + write_replicate_checkpoint( + {"training_seed": 31001, "status": "failed", "failure_reason": "crash"}, + checkpoint, + ) + assert read_replicate_checkpoint(checkpoint)["failure_reason"] == "crash" + original = checkpoint.read_bytes() + with pytest.raises(FileExistsError): + write_replicate_checkpoint( + {"training_seed": 31001, "status": "failed", "failure_reason": "other"}, + checkpoint, + ) + assert checkpoint.read_bytes() == original + checkpoint.write_text('{"training_seed":31001,"status":"completed"}', encoding="utf-8") + with pytest.raises(ValueError, match="checksum mismatch"): + read_replicate_checkpoint(checkpoint) diff --git a/tests/test_adaptation_invariants.py b/tests/test_adaptation_invariants.py index 483ad2a..0f22f7d 100644 --- a/tests/test_adaptation_invariants.py +++ b/tests/test_adaptation_invariants.py @@ -8,6 +8,7 @@ from adaptive_rl.benchmarking.adaptation_runner import ( _audit_replicate_invariants, + _restore_replicate, _return_vector_fingerprint, ) from adaptive_rl.protocol.constants import TRAINING_SEEDS @@ -83,3 +84,16 @@ def test_each_preregistered_invariant_violation_fails_audit( audit = _audit_replicate_invariants(replicate, schedule) assert audit[failed_check] is False assert audit["all_passed"] is False + + +def test_replicate_restore_reconstructs_terminal_state() -> None: + restored = _restore_replicate( + { + "training_seed": 31001, + "status": "failed", + "failure_reason": "recorded failure", + } + ) + assert restored.training_seed == 31001 + assert restored.status == "failed" + assert restored.failure_reason == "recorded failure" diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py index 9b0bf3a..fb33c94 100644 --- a/tests/test_adaptation_smoke.py +++ b/tests/test_adaptation_smoke.py @@ -50,3 +50,23 @@ def test_cli_adaptation_smoke_runs_complete_protocol_and_writes_artifacts(tmp_pa assert replicate["fixed_final_fingerprint"] == replicate["frozen_fingerprint"] assert artifact["paired_analysis"]["drone_disturbed/ppo"]["status"] == "inconclusive" assert (output_dir / "adaptation.csv").is_file() + + +def test_preregistered_study_rejects_subset_before_training(tmp_path: Path) -> None: + result = CliRunner().invoke( + app, + [ + "benchmark", + "adaptation", + "--study", + "prereg-v1", + "--run-id", + "subset-rejection", + "--training-seeds", + "31001", + "--output-dir", + str(tmp_path), + ], + ) + assert result.exit_code == 1 + assert "all ten seeds in order" in " ".join(result.output.split()) From 4b7f13a5ba5e25001f50a0b694d45ff9e79703cd Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 06:22:41 +0530 Subject: [PATCH 4/7] fix(experiment): harden study resume and PPO validity checks --- docs/research/issue-271.md | 37 ++- src/adaptive_rl/algorithms/adaptation.py | 8 + .../benchmarking/adaptation_artifacts.py | 185 +++++++++++- .../benchmarking/adaptation_runner.py | 166 +++++++++-- src/adaptive_rl/protocol/constants.py | 10 + tests/test_adaptation_artifacts.py | 159 ++++++++++- tests/test_adaptation_smoke.py | 46 +++ tests/test_algorithm_adaptation.py | 266 +++++++++++++++++- 8 files changed, 835 insertions(+), 42 deletions(-) diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md index f56a1cb..9180c74 100644 --- a/docs/research/issue-271.md +++ b/docs/research/issue-271.md @@ -24,16 +24,24 @@ analysis plan. The treatment is specified by The `adaptive-rl benchmark adaptation --study prereg-v1 --run-id RUN_ID` entrypoint runs all ten training seeds in preregistered order, rejects subsets and smoke mode, requires a clean committed tree, and writes into an immutable -run directory. A repeated completed run ID is refused. `--resume` accepts only -terminal per-replicate records whose SHA-256 sidecar verifies; it never trusts -partial training directories. An interrupted seed without a complete checkpoint -is recorded as failed, and remaining unstarted seeds continue. The JSON stores +run directory. Before training, `study_manifest.json` records the canonical +configuration, the frozen treatment and seed schedule, source revision, and +runtime identity. Its stable JSON representation is SHA-256 hashed. Prereg-v1 +also rejects any config or treatment-card hash that differs from the values +frozen here. `--resume` recomputes that identity and accepts only terminal +per-replicate records whose digest, study hash, protocol hash, and seed identity +verify; it never trusts partial training directories. An interrupted seed +without a complete checkpoint is recorded as failed, and remaining unstarted +seeds continue. Repeating resume after finalization validates and returns the +same immutable result. The JSON stores the raw trajectories, protocol analysis, seed schedule, outcomes, runtime invariants, and run status. The CSV has one row per replicate and arm, with finite-horizon `T_H`, status, per-episode return vectors, and seed vectors. `manifest.json` checksums every -file in the run directory; `validate_study_manifest()` detects missing or -modified files. +file in the run directory, including the pre-execution study manifest and +replicate checkpoints; `validate_study_manifest()` detects missing, changed, +malformed, or path-escaping entries. Result files are written atomically and +are not overwritten. The fixed arm has a prediction-only interface. Exact equality of its initial, per-episode, and final policy fingerprints establishes a zero weight delta; the @@ -94,6 +102,23 @@ The runner enables Torch deterministic algorithms in warn-only mode and cuDNN deterministic settings for the study, while recording that cross-hardware and cross-library bitwise reproducibility is not claimed. +## Scientific validity audit finding + +The frozen config sets `evaluation.deterministic: true`, and +`evaluate_episode()` passes that setting to PPO while collecting the post-shift +transitions later supplied to PPO's native clipped update. PPO's update assumes +actions were sampled from the recorded behavior distribution. A deterministic +mean action with its Gaussian density recorded as `behavior_log_prob` does not +have that sampling distribution, so the stored rollout is not a valid on-policy +PPO sample. The current preregistration does not define an action-sampling rule +that resolves this conflict. No treatment or analysis change is made here; the +prereg-v1 runner now rejects the frozen deterministic PPO configuration before +training. The PPO treatment must not be described as scientifically validated +until a prospective protocol amendment resolves the action-selection contract. +This limitation does not change the previously recorded artifact or its +descriptive statistics; its `COMPLETE` status describes harness execution, not +valid on-policy PPO evidence. + * Run ID: `issue271-prereg-v1-20260929-01` * Run status: `COMPLETE`; 10 completed, 0 failed, ordered seeds 31001–31010. * Executed commit: `a5250ffa5efaedaf76ad88c398f9087eb5ef49ca`; clean tree. diff --git a/src/adaptive_rl/algorithms/adaptation.py b/src/adaptive_rl/algorithms/adaptation.py index d414cb1..ca8d3c6 100644 --- a/src/adaptive_rl/algorithms/adaptation.py +++ b/src/adaptive_rl/algorithms/adaptation.py @@ -52,9 +52,17 @@ def _logger_ready(model: Any) -> Iterator[None]: old_logger = getattr(model, "_logger", None) if not had_logger: model.set_logger(Logger(folder=None, output_formats=[])) + logger = getattr(model, "logger", None) + old_values = dict(getattr(logger, "name_to_value", {})) + recorded_values = getattr(logger, "name_to_value", None) + if isinstance(recorded_values, dict): + recorded_values.clear() try: yield finally: + if isinstance(recorded_values, dict): + recorded_values.clear() + recorded_values.update(old_values) if had_logger: model._logger = old_logger else: diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py index ae3af2a..bcae90d 100644 --- a/src/adaptive_rl/benchmarking/adaptation_artifacts.py +++ b/src/adaptive_rl/benchmarking/adaptation_artifacts.py @@ -257,6 +257,71 @@ def sha256_file(path: str | Path) -> str: return digest.hexdigest() +def canonical_json_bytes(value: Any) -> bytes: + """Serialize JSON data canonically for semantic study identity.""" + return json.dumps( + _plain(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("ascii") + + +def make_study_manifest(inputs: Mapping[str, Any]) -> dict[str, Any]: + """Build the immutable, pre-execution study identity document.""" + plain_inputs = _plain(inputs) + if not isinstance(plain_inputs, dict): + raise TypeError("study manifest inputs must be a mapping") + return { + "schema_version": "1.0", + "study_hash": hashlib.sha256(canonical_json_bytes(plain_inputs)).hexdigest(), + "inputs": plain_inputs, + } + + +def write_or_verify_study_manifest( + inputs: Mapping[str, Any], manifest_path: str | Path, *, resume: bool +) -> str: + """Persist the expected study identity or fail closed when resuming.""" + manifest_path = Path(manifest_path) + expected = make_study_manifest(inputs) + if resume: + try: + existing = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("cannot resume without a valid immutable study manifest") from exc + if ( + not isinstance(existing, dict) + or existing.get("schema_version") != "1.0" + or not isinstance(existing.get("inputs"), dict) + ): + raise ValueError("study manifest has an invalid structure") + actual_hash = hashlib.sha256(canonical_json_bytes(existing["inputs"])).hexdigest() + if existing.get("study_hash") != actual_hash: + raise ValueError("study manifest hash is invalid") + if actual_hash != expected["study_hash"]: + raise ValueError("resume study hash mismatch: execution inputs changed") + return actual_hash + if manifest_path.exists(): + raise FileExistsError(f"refusing to overwrite study manifest: {manifest_path}") + manifest_path.parent.mkdir(parents=True, exist_ok=True) + encoded = (json.dumps(expected, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( + "utf-8" + ) + fd, temp_name = tempfile.mkstemp(prefix=".study-manifest-", dir=manifest_path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.link(temp_path, manifest_path) + finally: + temp_path.unlink(missing_ok=True) + return str(expected["study_hash"]) + + def write_study_manifest( artifact_path: str | Path, csv_path: str | Path, @@ -306,9 +371,12 @@ def write_study_manifest( for name in ("adaptive-rl", "gymnasium", "stable-baselines3", "torch", "numpy") }, "determinism": { - "pythonhashseed_env_recorded": False, + "pythonhashseed_env": os.environ.get("PYTHONHASHSEED"), "torch_deterministic_algorithms": _torch_deterministic_algorithms(), "torch_cudnn_deterministic": _torch_cudnn_deterministic(), + "torch_num_threads": _torch_num_threads(), + "cuda_available": _torch_cuda_available(), + "cuda_device_count": _torch_cuda_device_count(), "protocol_seed_schedule": "SHA-256 derived seeds; see adaptive_vs_fixed.json", }, "artifacts": { @@ -317,6 +385,10 @@ def write_study_manifest( if path.is_file() and path != manifest_path }, } + study_spec_path = manifest_path.parent / "study_manifest.json" + if study_spec_path.is_file(): + study_spec = json.loads(study_spec_path.read_text(encoding="utf-8")) + manifest["study_hash"] = study_spec.get("study_hash") plain = _plain(manifest) encoded = json.dumps(plain, indent=2, allow_nan=False) + "\n" manifest_path.parent.mkdir(parents=True, exist_ok=True) @@ -337,25 +409,68 @@ def validate_study_manifest(manifest_path: str | Path) -> None: """Raise when a listed immutable run artifact is missing or has changed.""" manifest_path = Path(manifest_path) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or manifest.get("schema_version") != "1.0": + raise ValueError("unsupported or malformed study artifact manifest") artifacts = manifest.get("artifacts") if not isinstance(artifacts, dict) or not artifacts: raise ValueError("manifest must list at least one artifact checksum") for relative_path, expected in artifacts.items(): - path = manifest_path.parent / relative_path + if ( + not isinstance(relative_path, str) + or not isinstance(expected, str) + or len(expected) != 64 + or any(character not in "0123456789abcdef" for character in expected) + ): + raise ValueError("manifest contains an invalid artifact path or SHA-256 digest") + relative = Path(relative_path) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"manifest artifact path escapes the run directory: {relative_path}") + path = manifest_path.parent / relative + try: + path.resolve().relative_to(manifest_path.parent.resolve()) + except (OSError, ValueError) as exc: + raise ValueError( + f"manifest artifact path escapes the run directory: {relative_path}" + ) from exc if not path.is_file(): raise ValueError(f"manifest artifact is missing: {relative_path}") actual = sha256_file(path) if actual != expected: raise ValueError(f"manifest checksum mismatch: {relative_path}") - - -def write_replicate_checkpoint(replicate: Mapping[str, Any], checkpoint_path: str | Path) -> None: - """Persist one terminal replicate result and a digest without overwriting.""" + if "study_hash" in manifest: + spec_path = manifest_path.parent / "study_manifest.json" + if not spec_path.is_file() or "study_manifest.json" not in artifacts: + raise ValueError("study artifact manifest does not bind its pre-execution study spec") + spec = json.loads(spec_path.read_text(encoding="utf-8")) + if not isinstance(spec, dict) or not isinstance(spec.get("inputs"), dict): + raise ValueError("pre-execution study manifest is malformed") + actual_study_hash = hashlib.sha256(canonical_json_bytes(spec["inputs"])).hexdigest() + if spec.get("study_hash") != actual_study_hash or manifest.get("study_hash") != actual_study_hash: + raise ValueError("study artifact manifest hash does not match its pre-execution spec") + + +def write_replicate_checkpoint( + replicate: Mapping[str, Any], + checkpoint_path: str | Path, + *, + study_hash: str, + protocol_hash: str, +) -> None: + """Persist a terminal replicate bound to its study and replicate identity.""" checkpoint_path = Path(checkpoint_path) if replicate.get("status") not in {"completed", "failed"}: raise ValueError("only terminal replicate states may be checkpointed") plain = _plain(dict(replicate)) - encoded = (json.dumps(plain, indent=2, allow_nan=False) + "\n").encode("utf-8") + envelope = { + "schema_version": "1.0", + "study_hash": study_hash, + "protocol_hash": protocol_hash, + "replicate_id": plain.get("training_seed"), + "replicate": plain, + } + encoded = (json.dumps(envelope, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( + "utf-8" + ) checkpoint_path.parent.mkdir(parents=True, exist_ok=True) fd, temp_name = tempfile.mkstemp(prefix=".replicate-", dir=checkpoint_path.parent) temp_path = Path(temp_name) @@ -389,8 +504,14 @@ def write_replicate_checkpoint(replicate: Mapping[str, Any], checkpoint_path: st temp_path.unlink(missing_ok=True) -def read_replicate_checkpoint(checkpoint_path: str | Path) -> dict[str, Any]: - """Read only a terminal checkpoint whose adjacent digest verifies.""" +def read_replicate_checkpoint( + checkpoint_path: str | Path, + *, + study_hash: str, + protocol_hash: str, + training_seed: int, +) -> dict[str, Any]: + """Read a terminal checkpoint after digest, study, protocol, and ID checks.""" checkpoint_path = Path(checkpoint_path) digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") payload = checkpoint_path.read_bytes() @@ -400,7 +521,21 @@ def read_replicate_checkpoint(checkpoint_path: str | Path) -> dict[str, Any]: actual = hashlib.sha256(payload).hexdigest() if actual != expected: raise ValueError(f"replicate checkpoint checksum mismatch: {checkpoint_path.name}") - result = json.loads(payload) + envelope = json.loads(payload) + if ( + not isinstance(envelope, dict) + or envelope.get("schema_version") != "1.0" + or not isinstance(envelope.get("replicate"), dict) + ): + raise ValueError("replicate checkpoint envelope is malformed") + if envelope.get("study_hash") != study_hash: + raise ValueError("replicate checkpoint study hash mismatch") + if envelope.get("protocol_hash") != protocol_hash: + raise ValueError("replicate checkpoint protocol hash mismatch") + replicate = envelope["replicate"] + if envelope.get("replicate_id") != training_seed or replicate.get("training_seed") != training_seed: + raise ValueError("replicate checkpoint identity mismatch") + result = replicate if result.get("status") not in {"completed", "failed"}: raise ValueError("replicate checkpoint is not terminal") return cast(dict[str, Any], result) @@ -431,6 +566,33 @@ def _torch_cudnn_deterministic() -> bool | None: return None +def _torch_num_threads() -> int | None: + try: + import torch + + return int(torch.get_num_threads()) + except ImportError: + return None + + +def _torch_cuda_available() -> bool | None: + try: + import torch + + return bool(torch.cuda.is_available()) + except ImportError: + return None + + +def _torch_cuda_device_count() -> int | None: + try: + import torch + + return int(torch.cuda.device_count()) + except ImportError: + return None + + def _write_csv(handle: Any, artifact: Mapping[str, Any]) -> None: writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS, extrasaction="raise") writer.writeheader() @@ -440,11 +602,14 @@ def _write_csv(handle: Any, artifact: Mapping[str, Any]) -> None: __all__ = [ "CSV_FIELDS", "STUDY_CSV_FIELDS", + "canonical_json_bytes", + "make_study_manifest", "sha256_file", "read_replicate_checkpoint", "validate_study_manifest", "write_adaptation_artifacts", "write_adaptive_vs_fixed_artifacts", "write_replicate_checkpoint", + "write_or_verify_study_manifest", "write_study_manifest", ] diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py index 05b088e..1425bcb 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -6,6 +6,7 @@ import importlib.metadata import json import logging +import os import platform import subprocess import sys @@ -24,9 +25,12 @@ run_adaptation_update, ) from adaptive_rl.benchmarking.adaptation_artifacts import ( + canonical_json_bytes, read_replicate_checkpoint, + validate_study_manifest, write_adaptation_artifacts, write_adaptive_vs_fixed_artifacts, + write_or_verify_study_manifest, write_replicate_checkpoint, write_study_manifest, ) @@ -40,7 +44,14 @@ build_update_batch, validate_block_sequence, ) -from adaptive_rl.protocol.constants import K_PRE, N_POST, PRIMARY_CELLS, TRAINING_SEEDS +from adaptive_rl.protocol.constants import ( + ISSUE271_CONFIG_SHA256, + ISSUE271_TREATMENT_CARD_SHA256, + K_PRE, + N_POST, + PRIMARY_CELLS, + TRAINING_SEEDS, +) from adaptive_rl.protocol.fork import fork_adaptive_and_fixed, model_fingerprint from adaptive_rl.protocol.recovery import compute_recovery from adaptive_rl.protocol.seeds import frozen_schedule, schedule_fingerprint @@ -76,7 +87,34 @@ def _repository_metadata() -> dict[str, Any]: versions[distribution] = importlib.metadata.version(distribution) except importlib.metadata.PackageNotFoundError: versions[distribution] = None - return {"repository_commit": commit, "working_tree_dirty": dirty, "runtime_versions": versions} + runtime_settings: dict[str, Any] = { + name: os.environ.get(name) + for name in ( + "PYTHONHASHSEED", + "CUBLAS_WORKSPACE_CONFIG", + "CUDA_VISIBLE_DEVICES", + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + ) + } + runtime_settings.update( + { + "torch_num_threads": torch.get_num_threads(), + "cuda_available": torch.cuda.is_available(), + "cuda_device_count": torch.cuda.device_count(), + "cuda_device_names": [ + torch.cuda.get_device_name(index) + for index in range(torch.cuda.device_count()) + ], + } + ) + return { + "repository_commit": commit, + "working_tree_dirty": dirty, + "runtime_versions": versions, + "runtime_settings": runtime_settings, + } def _sha256_file(path: Path) -> str: @@ -569,6 +607,16 @@ def run_adaptation_benchmark( raise ValueError("Issue #265 supports only PPO and SAC") if config.training is None: raise ValueError("Issue #265 requires a training section") + card = _card_path() + if not card.is_file(): + raise FileNotFoundError(f"Treatment Card is required before experiment execution: {card}") + card_sha = _sha256_file(card) + repository_root = Path.cwd().resolve() + config_arg = ( + Path(config_path).as_posix() + if config_path is not None + else "configs/drone_distribution_shift.yaml" + ) schedule = frozen_schedule() schedule_fp = schedule_fingerprint(schedule) if training_seeds is None: @@ -588,6 +636,30 @@ def run_adaptation_benchmark( raise ValueError("prereg-v1 requires one non-smoke attempt of all ten seeds in order") if output_dir is not None and Path(output_dir).is_absolute(): raise ValueError("prereg-v1 artifact output_dir must be relative to the repository") + selected_output_dir = Path(output_dir) if output_dir is not None else Path(config.output_dir) + try: + selected_output_dir.resolve().relative_to(repository_root) + except ValueError as exc: + raise ValueError("prereg-v1 artifact output_dir must stay inside the repository") from exc + if Path(config_arg).is_absolute(): + raise ValueError("prereg-v1 config path must be repository-relative") + try: + (repository_root / config_arg).resolve().relative_to(repository_root) + except ValueError as exc: + raise ValueError("prereg-v1 config path must stay inside the repository") from exc + if config.algorithm.name.strip().lower() != "ppo": + raise ValueError("prereg-v1 is frozen to the drone_disturbed/ppo cell") + if config.environment.name != "drone_disturbed" or benchmark.scenario != "TEST-B": + raise ValueError("prereg-v1 is frozen to drone_disturbed under TEST-B") + if compute_config_sha256(config) != ISSUE271_CONFIG_SHA256: + raise ValueError("prereg-v1 config differs from the frozen Issue #271 configuration") + if card_sha != ISSUE271_TREATMENT_CARD_SHA256: + raise ValueError("prereg-v1 Treatment Card differs from the frozen treatment") + if config.evaluation.deterministic: + raise ValueError( + "prereg-v1 cannot run native PPO adaptation with deterministic mean actions; " + "the recorded actions are not sampled from the behavior distribution" + ) try: dirty = subprocess.check_output( ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL @@ -602,10 +674,6 @@ def run_adaptation_benchmark( else: determinism = None - card = _card_path() - if not card.is_file(): - raise FileNotFoundError(f"Treatment Card is required before experiment execution: {card}") - card_sha = _sha256_file(card) config_file = Path(config_path).resolve() if config_path is not None else None config_file_sha = _sha256_file(config_file) if config_file is not None else None @@ -615,9 +683,63 @@ def run_adaptation_benchmark( target_dir.mkdir(parents=True, exist_ok=True) stem = "adaptive_vs_fixed" if study_run_id is not None else "adaptation" suffixes = (f"{stem}.json", f"{stem}.csv") - for suffix in (*suffixes, "manifest.json"): - if (target_dir / suffix).exists(): - raise FileExistsError(f"refusing to overwrite existing artifact: {target_dir / suffix}") + study_hash: str | None = None + protocol_hash: str | None = None + if study_run_id is not None: + runtime_identity = _repository_metadata() + study_inputs = { + "study": "adaptive-vs-fixed/prereg-v1", + "study_config": config.model_dump( + mode="json", exclude={"output_dir", "log_dir"} + ), + "canonical_config_sha256": compute_config_sha256(config), + "treatment_card_sha256": card_sha, + "protocol_version": benchmark.protocol_version, + "protocol_constants": { + "pre_episodes": K_PRE, + "post_episodes": N_POST, + "training_seeds": list(TRAINING_SEEDS), + "schedule": schedule, + "schedule_fingerprint": schedule_fp, + }, + "source_identity": runtime_identity, + "runtime_determinism": determinism, + } + study_hash = write_or_verify_study_manifest( + study_inputs, target_dir / "study_manifest.json", resume=resume + ) + protocol_hash = hashlib.sha256( + canonical_json_bytes( + { + "protocol_version": benchmark.protocol_version, + "treatment_card_sha256": card_sha, + "protocol_constants": study_inputs["protocol_constants"], + } + ) + ).hexdigest() + final_paths = [target_dir / suffix for suffix in (*suffixes, "manifest.json")] + if resume and all(path.is_file() for path in final_paths): + validate_study_manifest(final_paths[-1]) + try: + completed_artifact = json.loads(final_paths[0].read_text(encoding="utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("completed study artifact is malformed") from exc + if ( + not isinstance(completed_artifact, dict) + or completed_artifact.get("run_id") != study_run_id + or completed_artifact.get("study_hash") != study_hash + ): + raise ValueError("completed study artifact does not match the expected study hash") + return completed_artifact + for path in final_paths: + if path.exists(): + raise FileExistsError( + f"refusing to overwrite existing artifact or partial completion: {path}" + ) + else: + for suffix in (*suffixes, "manifest.json"): + if (target_dir / suffix).exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {target_dir / suffix}") training_root = target_dir / "training" state_root = target_dir / "replicate_state" if study_run_id is not None and state_root.exists() and not resume: @@ -633,9 +755,13 @@ def run_adaptation_benchmark( checkpoint_path = state_root / f"seed_{seed}.json" if study_run_id is not None and resume and checkpoint_path.exists(): try: - checkpoint = read_replicate_checkpoint(checkpoint_path) - if checkpoint.get("training_seed") != seed: - raise ValueError("replicate checkpoint seed does not match schedule position") + assert study_hash is not None and protocol_hash is not None + checkpoint = read_replicate_checkpoint( + checkpoint_path, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seed=seed, + ) results.append(_restore_replicate(checkpoint)) except (OSError, TypeError, ValueError, KeyError) as exc: failed = ReplicateResult( @@ -668,7 +794,13 @@ def run_adaptation_benchmark( ) results.append(replicate) if study_run_id is not None: - write_replicate_checkpoint(replicate.to_dict(), state_root / f"seed_{seed}.json") + assert study_hash is not None and protocol_hash is not None + write_replicate_checkpoint( + replicate.to_dict(), + state_root / f"seed_{seed}.json", + study_hash=study_hash, + protocol_hash=protocol_hash, + ) vectors: dict[str, tuple[list[Optional[float]], list[Optional[float]]]] = {} for cell in PRIMARY_CELLS: @@ -728,6 +860,7 @@ def outcome_count(arm: str, status: str) -> int: artifact: dict[str, Any] = { "schema_version": "1.0", "run_id": study_run_id, + "study_hash": study_hash, "run_status": run_status if study_run_id is not None else None, "protocol_version": benchmark.protocol_version, "issue": "271" if study_run_id is not None else "265", @@ -804,13 +937,6 @@ def outcome_count(arm: str, status: str) -> int: **({"manifest": str(target_dir / "manifest.json")} if study_run_id else {}), } if study_run_id is not None: - config_arg = ( - Path(config_path).as_posix() - if config_path is not None - else "configs/drone_distribution_shift.yaml" - ) - if Path(config_arg).is_absolute(): - raise ValueError("prereg-v1 config path must be repository-relative") output_arg = f"--output-dir {Path(output_dir)} " if output_dir is not None else "" executable = Path(sys.argv[0]) try: diff --git a/src/adaptive_rl/protocol/constants.py b/src/adaptive_rl/protocol/constants.py index b1edf20..54f8931 100644 --- a/src/adaptive_rl/protocol/constants.py +++ b/src/adaptive_rl/protocol/constants.py @@ -101,6 +101,14 @@ #: Number of planned replicate pairs per cell (len(TRAINING_SEEDS)). PLANNED_N: int = len(TRAINING_SEEDS) +# Issue #271 prereg-v1 freezes one PPO cell and its semantic config/treatment. +# These hashes are declared in docs/research/issue-271.md and require a new +# study version if either scientific input changes. +ISSUE271_CONFIG_SHA256: str = "0039c298b5048254b2d211cc66967e9d3e1d71275575fcfe28736485c65937b5" +ISSUE271_TREATMENT_CARD_SHA256: str = ( + "8383e736f02ff31393474b241b06d2dc92b036910c93c26081c1673c92a317e3" +) + __all__ = [ "ALPHA", "BOOTSTRAP_REPS", @@ -108,6 +116,8 @@ "CONFIG_TEST_POOL", "CONFIG_TRAIN_POOL", "HORIZON", + "ISSUE271_CONFIG_SHA256", + "ISSUE271_TREATMENT_CARD_SHA256", "K_PRE", "MIN_DEGRADATION_SE_MULTIPLIER", "MIN_VALID_N", diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py index f5331e7..1302129 100644 --- a/tests/test_adaptation_artifacts.py +++ b/tests/test_adaptation_artifacts.py @@ -9,14 +9,22 @@ import pytest from adaptive_rl.benchmarking.adaptation_artifacts import ( + canonical_json_bytes, + make_study_manifest, read_replicate_checkpoint, sha256_file, validate_study_manifest, write_adaptation_artifacts, write_adaptive_vs_fixed_artifacts, + write_or_verify_study_manifest, write_replicate_checkpoint, write_study_manifest, ) +from adaptive_rl.config import compute_config_sha256, load_config +from adaptive_rl.protocol.constants import ( + ISSUE271_CONFIG_SHA256, + ISSUE271_TREATMENT_CARD_SHA256, +) def _artifact(): @@ -117,6 +125,70 @@ def test_manifest_hashes_all_artifacts_and_detects_tampering(tmp_path) -> None: validate_study_manifest(manifest_path) +def test_completed_manifest_binds_preexecution_study_hash(tmp_path) -> None: + spec_path = tmp_path / "study_manifest.json" + study_hash = write_or_verify_study_manifest( + {"protocol": "prereg-v1", "seed": 31001}, spec_path, resume=False + ) + json_path, csv_path = write_adaptation_artifacts( + _artifact(), tmp_path, stem="adaptive_vs_fixed" + ) + manifest_path = tmp_path / "manifest.json" + write_study_manifest( + json_path, + csv_path, + manifest_path, + run_id="bound-study", + command="adaptive-rl benchmark adaptation --study prereg-v1", + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["study_hash"] == study_hash + validate_study_manifest(manifest_path) + + changed_spec = json.loads(spec_path.read_text(encoding="utf-8")) + changed_spec["inputs"]["seed"] = 31002 + spec_path.write_text(json.dumps(changed_spec), encoding="utf-8") + manifest["artifacts"]["study_manifest.json"] = sha256_file(spec_path) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ValueError, match="does not match its pre-execution spec"): + validate_study_manifest(manifest_path) + + +def test_manifest_rejects_path_escape_and_unsupported_schema(tmp_path) -> None: + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps({"schema_version": "1.0", "artifacts": {"../outside": "0" * 64}}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="escapes the run directory"): + validate_study_manifest(manifest_path) + manifest_path.write_text( + json.dumps({"schema_version": "2.0", "artifacts": {"x": "0" * 64}}), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="unsupported or malformed"): + validate_study_manifest(manifest_path) + + +def test_manifest_rejects_symlinked_artifact_outside_run_directory(tmp_path) -> None: + outside = tmp_path.parent / f"{tmp_path.name}-outside" + outside.write_bytes(b"outside") + link = tmp_path / "external.bin" + link.symlink_to(outside) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "artifacts": {"external.bin": sha256_file(outside)}, + } + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="escapes the run directory"): + validate_study_manifest(manifest_path) + + def test_study_csv_has_one_row_per_arm_and_preserves_finite_censoring(tmp_path) -> None: data = { "replicates": [ @@ -145,18 +217,99 @@ def test_study_csv_has_one_row_per_arm_and_preserves_finite_censoring(tmp_path) def test_replicate_checkpoint_is_terminal_hashed_and_tamper_evident(tmp_path) -> None: checkpoint = tmp_path / "replicate_state" / "seed_31001.json" + study_hash = "study-hash" + protocol_hash = "protocol-hash" + replicate = {"training_seed": 31001, "status": "failed", "failure_reason": "crash"} write_replicate_checkpoint( - {"training_seed": 31001, "status": "failed", "failure_reason": "crash"}, + replicate, checkpoint, + study_hash=study_hash, + protocol_hash=protocol_hash, ) - assert read_replicate_checkpoint(checkpoint)["failure_reason"] == "crash" + assert read_replicate_checkpoint( + checkpoint, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seed=31001, + )["failure_reason"] == "crash" original = checkpoint.read_bytes() with pytest.raises(FileExistsError): write_replicate_checkpoint( {"training_seed": 31001, "status": "failed", "failure_reason": "other"}, checkpoint, + study_hash=study_hash, + protocol_hash=protocol_hash, ) assert checkpoint.read_bytes() == original + with pytest.raises(ValueError, match="study hash mismatch"): + read_replicate_checkpoint( + checkpoint, + study_hash="different-study", + protocol_hash=protocol_hash, + training_seed=31001, + ) + with pytest.raises(ValueError, match="protocol hash mismatch"): + read_replicate_checkpoint( + checkpoint, + study_hash=study_hash, + protocol_hash="different-protocol", + training_seed=31001, + ) + with pytest.raises(ValueError, match="identity mismatch"): + read_replicate_checkpoint( + checkpoint, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seed=31002, + ) checkpoint.write_text('{"training_seed":31001,"status":"completed"}', encoding="utf-8") with pytest.raises(ValueError, match="checksum mismatch"): - read_replicate_checkpoint(checkpoint) + read_replicate_checkpoint( + checkpoint, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seed=31001, + ) + + +def test_canonical_study_hash_ignores_mapping_order_and_json_formatting(tmp_path) -> None: + left = {"config": {"seed": 31001, "lr": 0.001}, "schedule": [1, 2, 3]} + right = {"schedule": [1, 2, 3], "config": {"lr": 0.001, "seed": 31001}} + assert canonical_json_bytes(left) == canonical_json_bytes(right) + assert make_study_manifest(left)["study_hash"] == make_study_manifest(right)["study_hash"] + + path = tmp_path / "study_manifest.json" + digest = write_or_verify_study_manifest(left, path, resume=False) + path.write_text(json.dumps(json.loads(path.read_text()), indent=4), encoding="utf-8") + assert write_or_verify_study_manifest(right, path, resume=True) == digest + + +def test_study_manifest_material_change_and_tampered_hash_fail_resume(tmp_path) -> None: + path = tmp_path / "study_manifest.json" + inputs = {"config": {"gamma": 0.99}, "seeds": [1, 2]} + write_or_verify_study_manifest(inputs, path, resume=False) + with pytest.raises(ValueError, match="study hash mismatch"): + write_or_verify_study_manifest( + {"config": {"gamma": 0.98}, "seeds": [1, 2]}, path, resume=True + ) + + document = json.loads(path.read_text(encoding="utf-8")) + document["study_hash"] = "0" * 64 + path.write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(ValueError, match="hash is invalid"): + write_or_verify_study_manifest(inputs, path, resume=True) + + +def test_resume_rejects_missing_or_malformed_study_manifest(tmp_path) -> None: + path = tmp_path / "study_manifest.json" + with pytest.raises(ValueError, match="valid immutable study manifest"): + write_or_verify_study_manifest({"a": 1}, path, resume=True) + path.write_text("{broken", encoding="utf-8") + with pytest.raises(ValueError, match="valid immutable study manifest"): + write_or_verify_study_manifest({"a": 1}, path, resume=True) + + +def test_issue271_preregistered_inputs_match_frozen_hashes() -> None: + config = load_config("configs/drone_distribution_shift.yaml") + assert compute_config_sha256(config) == ISSUE271_CONFIG_SHA256 + assert sha256_file("docs/research/TREATMENT_CARD.md") == ISSUE271_TREATMENT_CARD_SHA256 diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py index fb33c94..71b426f 100644 --- a/tests/test_adaptation_smoke.py +++ b/tests/test_adaptation_smoke.py @@ -5,9 +5,12 @@ import json from pathlib import Path +import pytest from typer.testing import CliRunner +from adaptive_rl.benchmarking.adaptation_runner import run_adaptation_benchmark from adaptive_rl.cli import app +from adaptive_rl.config import load_config from adaptive_rl.protocol.constants import TRAINING_SEEDS from adaptive_rl.protocol.seeds import frozen_schedule, schedule_fingerprint @@ -70,3 +73,46 @@ def test_preregistered_study_rejects_subset_before_training(tmp_path: Path) -> N ) assert result.exit_code == 1 assert "all ten seeds in order" in " ".join(result.output.split()) + + +def test_preregistered_study_rejects_changed_scientific_config_before_training() -> None: + config = load_config("configs/drone_distribution_shift.yaml") + algorithm = config.algorithm.model_copy(update={"learning_rate": 0.0002}, deep=True) + changed = config.model_copy(update={"algorithm": algorithm}, deep=True) + with pytest.raises(ValueError, match="frozen Issue #271 configuration"): + run_adaptation_benchmark( + changed, + study_run_id="changed-config", + training_seeds=TRAINING_SEEDS, + ) + + +def test_preregistered_study_rejects_deterministic_ppo_rollout_before_training() -> None: + with pytest.raises(ValueError, match="not sampled from the behavior distribution"): + run_adaptation_benchmark( + load_config("configs/drone_distribution_shift.yaml"), + study_run_id="deterministic-ppo-rejection", + training_seeds=TRAINING_SEEDS, + ) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"output_dir": Path("/tmp/absolute-study-output")}, "must be relative"), + ( + {"config_path": Path.cwd() / "configs/drone_distribution_shift.yaml"}, + "config path must be repository-relative", + ), + ], +) +def test_preregistered_study_rejects_nonreproducible_paths_before_training( + kwargs: dict[str, Path], message: str +) -> None: + with pytest.raises(ValueError, match=message): + run_adaptation_benchmark( + load_config("configs/drone_distribution_shift.yaml"), + study_run_id="invalid-path", + training_seeds=TRAINING_SEEDS, + **kwargs, + ) diff --git a/tests/test_algorithm_adaptation.py b/tests/test_algorithm_adaptation.py index 1369af5..5ab8ce9 100644 --- a/tests/test_algorithm_adaptation.py +++ b/tests/test_algorithm_adaptation.py @@ -2,10 +2,14 @@ from __future__ import annotations +import random + import gymnasium as gym import numpy as np +import pytest import torch from gymnasium import spaces +from stable_baselines3.common.logger import Logger from adaptive_rl.algorithms.adaptation import ( PPOAdaptationAdapter, @@ -15,7 +19,12 @@ from adaptive_rl.algorithms.ppo import PPOAlgorithm from adaptive_rl.algorithms.sac import SACAlgorithm from adaptive_rl.benchmarking.adaptation_runtime import evaluate_episode -from adaptive_rl.protocol.adaptation import PostShiftEpisode, Transition, build_update_batch +from adaptive_rl.protocol.adaptation import ( + PostShiftEpisode, + Transition, + UpdateBatch, + build_update_batch, +) from adaptive_rl.protocol.fork import model_fingerprint from adaptive_rl.protocol.seeds import derive_seed @@ -48,7 +57,11 @@ def _batch(algorithm, training_seed: int = 31001): for episode_index in range(1, 6): transitions = [] for step in range(3): - observation = np.full(model.observation_space.shape, step * 0.1, dtype=np.float32) + observation = np.full( + model.observation_space.shape, + episode_index * 0.01 + step * 0.1, + dtype=np.float32, + ) next_observation = observation + 0.05 if isinstance(algorithm, PPOAlgorithm): obs_tensor, _ = model.policy.obs_to_tensor(observation) @@ -88,6 +101,28 @@ def _batch(algorithm, training_seed: int = 31001): return build_update_batch(training_seed, episodes, block_episode=5) +def _static_ppo_batch(seed: int) -> UpdateBatch: + transitions = tuple( + Transition( + observation=np.full(3, index / 10.0, dtype=np.float32), + action=np.zeros(1, dtype=np.float32), + reward=float(index % 4), + next_observation=np.full(3, (index + 1) / 10.0, dtype=np.float32), + terminated=index % 4 == 3, + truncated=False, + behavior_log_prob=-0.5, + behavior_value=0.0, + ) + for index in range(8) + ) + return UpdateBatch( + block_episode=5, + seed=seed, + visible_episode_indices=(1, 2, 3, 4, 5), + transitions=transitions, + ) + + def test_ppo_native_update_uses_recorded_rollout_without_environment_steps() -> None: env = gym.make("Pendulum-v1") try: @@ -97,9 +132,12 @@ def test_ppo_native_update_uses_recorded_rollout_without_environment_steps() -> batch = _batch(algorithm) model = algorithm.model assert model is not None + model.set_logger(Logger(folder=None, output_formats=[])) + model.logger.record("train/stale_loss", 999.0) before = model_fingerprint(algorithm) log = run_adaptation_update(algorithm, PPOAdaptationAdapter(), batch) assert log.loss_metrics + assert "train/stale_loss" not in log.loss_metrics assert log.block_episode == 5 assert log.transition_count == 15 assert log.visible_episode_indices == (1, 2, 3, 4, 5) @@ -112,6 +150,196 @@ def test_ppo_native_update_uses_recorded_rollout_without_environment_steps() -> env.close() +def test_ppo_buffer_gae_matches_hand_calculation_for_terminal_and_truncation() -> None: + env = gym.make("Pendulum-v1") + try: + algorithm = PPOAlgorithm( + env=env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" + ) + model = algorithm.model + assert model is not None + values = (0.1, 0.2, 0.3, 0.4) + transitions = ( + Transition( + observation=np.zeros(3, dtype=np.float32), + action=np.zeros(1, dtype=np.float32), + reward=1.0, + next_observation=np.ones(3, dtype=np.float32), + terminated=False, + truncated=False, + behavior_log_prob=0.0, + behavior_value=values[0], + ), + Transition( + observation=np.ones(3, dtype=np.float32), + action=np.zeros(1, dtype=np.float32), + reward=2.0, + next_observation=np.full(3, 2.0, dtype=np.float32), + terminated=False, + truncated=True, + behavior_log_prob=0.0, + behavior_value=values[1], + behavior_next_value=0.5, + ), + Transition( + observation=np.full(3, 3.0, dtype=np.float32), + action=np.zeros(1, dtype=np.float32), + reward=3.0, + next_observation=np.full(3, 4.0, dtype=np.float32), + terminated=False, + truncated=False, + behavior_log_prob=0.0, + behavior_value=values[2], + ), + Transition( + observation=np.full(3, 4.0, dtype=np.float32), + action=np.zeros(1, dtype=np.float32), + reward=4.0, + next_observation=np.full(3, 5.0, dtype=np.float32), + terminated=True, + truncated=False, + behavior_log_prob=0.0, + behavior_value=values[3], + ), + ) + batch = UpdateBatch( + block_episode=5, + seed=derive_seed(31001, "update", 0), + visible_episode_indices=(1, 2, 3, 4, 5), + transitions=transitions, + ) + gamma = model.gamma + gae_lambda = model.gae_lambda + expected_last_truncated = 2.0 + gamma * 0.5 - values[1] + expected_first_truncated = ( + 1.0 + gamma * values[1] - values[0] + + gamma * gae_lambda * expected_last_truncated + ) + expected_last_terminated = 4.0 - values[3] + expected_first_terminated = ( + 3.0 + gamma * values[3] - values[2] + + gamma * gae_lambda * expected_last_terminated + ) + expected_advantages = np.asarray( + [ + expected_first_truncated, + expected_last_truncated, + expected_first_terminated, + expected_last_terminated, + ], + dtype=np.float32, + ) + captured: dict[str, np.ndarray] = {} + original_train = model.train + + def capture_buffer() -> None: + captured["advantages"] = model.rollout_buffer.advantages.copy().reshape(-1) + captured["returns"] = model.rollout_buffer.returns.copy().reshape(-1) + model.logger.record("train/loss", 1.0) + + model.train = capture_buffer + try: + PPOAdaptationAdapter().update(algorithm, batch) + finally: + model.train = original_train + np.testing.assert_allclose(captured["advantages"], expected_advantages, rtol=1e-6) + np.testing.assert_allclose( + captured["returns"], expected_advantages + np.asarray(values), rtol=1e-6 + ) + finally: + env.close() + + +def test_ppo_update_restores_caller_rng_and_repeats_from_same_update_seed() -> None: + envs = [gym.make("Pendulum-v1") for _ in range(3)] + try: + algorithms = [ + PPOAlgorithm( + env=env, + n_steps=8, + batch_size=4, + n_epochs=2, + seed=31001, + device="cpu", + ) + for env in envs + ] + assert len({model_fingerprint(algorithm) for algorithm in algorithms}) == 1 + python_state = random.getstate() + numpy_state = np.random.get_state() + torch_state = torch.random.get_rng_state().clone() + batch = _static_ppo_batch(derive_seed(31001, "update", 0)) + fingerprints = [] + for algorithm in algorithms[:2]: + run_adaptation_update(algorithm, PPOAdaptationAdapter(), batch) + fingerprints.append(model_fingerprint(algorithm)) + assert random.getstate() == python_state + after_numpy = np.random.get_state() + assert after_numpy[0] == numpy_state[0] + np.testing.assert_array_equal(after_numpy[1], numpy_state[1]) + assert after_numpy[2:] == numpy_state[2:] + assert torch.equal(torch.random.get_rng_state(), torch_state) + assert fingerprints[0] == fingerprints[1] + + other_batch = _static_ppo_batch(derive_seed(31001, "update", 1)) + run_adaptation_update(algorithms[2], PPOAdaptationAdapter(), other_batch) + assert model_fingerprint(algorithms[2]) != fingerprints[0] + finally: + for env in envs: + env.close() + + +@pytest.mark.parametrize( + ("failure", "error_match"), + [ + ("exception", "forced update failure"), + ("nonfinite_parameter", "non-finite model state"), + ("nonfinite_optimizer", "non-finite optimizer"), + ("invalid_shape", "shape or dtype"), + ], +) +def test_failed_ppo_update_restores_model_and_optimizer_state( + failure: str, error_match: str +) -> None: + env = gym.make("Pendulum-v1") + try: + algorithm = PPOAlgorithm( + env=env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" + ) + model = algorithm.model + assert model is not None + before_fingerprint = model_fingerprint(algorithm) + before_num_timesteps = model.num_timesteps + before_optimizer = model.policy.optimizer.state_dict() + + class FailingAdapter: + def update(self, target, batch) -> None: + del batch + target_model = target.model + parameter = next(target_model.policy.parameters()) + target_model.num_timesteps = 999 + if failure == "exception": + raise RuntimeError("forced update failure") + if failure == "nonfinite_parameter": + parameter.data.fill_(float("nan")) + elif failure == "nonfinite_optimizer": + target_model.policy.optimizer.state[parameter]["exp_avg"] = torch.full_like( + parameter, float("inf") + ) + else: + parameter.data = parameter.data.reshape(-1)[:-1] + + with pytest.raises((RuntimeError, FloatingPointError), match=error_match): + run_adaptation_update(algorithm, FailingAdapter(), _static_ppo_batch(1)) + restored = algorithm.model + assert restored is not None + assert model_fingerprint(algorithm) == before_fingerprint + assert restored.num_timesteps == before_num_timesteps + assert restored.policy.optimizer.state_dict() == before_optimizer + finally: + env.close() + + def test_sac_native_update_uses_fresh_post_only_replay_buffer() -> None: env = gym.make("Pendulum-v1") try: @@ -129,7 +357,24 @@ def test_sac_native_update_uses_fresh_post_only_replay_buffer() -> None: assert model is not None original_buffer = model.replay_buffer before = model_fingerprint(algorithm) - log = run_adaptation_update(algorithm, SACAdaptationAdapter(), batch) + native_train = model.train + replay_snapshot: dict[str, np.ndarray] = {} + + def inspect_replay(gradient_steps: int, batch_size: int) -> None: + replay = model.replay_buffer + assert replay is not None + size = replay.size() + replay_snapshot["observations"] = replay.observations[:size, 0].copy() + replay_snapshot["rewards"] = replay.rewards[:size, 0].copy() + replay_snapshot["dones"] = replay.dones[:size, 0].copy() + replay_snapshot["timeouts"] = replay.timeouts[:size, 0].copy() + native_train(gradient_steps=gradient_steps, batch_size=batch_size) + + model.train = inspect_replay + try: + log = run_adaptation_update(algorithm, SACAdaptationAdapter(), batch) + finally: + model.train = native_train assert log.loss_metrics assert log.block_episode == 5 assert log.transition_count == 15 @@ -141,6 +386,21 @@ def test_sac_native_update_uses_fresh_post_only_replay_buffer() -> None: assert model.replay_buffer is original_buffer assert original_buffer is not None and original_buffer.size() == 0 assert model.num_timesteps == 0 + np.testing.assert_allclose( + replay_snapshot["observations"], + np.stack([transition.observation for transition in batch.transitions]), + ) + np.testing.assert_allclose( + replay_snapshot["rewards"], [transition.reward for transition in batch.transitions] + ) + np.testing.assert_array_equal( + replay_snapshot["dones"], + [float(transition.terminated or transition.truncated) for transition in batch.transitions], + ) + np.testing.assert_array_equal( + replay_snapshot["timeouts"], + [float(transition.truncated and not transition.terminated) for transition in batch.transitions], + ) finally: env.close() From 634f33f314b0fd365196e707c0827523fce91625 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 06:30:46 +0530 Subject: [PATCH 5/7] fix(ppo): reject deterministic adaptation rollouts --- docs/research/issue-265.md | 17 +++++++++-- docs/research/issue-271.md | 9 ++++-- .../benchmarking/adaptation_runner.py | 12 ++++++-- src/adaptive_rl/cli.py | 4 ++- tests/test_adaptation_smoke.py | 29 ++++++++++++++++++- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/docs/research/issue-265.md b/docs/research/issue-265.md index 28a92c1..7293cea 100644 --- a/docs/research/issue-265.md +++ b/docs/research/issue-265.md @@ -39,10 +39,17 @@ Run a reduced, explicitly labeled machinery check: ```bash adaptive-rl benchmark adaptation \ --config configs/drone_distribution_shift.yaml \ + --stochastic \ --smoke \ --output-dir artifacts/issue265_smoke_ppo ``` +PPO adaptation requires stochastic behavior-policy actions: deterministic mean +actions do not form valid on-policy PPO rollout data. The runner rejects +deterministic PPO collection before training. This machinery check does not +resolve the separate conflict between that requirement and Issue #271's frozen +deterministic-evaluation rule; it is not a valid run of the preregistered study. + Run the smoke path for SAC: ```bash @@ -53,12 +60,18 @@ adaptive-rl benchmark adaptation \ --output-dir artifacts/issue265_smoke_sac ``` -Run the complete ten-seed PPO experiment (the same command accepts `--algorithm -sac` for SAC): +For an Issue #265 PPO run, explicitly select stochastic evaluation actions (the +same command accepts `--algorithm sac` for SAC): + +The stochastic PPO setting conflicts with Issue #271's frozen deterministic +evaluation rule. Treat this command as Issue #265 machinery/diagnostic output; +do not report it as a valid preregistered-study result without a prospective +protocol amendment. ```bash adaptive-rl benchmark adaptation \ --config configs/drone_distribution_shift.yaml \ + --stochastic \ --output-dir artifacts/issue265_ppo ``` diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md index 9180c74..4500bd8 100644 --- a/docs/research/issue-271.md +++ b/docs/research/issue-271.md @@ -112,9 +112,12 @@ mean action with its Gaussian density recorded as `behavior_log_prob` does not have that sampling distribution, so the stored rollout is not a valid on-policy PPO sample. The current preregistration does not define an action-sampling rule that resolves this conflict. No treatment or analysis change is made here; the -prereg-v1 runner now rejects the frozen deterministic PPO configuration before -training. The PPO treatment must not be described as scientifically validated -until a prospective protocol amendment resolves the action-selection contract. +adaptation runner now rejects deterministic PPO collection before training, +including the frozen prereg-v1 configuration. Issue #265's general PPO CLI can +run a machinery check with explicit stochastic collection, but that does not +resolve the conflict with this study's deterministic-evaluation rule. The PPO +treatment must not be described as scientifically validated until a prospective +protocol amendment resolves the action-selection contract. This limitation does not change the previously recorded artifact or its descriptive statistics; its `COMPLETE` status describes harness execution, not valid on-policy PPO evidence. diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py index 1425bcb..093c776 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -657,8 +657,8 @@ def run_adaptation_benchmark( raise ValueError("prereg-v1 Treatment Card differs from the frozen treatment") if config.evaluation.deterministic: raise ValueError( - "prereg-v1 cannot run native PPO adaptation with deterministic mean actions; " - "the recorded actions are not sampled from the behavior distribution" + "PPO adaptation requires stochastic behavior-policy action sampling; " + "deterministic mean actions are not valid on-policy rollout data" ) try: dirty = subprocess.check_output( @@ -673,6 +673,14 @@ def run_adaptation_benchmark( determinism = _enable_study_determinism() else: determinism = None + if ( + config.algorithm.name.strip().lower() == "ppo" + and config.evaluation.deterministic + ): + raise ValueError( + "PPO adaptation requires stochastic behavior-policy action sampling; " + "deterministic mean actions are not valid on-policy rollout data" + ) config_file = Path(config_path).resolve() if config_path is not None else None config_file_sha = _sha256_file(config_file) if config_file is not None else None diff --git a/src/adaptive_rl/cli.py b/src/adaptive_rl/cli.py index fe08900..d46fd60 100644 --- a/src/adaptive_rl/cli.py +++ b/src/adaptive_rl/cli.py @@ -412,7 +412,9 @@ def benchmark_adaptation( help="Reuse only complete hashed replicate checkpoints for an unfinished study run", ), deterministic: Optional[bool] = typer.Option( - None, "--deterministic/--stochastic", help="Override action selection for all evaluations" + None, + "--deterministic/--stochastic", + help="Override evaluation action selection; PPO adaptation requires stochastic actions", ), smoke: bool = typer.Option( False, diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py index 71b426f..9d54db7 100644 --- a/tests/test_adaptation_smoke.py +++ b/tests/test_adaptation_smoke.py @@ -23,6 +23,7 @@ def test_cli_adaptation_smoke_runs_complete_protocol_and_writes_artifacts(tmp_pa "benchmark", "adaptation", "--smoke", + "--stochastic", "--output-dir", str(output_dir), ], @@ -55,6 +56,19 @@ def test_cli_adaptation_smoke_runs_complete_protocol_and_writes_artifacts(tmp_pa assert (output_dir / "adaptation.csv").is_file() +def test_cli_adaptation_rejects_deterministic_ppo_before_creating_outputs( + tmp_path: Path, +) -> None: + output_dir = tmp_path / "invalid-deterministic-ppo" + result = CliRunner().invoke( + app, + ["benchmark", "adaptation", "--smoke", "--output-dir", str(output_dir)], + ) + assert result.exit_code == 1 + assert "not valid on-policy rollout data" in result.output + assert not output_dir.exists() + + def test_preregistered_study_rejects_subset_before_training(tmp_path: Path) -> None: result = CliRunner().invoke( app, @@ -88,7 +102,7 @@ def test_preregistered_study_rejects_changed_scientific_config_before_training() def test_preregistered_study_rejects_deterministic_ppo_rollout_before_training() -> None: - with pytest.raises(ValueError, match="not sampled from the behavior distribution"): + with pytest.raises(ValueError, match="not valid on-policy rollout data"): run_adaptation_benchmark( load_config("configs/drone_distribution_shift.yaml"), study_run_id="deterministic-ppo-rejection", @@ -96,6 +110,19 @@ def test_preregistered_study_rejects_deterministic_ppo_rollout_before_training() ) +def test_issue265_rejects_deterministic_ppo_rollout_before_creating_outputs( + tmp_path: Path, +) -> None: + output_dir = tmp_path / "must-not-be-created" + with pytest.raises(ValueError, match="not valid on-policy rollout data"): + run_adaptation_benchmark( + load_config("configs/drone_distribution_shift.yaml"), + output_dir=output_dir, + training_seeds=TRAINING_SEEDS[:1], + ) + assert not output_dir.exists() + + @pytest.mark.parametrize( ("kwargs", "message"), [ From dd7be499b772cbd10bf94be862cffd5c3d1817ac Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 06:45:38 +0530 Subject: [PATCH 6/7] fix(experiment): fail closed on corrupt resumes --- docs/research/issue-271.md | 9 +- .../benchmarking/adaptation_artifacts.py | 12 +- .../benchmarking/adaptation_runner.py | 128 +++++++++++------- src/adaptive_rl/cli.py | 10 +- tests/test_adaptation_artifacts.py | 19 +++ tests/test_adaptation_smoke.py | 97 ++++++++++++- 6 files changed, 223 insertions(+), 52 deletions(-) diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md index 4500bd8..7964b4c 100644 --- a/docs/research/issue-271.md +++ b/docs/research/issue-271.md @@ -32,10 +32,15 @@ frozen here. `--resume` recomputes that identity and accepts only terminal per-replicate records whose digest, study hash, protocol hash, and seed identity verify; it never trusts partial training directories. An interrupted seed without a complete checkpoint is recorded as failed, and remaining unstarted -seeds continue. Repeating resume after finalization validates and returns the -same immutable result. The JSON stores +seeds continue. A present but malformed, stale, or mismatched checkpoint aborts +resume rather than becoming a failed replicate. Final manifest validation also +rejects files added to the completed run directory after its artifact set was +recorded. Repeating resume after finalization validates and returns the same +immutable result. The JSON stores the raw trajectories, protocol analysis, seed schedule, outcomes, runtime invariants, and run status. +The runner returns that same persisted JSON on initial completion and repeated +resume; its artifact paths are stable and relative to the run directory. The CSV has one row per replicate and arm, with finite-horizon `T_H`, status, per-episode return vectors, and seed vectors. `manifest.json` checksums every file in the run directory, including the pre-execution study manifest and diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py index bcae90d..bcc7fcc 100644 --- a/src/adaptive_rl/benchmarking/adaptation_artifacts.py +++ b/src/adaptive_rl/benchmarking/adaptation_artifacts.py @@ -414,6 +414,7 @@ def validate_study_manifest(manifest_path: str | Path) -> None: artifacts = manifest.get("artifacts") if not isinstance(artifacts, dict) or not artifacts: raise ValueError("manifest must list at least one artifact checksum") + root = manifest_path.parent.resolve() for relative_path, expected in artifacts.items(): if ( not isinstance(relative_path, str) @@ -427,7 +428,7 @@ def validate_study_manifest(manifest_path: str | Path) -> None: raise ValueError(f"manifest artifact path escapes the run directory: {relative_path}") path = manifest_path.parent / relative try: - path.resolve().relative_to(manifest_path.parent.resolve()) + path.resolve().relative_to(root) except (OSError, ValueError) as exc: raise ValueError( f"manifest artifact path escapes the run directory: {relative_path}" @@ -437,6 +438,15 @@ def validate_study_manifest(manifest_path: str | Path) -> None: actual = sha256_file(path) if actual != expected: raise ValueError(f"manifest checksum mismatch: {relative_path}") + actual_artifacts = { + path.relative_to(manifest_path.parent).as_posix() + for path in manifest_path.parent.rglob("*") + if path != manifest_path and (path.is_file() or path.is_symlink()) + } + unlisted_artifacts = actual_artifacts - set(artifacts) + if unlisted_artifacts: + names = ", ".join(sorted(unlisted_artifacts)) + raise ValueError(f"run directory contains unlisted artifacts: {names}") if "study_hash" in manifest: spec_path = manifest_path.parent / "study_manifest.json" if not spec_path.is_file() or "study_manifest.json" not in artifacts: diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py index 093c776..54b8d4a 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -398,6 +398,59 @@ def _restore_replicate(data: dict[str, Any]) -> ReplicateResult: return ReplicateResult(**restored) +def _resume_replicate_checkpoint( + checkpoint_path: Path, + *, + study_hash: str, + protocol_hash: str, + training_seed: int, +) -> ReplicateResult: + """Restore one authenticated terminal replicate or fail the whole resume.""" + try: + checkpoint = read_replicate_checkpoint( + checkpoint_path, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seed=training_seed, + ) + return _restore_replicate(checkpoint) + except (OSError, TypeError, ValueError, KeyError) as exc: + raise ValueError( + f"resume rejected invalid checkpoint for seed {training_seed}: {exc}" + ) from exc + + +def _read_completed_study_artifact( + artifact_path: Path, + manifest_path: Path, + *, + run_id: str, + study_hash: str, +) -> dict[str, Any]: + """Return the persisted result only after validating its immutable envelope.""" + validate_study_manifest(manifest_path) + try: + artifact = json.loads(artifact_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("completed study artifact is malformed") from exc + artifact_paths = artifact.get("artifact_paths") if isinstance(artifact, dict) else None + if ( + not isinstance(artifact, dict) + or artifact.get("schema_version") != "1.0" + or artifact.get("run_id") != run_id + or artifact.get("study_hash") != study_hash + or artifact.get("run_status") not in {"COMPLETE", "PARTIAL"} + or artifact_paths + != { + "json": "adaptive_vs_fixed.json", + "csv": "adaptive_vs_fixed.csv", + "manifest": "manifest.json", + } + ): + raise ValueError("completed study artifact has an invalid schema or study identity") + return artifact + + def _run_replicate( config: ExperimentConfig, training_seed: int, @@ -655,11 +708,15 @@ def run_adaptation_benchmark( raise ValueError("prereg-v1 config differs from the frozen Issue #271 configuration") if card_sha != ISSUE271_TREATMENT_CARD_SHA256: raise ValueError("prereg-v1 Treatment Card differs from the frozen treatment") - if config.evaluation.deterministic: - raise ValueError( - "PPO adaptation requires stochastic behavior-policy action sampling; " - "deterministic mean actions are not valid on-policy rollout data" - ) + if ( + config.algorithm.name.strip().lower() == "ppo" + and config.evaluation.deterministic + ): + raise ValueError( + "PPO adaptation requires stochastic behavior-policy action sampling; " + "deterministic mean actions are not valid on-policy rollout data" + ) + if study_run_id is not None: try: dirty = subprocess.check_output( ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL @@ -670,17 +727,7 @@ def run_adaptation_benchmark( ) from exc if dirty: raise RuntimeError("prereg-v1 execution requires a clean, committed working tree") - determinism = _enable_study_determinism() - else: - determinism = None - if ( - config.algorithm.name.strip().lower() == "ppo" - and config.evaluation.deterministic - ): - raise ValueError( - "PPO adaptation requires stochastic behavior-policy action sampling; " - "deterministic mean actions are not valid on-policy rollout data" - ) + determinism = _enable_study_determinism() if study_run_id is not None else None config_file = Path(config_path).resolve() if config_path is not None else None config_file_sha = _sha256_file(config_file) if config_file is not None else None @@ -727,18 +774,12 @@ def run_adaptation_benchmark( ).hexdigest() final_paths = [target_dir / suffix for suffix in (*suffixes, "manifest.json")] if resume and all(path.is_file() for path in final_paths): - validate_study_manifest(final_paths[-1]) - try: - completed_artifact = json.loads(final_paths[0].read_text(encoding="utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ValueError("completed study artifact is malformed") from exc - if ( - not isinstance(completed_artifact, dict) - or completed_artifact.get("run_id") != study_run_id - or completed_artifact.get("study_hash") != study_hash - ): - raise ValueError("completed study artifact does not match the expected study hash") - return completed_artifact + return _read_completed_study_artifact( + final_paths[0], + final_paths[-1], + run_id=study_run_id, + study_hash=study_hash, + ) for path in final_paths: if path.exists(): raise FileExistsError( @@ -762,22 +803,15 @@ def run_adaptation_benchmark( training_dir = training_root / f"seed_{seed}" checkpoint_path = state_root / f"seed_{seed}.json" if study_run_id is not None and resume and checkpoint_path.exists(): - try: - assert study_hash is not None and protocol_hash is not None - checkpoint = read_replicate_checkpoint( + assert study_hash is not None and protocol_hash is not None + results.append( + _resume_replicate_checkpoint( checkpoint_path, study_hash=study_hash, protocol_hash=protocol_hash, training_seed=seed, ) - results.append(_restore_replicate(checkpoint)) - except (OSError, TypeError, ValueError, KeyError) as exc: - failed = ReplicateResult( - training_seed=seed, - status="failed", - failure_reason=f"untrusted replicate checkpoint: {type(exc).__name__}: {exc}", - ) - results.append(failed) + ) continue if study_run_id is not None and resume and training_dir.exists(): results.append( @@ -939,11 +973,6 @@ def outcome_count(arm: str, status: str) -> int: json_path, csv_path = write_adaptive_vs_fixed_artifacts(artifact, target_dir) else: json_path, csv_path = write_adaptation_artifacts(artifact, target_dir, stem=stem) - artifact["artifact_paths"] = { - "json": str(json_path), - "csv": str(csv_path), - **({"manifest": str(target_dir / "manifest.json")} if study_run_id else {}), - } if study_run_id is not None: output_arg = f"--output-dir {Path(output_dir)} " if output_dir is not None else "" executable = Path(sys.argv[0]) @@ -956,14 +985,21 @@ def outcome_count(arm: str, status: str) -> int: f"--config {config_arg} {output_arg}" f"--study prereg-v1 --run-id {study_run_id}" ) - manifest = write_study_manifest( + write_study_manifest( json_path, csv_path, target_dir / "manifest.json", run_id=study_run_id, command=command, ) - artifact["manifest"] = manifest + assert study_hash is not None + return _read_completed_study_artifact( + json_path, + target_dir / "manifest.json", + run_id=study_run_id, + study_hash=study_hash, + ) + artifact["artifact_paths"] = {"json": str(json_path), "csv": str(csv_path)} return artifact diff --git a/src/adaptive_rl/cli.py b/src/adaptive_rl/cli.py index d46fd60..52a2d68 100644 --- a/src/adaptive_rl/cli.py +++ b/src/adaptive_rl/cli.py @@ -490,6 +490,12 @@ def benchmark_adaptation( failed = artifact["failure_summary"]["failed_replicates"] completed = artifact["failure_summary"]["completed_replicates"] + json_artifact_path = Path(artifact["artifact_paths"]["json"]) + csv_artifact_path = Path(artifact["artifact_paths"]["csv"]) + if study is not None and run_id is not None: + study_dir = (output_dir or exp_config.output_dir) / run_id + json_artifact_path = study_dir / json_artifact_path + csv_artifact_path = study_dir / csv_artifact_path console.print( Panel.fit( f"[bold]{'Smoke check' if smoke else 'Issue #265 benchmark'} finished[/bold]\n\n" @@ -497,8 +503,8 @@ def benchmark_adaptation( f"• [bold]Algorithm:[/bold] {artifact['experiment']['algorithm']}\n" f"• [bold]Completed replicates:[/bold] {completed}\n" f"• [bold]Failed replicates:[/bold] {len(failed)}\n" - f"• [bold]JSON:[/bold] {artifact['artifact_paths']['json']}\n" - f"• [bold]CSV:[/bold] {artifact['artifact_paths']['csv']}\n" + f"• [bold]JSON:[/bold] {json_artifact_path}\n" + f"• [bold]CSV:[/bold] {csv_artifact_path}\n" f"• [bold]Scientific result:[/bold] not established by harness execution", title="Online Adaptation Benchmark", border_style="yellow" if smoke or failed else "green", diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py index 1302129..462f8f4 100644 --- a/tests/test_adaptation_artifacts.py +++ b/tests/test_adaptation_artifacts.py @@ -125,6 +125,25 @@ def test_manifest_hashes_all_artifacts_and_detects_tampering(tmp_path) -> None: validate_study_manifest(manifest_path) +def test_manifest_rejects_unlisted_files_added_after_completion(tmp_path) -> None: + json_path, csv_path = write_adaptation_artifacts( + _artifact(), tmp_path, stem="adaptive_vs_fixed" + ) + manifest_path = tmp_path / "manifest.json" + write_study_manifest( + json_path, + csv_path, + manifest_path, + run_id="test-run", + command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id test-run", + ) + validate_study_manifest(manifest_path) + + (tmp_path / "unexpected.json").write_text("{}\n", encoding="utf-8") + with pytest.raises(ValueError, match="unlisted artifacts: unexpected.json"): + validate_study_manifest(manifest_path) + + def test_completed_manifest_binds_preexecution_study_hash(tmp_path) -> None: spec_path = tmp_path / "study_manifest.json" study_hash = write_or_verify_study_manifest( diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py index 9d54db7..ac76b1a 100644 --- a/tests/test_adaptation_smoke.py +++ b/tests/test_adaptation_smoke.py @@ -8,7 +8,17 @@ import pytest from typer.testing import CliRunner -from adaptive_rl.benchmarking.adaptation_runner import run_adaptation_benchmark +from adaptive_rl.benchmarking.adaptation_artifacts import ( + write_adaptive_vs_fixed_artifacts, + write_or_verify_study_manifest, + write_replicate_checkpoint, + write_study_manifest, +) +from adaptive_rl.benchmarking.adaptation_runner import ( + _read_completed_study_artifact, + _resume_replicate_checkpoint, + run_adaptation_benchmark, +) from adaptive_rl.cli import app from adaptive_rl.config import load_config from adaptive_rl.protocol.constants import TRAINING_SEEDS @@ -69,6 +79,25 @@ def test_cli_adaptation_rejects_deterministic_ppo_before_creating_outputs( assert not output_dir.exists() +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--algorithm", "dqn"], "--algorithm must be 'ppo' or 'sac'"), + (["--resume"], "--resume requires --study prereg-v1 and --run-id"), + ( + ["--study", "prereg-v1", "--run-id", "hash-mismatch", "--stochastic"], + "frozen Issue #271 configuration", + ), + ], +) +def test_cli_adaptation_rejects_invalid_or_mismatched_requests( + args: list[str], message: str +) -> None: + result = CliRunner().invoke(app, ["benchmark", "adaptation", *args]) + assert result.exit_code == 1 + assert message in " ".join(result.output.split()) + + def test_preregistered_study_rejects_subset_before_training(tmp_path: Path) -> None: result = CliRunner().invoke( app, @@ -123,6 +152,72 @@ def test_issue265_rejects_deterministic_ppo_rollout_before_creating_outputs( assert not output_dir.exists() +def test_checkpoint_resume_helper_rejects_corrupted_terminal_record( + tmp_path: Path, +) -> None: + checkpoint_path = tmp_path / "seed_31001.json" + write_replicate_checkpoint( + {"training_seed": TRAINING_SEEDS[0], "status": "completed"}, + checkpoint_path, + study_hash="expected-study", + protocol_hash="expected-protocol", + ) + checkpoint_path.write_text("truncated", encoding="utf-8") + + with pytest.raises(ValueError, match="resume rejected invalid checkpoint"): + _resume_replicate_checkpoint( + checkpoint_path, + study_hash="expected-study", + protocol_hash="expected-protocol", + training_seed=TRAINING_SEEDS[0], + ) + + +def test_completed_study_reader_returns_stable_persisted_artifact( + tmp_path: Path, +) -> None: + study_hash = write_or_verify_study_manifest( + {"protocol": "test", "seed": TRAINING_SEEDS[0]}, + tmp_path / "study_manifest.json", + resume=False, + ) + artifact = { + "schema_version": "1.0", + "run_id": "idempotent-run", + "study_hash": study_hash, + "run_status": "COMPLETE", + "replicates": [], + "artifact_paths": { + "json": "adaptive_vs_fixed.json", + "csv": "adaptive_vs_fixed.csv", + "manifest": "manifest.json", + }, + } + json_path, csv_path = write_adaptive_vs_fixed_artifacts(artifact, tmp_path) + manifest_path = tmp_path / "manifest.json" + write_study_manifest( + json_path, + csv_path, + manifest_path, + run_id="idempotent-run", + command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id idempotent-run", + ) + + first_result = _read_completed_study_artifact( + json_path, + manifest_path, + run_id="idempotent-run", + study_hash=study_hash, + ) + resumed_result = _read_completed_study_artifact( + json_path, + manifest_path, + run_id="idempotent-run", + study_hash=study_hash, + ) + assert first_result == resumed_result == json.loads(json_path.read_text(encoding="utf-8")) + + @pytest.mark.parametrize( ("kwargs", "message"), [ From 4270cf5e58354348757fbd5bf3b72ad55f6af151 Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 07:01:09 +0530 Subject: [PATCH 7/7] fix(experiment): validate resumed replicate artifacts --- docs/research/issue-271.md | 4 +- .../benchmarking/adaptation_artifacts.py | 157 +++++++++++++++++- .../benchmarking/adaptation_runner.py | 73 ++++++-- tests/test_adaptation_artifacts.py | 71 ++++++++ tests/test_adaptation_smoke.py | 17 ++ 5 files changed, 299 insertions(+), 23 deletions(-) diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md index 7964b4c..6008698 100644 --- a/docs/research/issue-271.md +++ b/docs/research/issue-271.md @@ -35,7 +35,9 @@ without a complete checkpoint is recorded as failed, and remaining unstarted seeds continue. A present but malformed, stale, or mismatched checkpoint aborts resume rather than becoming a failed replicate. Final manifest validation also rejects files added to the completed run directory after its artifact set was -recorded. Repeating resume after finalization validates and returns the same +recorded. Replicate checkpoints bind the training files they rely on, and the +runner validates all checkpoint state before resuming any unfinished seed. +Repeating resume after finalization validates and returns the same immutable result. The JSON stores the raw trajectories, protocol analysis, seed schedule, outcomes, runtime invariants, and run status. diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py index bcc7fcc..2a9b386 100644 --- a/src/adaptive_rl/benchmarking/adaptation_artifacts.py +++ b/src/adaptive_rl/benchmarking/adaptation_artifacts.py @@ -12,10 +12,14 @@ import subprocess import tempfile from pathlib import Path -from typing import Any, Iterable, Mapping, cast +from typing import Any, Iterable, Mapping, Sequence, cast import numpy as np +STUDY_MANIFEST_SCHEMA_VERSION = "1.0" +STUDY_ARTIFACT_SCHEMA_VERSION = "1.0" +REPLICATE_CHECKPOINT_SCHEMA_VERSION = "1.1" + CSV_FIELDS = ( "replicate_index", "training_seed", @@ -274,7 +278,7 @@ def make_study_manifest(inputs: Mapping[str, Any]) -> dict[str, Any]: if not isinstance(plain_inputs, dict): raise TypeError("study manifest inputs must be a mapping") return { - "schema_version": "1.0", + "schema_version": STUDY_MANIFEST_SCHEMA_VERSION, "study_hash": hashlib.sha256(canonical_json_bytes(plain_inputs)).hexdigest(), "inputs": plain_inputs, } @@ -293,7 +297,7 @@ def write_or_verify_study_manifest( raise ValueError("cannot resume without a valid immutable study manifest") from exc if ( not isinstance(existing, dict) - or existing.get("schema_version") != "1.0" + or existing.get("schema_version") != STUDY_MANIFEST_SCHEMA_VERSION or not isinstance(existing.get("inputs"), dict) ): raise ValueError("study manifest has an invalid structure") @@ -351,8 +355,14 @@ def write_study_manifest( diff_hash = hashlib.sha256(diff).hexdigest() except (OSError, subprocess.CalledProcessError): commit, dirty, diff_hash = None, None, None + artifact_files: list[Path] = [] + for path in sorted(manifest_path.parent.rglob("*")): + if path.is_symlink(): + raise ValueError(f"run directory contains a symlinked artifact: {path.name}") + if path.is_file() and path != manifest_path: + artifact_files.append(path) manifest = { - "schema_version": "1.0", + "schema_version": STUDY_MANIFEST_SCHEMA_VERSION, "study": "adaptive-vs-fixed/prereg-v1", "run_id": run_id, "commit_sha": commit, @@ -381,8 +391,7 @@ def write_study_manifest( }, "artifacts": { str(path.relative_to(manifest_path.parent).as_posix()): sha256_file(path) - for path in sorted(manifest_path.parent.rglob("*")) - if path.is_file() and path != manifest_path + for path in artifact_files }, } study_spec_path = manifest_path.parent / "study_manifest.json" @@ -408,8 +417,13 @@ def write_study_manifest( def validate_study_manifest(manifest_path: str | Path) -> None: """Raise when a listed immutable run artifact is missing or has changed.""" manifest_path = Path(manifest_path) + if manifest_path.is_symlink(): + raise ValueError("study artifact manifest cannot be a symlink") manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if not isinstance(manifest, dict) or manifest.get("schema_version") != "1.0": + if ( + not isinstance(manifest, dict) + or manifest.get("schema_version") != STUDY_MANIFEST_SCHEMA_VERSION + ): raise ValueError("unsupported or malformed study artifact manifest") artifacts = manifest.get("artifacts") if not isinstance(artifacts, dict) or not artifacts: @@ -433,6 +447,8 @@ def validate_study_manifest(manifest_path: str | Path) -> None: raise ValueError( f"manifest artifact path escapes the run directory: {relative_path}" ) from exc + if path.is_symlink(): + raise ValueError(f"manifest artifact is a symlink: {relative_path}") if not path.is_file(): raise ValueError(f"manifest artifact is missing: {relative_path}") actual = sha256_file(path) @@ -465,17 +481,27 @@ def write_replicate_checkpoint( *, study_hash: str, protocol_hash: str, + artifact_root: str | Path | None = None, + artifact_directories: Sequence[str | Path] = (), ) -> None: """Persist a terminal replicate bound to its study and replicate identity.""" checkpoint_path = Path(checkpoint_path) if replicate.get("status") not in {"completed", "failed"}: raise ValueError("only terminal replicate states may be checkpointed") + if artifact_directories and artifact_root is None: + raise ValueError("artifact_root is required when checkpoint artifacts are declared") plain = _plain(dict(replicate)) + artifact_integrity = ( + _snapshot_artifact_directories(artifact_root, artifact_directories) + if artifact_root is not None + else [] + ) envelope = { - "schema_version": "1.0", + "schema_version": REPLICATE_CHECKPOINT_SCHEMA_VERSION, "study_hash": study_hash, "protocol_hash": protocol_hash, "replicate_id": plain.get("training_seed"), + "artifact_integrity": artifact_integrity, "replicate": plain, } encoded = (json.dumps(envelope, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( @@ -534,7 +560,7 @@ def read_replicate_checkpoint( envelope = json.loads(payload) if ( not isinstance(envelope, dict) - or envelope.get("schema_version") != "1.0" + or envelope.get("schema_version") != REPLICATE_CHECKPOINT_SCHEMA_VERSION or not isinstance(envelope.get("replicate"), dict) ): raise ValueError("replicate checkpoint envelope is malformed") @@ -543,6 +569,10 @@ def read_replicate_checkpoint( if envelope.get("protocol_hash") != protocol_hash: raise ValueError("replicate checkpoint protocol hash mismatch") replicate = envelope["replicate"] + _validate_checkpoint_artifacts( + checkpoint_path.parent.parent, + envelope.get("artifact_integrity"), + ) if envelope.get("replicate_id") != training_seed or replicate.get("training_seed") != training_seed: raise ValueError("replicate checkpoint identity mismatch") result = replicate @@ -551,6 +581,115 @@ def read_replicate_checkpoint( return cast(dict[str, Any], result) +def _snapshot_artifact_directories( + artifact_root: str | Path, + artifact_directories: Sequence[str | Path], +) -> list[dict[str, Any]]: + root = Path(artifact_root).resolve() + snapshots: list[dict[str, Any]] = [] + seen_directories: set[str] = set() + for directory_value in artifact_directories: + directory = Path(directory_value) + if not directory.is_absolute(): + directory = root / directory + if directory.is_symlink(): + raise ValueError(f"checkpoint artifact directory is a symlink: {directory}") + resolved = directory.resolve() + try: + relative_directory = resolved.relative_to(root).as_posix() + except ValueError as exc: + raise ValueError(f"checkpoint artifact directory escapes its run: {directory}") from exc + if relative_directory in seen_directories: + raise ValueError(f"duplicate checkpoint artifact directory: {relative_directory}") + seen_directories.add(relative_directory) + exists = directory.exists() + if exists and not directory.is_dir(): + raise ValueError(f"checkpoint artifact path is not a directory: {directory}") + files: dict[str, str] = {} + if exists: + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError(f"checkpoint artifact contains a symlink: {path}") + if path.is_file(): + relative_path = path.resolve().relative_to(root).as_posix() + files[relative_path] = sha256_file(path) + snapshots.append( + {"path": relative_directory, "exists": exists, "files": files} + ) + return snapshots + + +def _validate_checkpoint_artifacts(artifact_root: str | Path, snapshots: Any) -> None: + if not isinstance(snapshots, list): + raise ValueError("replicate checkpoint artifact integrity is malformed") + root = Path(artifact_root).resolve() + seen_directories: set[str] = set() + for snapshot in snapshots: + if not isinstance(snapshot, dict): + raise ValueError("replicate checkpoint artifact directory record is malformed") + relative_directory = snapshot.get("path") + exists = snapshot.get("exists") + files = snapshot.get("files") + if ( + not isinstance(relative_directory, str) + or not relative_directory + or Path(relative_directory).is_absolute() + or ".." in Path(relative_directory).parts + or not isinstance(exists, bool) + or not isinstance(files, dict) + or relative_directory in seen_directories + ): + raise ValueError("replicate checkpoint artifact directory record is invalid") + seen_directories.add(relative_directory) + directory = root / relative_directory + try: + directory.resolve().relative_to(root) + except (OSError, ValueError) as exc: + raise ValueError("replicate checkpoint artifact path escapes its run") from exc + if not exists: + if directory.exists() or directory.is_symlink(): + raise ValueError("checkpoint artifact directory appeared after checkpointing") + if files: + raise ValueError("absent checkpoint artifact directory has recorded files") + continue + if directory.is_symlink() or not directory.is_dir(): + raise ValueError("checkpoint artifact directory is missing or changed type") + expected_files: set[str] = set() + for relative_path, expected_hash in files.items(): + relative = Path(relative_path) if isinstance(relative_path, str) else Path("/") + if ( + not isinstance(relative_path, str) + or relative.is_absolute() + or ".." in relative.parts + or not isinstance(expected_hash, str) + or len(expected_hash) != 64 + or any(character not in "0123456789abcdef" for character in expected_hash) + ): + raise ValueError("checkpoint artifact file record is invalid") + if relative.parts[: len(Path(relative_directory).parts)] != Path( + relative_directory + ).parts: + raise ValueError("checkpoint artifact file is outside its recorded directory") + path = root / relative + try: + path.resolve().relative_to(root) + except (OSError, ValueError) as exc: + raise ValueError("checkpoint artifact file escapes its run") from exc + if path.is_symlink() or not path.is_file(): + raise ValueError(f"checkpoint artifact file is missing or changed: {relative_path}") + if sha256_file(path) != expected_hash: + raise ValueError(f"checkpoint artifact checksum mismatch: {relative_path}") + expected_files.add(relative_path) + actual_files: set[str] = set() + for path in directory.rglob("*"): + if path.is_symlink(): + raise ValueError(f"checkpoint artifact directory contains a symlink: {path}") + if path.is_file(): + actual_files.add(path.resolve().relative_to(root).as_posix()) + if actual_files != expected_files: + raise ValueError("checkpoint artifact file set changed after checkpointing") + + def _distribution_version(name: str) -> str | None: try: return importlib.metadata.version(name) diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py index 54b8d4a..27cf2b8 100644 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ b/src/adaptive_rl/benchmarking/adaptation_runner.py @@ -25,6 +25,9 @@ run_adaptation_update, ) from adaptive_rl.benchmarking.adaptation_artifacts import ( + REPLICATE_CHECKPOINT_SCHEMA_VERSION, + STUDY_ARTIFACT_SCHEMA_VERSION, + STUDY_MANIFEST_SCHEMA_VERSION, canonical_json_bytes, read_replicate_checkpoint, validate_study_manifest, @@ -420,6 +423,45 @@ def _resume_replicate_checkpoint( ) from exc +def _load_resume_replicates( + state_root: Path, + *, + study_hash: str, + protocol_hash: str, + training_seeds: Sequence[int], +) -> dict[int, ReplicateResult]: + """Validate every stored checkpoint before resuming any unfinished seed.""" + if state_root.is_symlink(): + raise ValueError("resume replicate state path is not a regular directory") + if not state_root.exists(): + return {} + if not state_root.is_dir(): + raise ValueError("resume replicate state path is not a regular directory") + allowed_names = { + name + for seed in training_seeds + for name in (f"seed_{seed}.json", f"seed_{seed}.json.sha256") + } + for path in state_root.iterdir(): + if path.name not in allowed_names or path.is_symlink() or not path.is_file(): + raise ValueError(f"resume replicate state contains an unexpected entry: {path.name}") + + restored: dict[int, ReplicateResult] = {} + for seed in training_seeds: + checkpoint_path = state_root / f"seed_{seed}.json" + digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") + if checkpoint_path.exists() != digest_path.exists(): + raise ValueError(f"resume checkpoint or digest is missing for seed {seed}") + if checkpoint_path.exists(): + restored[seed] = _resume_replicate_checkpoint( + checkpoint_path, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seed=seed, + ) + return restored + + def _read_completed_study_artifact( artifact_path: Path, manifest_path: Path, @@ -436,7 +478,7 @@ def _read_completed_study_artifact( artifact_paths = artifact.get("artifact_paths") if isinstance(artifact, dict) else None if ( not isinstance(artifact, dict) - or artifact.get("schema_version") != "1.0" + or artifact.get("schema_version") != STUDY_ARTIFACT_SCHEMA_VERSION or artifact.get("run_id") != run_id or artifact.get("study_hash") != study_hash or artifact.get("run_status") not in {"COMPLETE", "PARTIAL"} @@ -744,6 +786,9 @@ def run_adaptation_benchmark( runtime_identity = _repository_metadata() study_inputs = { "study": "adaptive-vs-fixed/prereg-v1", + "artifact_schema_version": STUDY_ARTIFACT_SCHEMA_VERSION, + "manifest_schema_version": STUDY_MANIFEST_SCHEMA_VERSION, + "replicate_checkpoint_schema_version": REPLICATE_CHECKPOINT_SCHEMA_VERSION, "study_config": config.model_dump( mode="json", exclude={"output_dir", "log_dir"} ), @@ -793,6 +838,15 @@ def run_adaptation_benchmark( state_root = target_dir / "replicate_state" if study_run_id is not None and state_root.exists() and not resume: raise FileExistsError("run state already exists; pass --resume to use hashed replicates") + resumed_replicates: dict[int, ReplicateResult] = {} + if study_run_id is not None and resume: + assert study_hash is not None and protocol_hash is not None + resumed_replicates = _load_resume_replicates( + state_root, + study_hash=study_hash, + protocol_hash=protocol_hash, + training_seeds=selected_seeds, + ) if study_run_id is None: for seed in selected_seeds: if (training_root / f"seed_{seed}").exists(): @@ -801,17 +855,8 @@ def run_adaptation_benchmark( results: list[ReplicateResult] = [] for seed in selected_seeds: training_dir = training_root / f"seed_{seed}" - checkpoint_path = state_root / f"seed_{seed}.json" - if study_run_id is not None and resume and checkpoint_path.exists(): - assert study_hash is not None and protocol_hash is not None - results.append( - _resume_replicate_checkpoint( - checkpoint_path, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seed=seed, - ) - ) + if seed in resumed_replicates: + results.append(resumed_replicates[seed]) continue if study_run_id is not None and resume and training_dir.exists(): results.append( @@ -842,6 +887,8 @@ def run_adaptation_benchmark( state_root / f"seed_{seed}.json", study_hash=study_hash, protocol_hash=protocol_hash, + artifact_root=target_dir, + artifact_directories=(training_dir,), ) vectors: dict[str, tuple[list[Optional[float]], list[Optional[float]]]] = {} @@ -900,7 +947,7 @@ def outcome_count(arm: str, status: str) -> int: for arm in ("adaptive", "fixed") } artifact: dict[str, Any] = { - "schema_version": "1.0", + "schema_version": STUDY_ARTIFACT_SCHEMA_VERSION, "run_id": study_run_id, "study_hash": study_hash, "run_status": run_status if study_run_id is not None else None, diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py index 462f8f4..028483c 100644 --- a/tests/test_adaptation_artifacts.py +++ b/tests/test_adaptation_artifacts.py @@ -208,6 +208,25 @@ def test_manifest_rejects_symlinked_artifact_outside_run_directory(tmp_path) -> validate_study_manifest(manifest_path) +def test_manifest_rejects_internal_symlinked_artifact(tmp_path) -> None: + target = tmp_path / "target.bin" + target.write_bytes(b"bound-bytes") + link = tmp_path / "alias.bin" + link.symlink_to(target) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "artifacts": {"alias.bin": sha256_file(target)}, + } + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="artifact is a symlink"): + validate_study_manifest(manifest_path) + + def test_study_csv_has_one_row_per_arm_and_preserves_finite_censoring(tmp_path) -> None: data = { "replicates": [ @@ -291,6 +310,47 @@ def test_replicate_checkpoint_is_terminal_hashed_and_tamper_evident(tmp_path) -> ) +def test_replicate_checkpoint_binds_training_artifact_files(tmp_path) -> None: + run_root = tmp_path / "run" + training_dir = run_root / "training" / "seed_31001" + training_dir.mkdir(parents=True) + model_path = training_dir / "model.zip" + model_path.write_bytes(b"frozen-model") + checkpoint_path = run_root / "replicate_state" / "seed_31001.json" + write_replicate_checkpoint( + {"training_seed": 31001, "status": "completed"}, + checkpoint_path, + study_hash="study-hash", + protocol_hash="protocol-hash", + artifact_root=run_root, + artifact_directories=(training_dir,), + ) + + read_replicate_checkpoint( + checkpoint_path, + study_hash="study-hash", + protocol_hash="protocol-hash", + training_seed=31001, + ) + model_path.write_bytes(b"changed-model") + with pytest.raises(ValueError, match="artifact checksum mismatch"): + read_replicate_checkpoint( + checkpoint_path, + study_hash="study-hash", + protocol_hash="protocol-hash", + training_seed=31001, + ) + model_path.write_bytes(b"frozen-model") + (training_dir / "unexpected.json").write_text("{}\n", encoding="utf-8") + with pytest.raises(ValueError, match="artifact file set changed"): + read_replicate_checkpoint( + checkpoint_path, + study_hash="study-hash", + protocol_hash="protocol-hash", + training_seed=31001, + ) + + def test_canonical_study_hash_ignores_mapping_order_and_json_formatting(tmp_path) -> None: left = {"config": {"seed": 31001, "lr": 0.001}, "schedule": [1, 2, 3]} right = {"schedule": [1, 2, 3], "config": {"lr": 0.001, "seed": 31001}} @@ -303,6 +363,17 @@ def test_canonical_study_hash_ignores_mapping_order_and_json_formatting(tmp_path assert write_or_verify_study_manifest(right, path, resume=True) == digest +def test_study_hash_binds_artifact_schema_versions() -> None: + base = { + "protocol": "prereg-v1", + "artifact_schema_version": "1.0", + "manifest_schema_version": "1.0", + "replicate_checkpoint_schema_version": "1.1", + } + changed = {**base, "artifact_schema_version": "2.0"} + assert make_study_manifest(base)["study_hash"] != make_study_manifest(changed)["study_hash"] + + def test_study_manifest_material_change_and_tampered_hash_fail_resume(tmp_path) -> None: path = tmp_path / "study_manifest.json" inputs = {"config": {"gamma": 0.99}, "seeds": [1, 2]} diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py index ac76b1a..0bdfee7 100644 --- a/tests/test_adaptation_smoke.py +++ b/tests/test_adaptation_smoke.py @@ -15,6 +15,7 @@ write_study_manifest, ) from adaptive_rl.benchmarking.adaptation_runner import ( + _load_resume_replicates, _read_completed_study_artifact, _resume_replicate_checkpoint, run_adaptation_benchmark, @@ -173,6 +174,22 @@ def test_checkpoint_resume_helper_rejects_corrupted_terminal_record( ) +def test_resume_preflight_rejects_orphan_checkpoint_digest_before_training( + tmp_path: Path, +) -> None: + state_root = tmp_path / "replicate_state" + state_root.mkdir() + (state_root / "seed_31001.json.sha256").write_text("0" * 64, encoding="ascii") + + with pytest.raises(ValueError, match="checkpoint or digest is missing for seed 31001"): + _load_resume_replicates( + state_root, + study_hash="study", + protocol_hash="protocol", + training_seeds=(TRAINING_SEEDS[0], TRAINING_SEEDS[1]), + ) + + def test_completed_study_reader_returns_stable_persisted_artifact( tmp_path: Path, ) -> None: