From 59982616d4a2fa2eb4618d877f94d695ff5296fe Mon Sep 17 00:00:00 2001 From: Aryan Date: Tue, 29 Sep 2026 19:06:26 +0530 Subject: [PATCH] Revert "Feat/issue 271 preregistered study" --- configs/drone_distribution_shift.yaml | 28 - docs/research/TREATMENT_CARD.md | 69 -- docs/research/adaptive_rl_hypothesis.md | 53 +- docs/research/issue-265.md | 105 -- docs/research/issue-271.md | 187 --- src/adaptive_rl/algorithms/__init__.py | 10 - src/adaptive_rl/algorithms/adaptation.py | 353 ------ src/adaptive_rl/benchmarking/__init__.py | 10 - .../benchmarking/adaptation_artifacts.py | 764 ------------ .../benchmarking/adaptation_runner.py | 1053 ----------------- .../benchmarking/adaptation_runtime.py | 246 ---- .../benchmarking/adaptation_statistics.py | 147 --- src/adaptive_rl/cli.py | 131 -- src/adaptive_rl/config.py | 27 - src/adaptive_rl/environments/__init__.py | 13 - src/adaptive_rl/environments/drone.py | 60 +- src/adaptive_rl/protocol/__init__.py | 16 - src/adaptive_rl/protocol/adaptation.py | 163 --- src/adaptive_rl/protocol/constants.py | 10 - src/adaptive_rl/protocol/fork.py | 121 -- src/adaptive_rl/training/trainer.py | 27 - tests/test_adaptation_artifacts.py | 405 ------- tests/test_adaptation_environment.py | 67 -- tests/test_adaptation_invariants.py | 99 -- tests/test_adaptation_smoke.py | 257 ---- tests/test_adaptation_statistics.py | 55 - tests/test_algorithm_adaptation.py | 439 ------- tests/test_protocol_adaptation_boundary.py | 86 -- tests/test_protocol_fork.py | 63 - 29 files changed, 28 insertions(+), 5036 deletions(-) delete mode 100644 docs/research/issue-265.md delete mode 100644 docs/research/issue-271.md delete mode 100644 src/adaptive_rl/algorithms/adaptation.py delete mode 100644 src/adaptive_rl/benchmarking/adaptation_artifacts.py delete mode 100644 src/adaptive_rl/benchmarking/adaptation_runner.py delete mode 100644 src/adaptive_rl/benchmarking/adaptation_runtime.py delete mode 100644 src/adaptive_rl/benchmarking/adaptation_statistics.py delete mode 100644 src/adaptive_rl/protocol/adaptation.py delete mode 100644 src/adaptive_rl/protocol/fork.py delete mode 100644 tests/test_adaptation_artifacts.py delete mode 100644 tests/test_adaptation_environment.py delete mode 100644 tests/test_adaptation_invariants.py delete mode 100644 tests/test_adaptation_smoke.py delete mode 100644 tests/test_adaptation_statistics.py delete mode 100644 tests/test_algorithm_adaptation.py delete mode 100644 tests/test_protocol_adaptation_boundary.py delete mode 100644 tests/test_protocol_fork.py diff --git a/configs/drone_distribution_shift.yaml b/configs/drone_distribution_shift.yaml index 59474a6..69db635 100644 --- a/configs/drone_distribution_shift.yaml +++ b/configs/drone_distribution_shift.yaml @@ -1,8 +1,3 @@ -# Issue #265: primary moderate TEST-B condition. Training receives only the -# nominal environment.parameters block. shift_parameters are held back until -# after training and the shared pre-shift evaluation. -name: "drone_online_adaptation" -seed: 31001 # Issue #105 / Issue #265: controlled distribution-shift benchmark for drone navigation. # # Protocol: the policy is trained ONLY on TRAIN conditions (8 static obstacles, @@ -37,7 +32,6 @@ algorithm: batch_size: 128 parameters: n_steps: 1024 - n_epochs: 10 clip_range: 0.2 ent_coef: 0.01 @@ -46,18 +40,6 @@ environment: max_steps: 300 parameters: bounds: [50.0, 50.0, 25.0] - num_obstacles: 8 - num_lidar_rays: 16 - linear_damping: 0.05 - max_acceleration: 4.0 - wind_speed: 0.5 - gust_sigma: 0.15 - gust_theta: 0.15 - wind_direction: [1.5, 0.5, 0.0] - -training: - total_timesteps: 60000 - checkpoint_freq: 0 dynamic_obstacle_speed: 1.5 training: @@ -69,16 +51,6 @@ evaluation: eval_episodes: 15 deterministic: true -adaptation_benchmark: - protocol_version: "2.0" - scenario: "TEST-B" - shift_parameters: - num_obstacles: 12 - wind_speed: 4.0 - gust_sigma: 0.6 - -output_dir: "artifacts/adaptation" -log_dir: "artifacts/adaptation/logs" output_dir: "artifacts" log_dir: "artifacts/logs" diff --git a/docs/research/TREATMENT_CARD.md b/docs/research/TREATMENT_CARD.md index a938773..256e0bc 100644 --- a/docs/research/TREATMENT_CARD.md +++ b/docs/research/TREATMENT_CARD.md @@ -1,72 +1,3 @@ -# Issue #265 Treatment Card - -**Status: frozen before benchmark execution.** This card defines the adaptation -treatment. Its SHA-256 is stored in every Issue #265 artifact. No benchmark -outcome may be used to change these choices. - -## Fixed treatment choices - -- **Scope:** adapt the already trained PPO or SAC policy using only transitions - collected in completed post-shift episodes. Base training configuration, - checkpoint, and environment interaction schedule are identical between arms. -- **Mechanism:** one cumulative, episode-bounded batch update at each boundary - B5 through B14. Each batch is the concatenation of all post-shift transitions - from episodes 1 through the boundary episode. The batch contains observation, - action, reward, next observation, terminated, and truncated values. No nominal - data is retained in or passed to the adaptation path. -- **PPO update:** store the behavior-policy log-probability and value estimate - with each transition at collection time, plus the behavior value of a - truncated transition's next observation. At each block, construct a fresh - Stable-Baselines3 `RolloutBuffer` from the permitted cumulative prefix, use - the configured `gamma` and `gae_lambda`, bootstrap truncated transitions - from their recorded next observation, and run the native clipped PPO update - for the config's `n_epochs` and `batch_size`. The configured entropy and - value-function coefficients are unchanged. Previously seen prefix data is - reused at later blocks with its stored behavior quantities. -- **SAC update:** construct a fresh Stable-Baselines3 `ReplayBuffer` for each - block from the permitted cumulative prefix; the training replay buffer is - never reused. Its capacity is `max(prefix_transition_count, configured - batch_size)`. Run native SAC training for the configured `gradient_steps` - and `batch_size`, retaining the configured learning rate, target update, and - entropy mechanism. A sampled minibatch may draw with replacement when it is - larger than the currently available transition count. Time-limit truncations - are marked according to SB3's timeout-mask convention. -- **Randomness:** each block seeds Python, NumPy, and Torch with its one - preregistered `update` seed while sampling/updating, then restores the caller's - RNG states. No environment step occurs in either adapter. -- **Learning rate, batch size, epochs/steps, and regularization:** all values - come from the base cell config; no additional optimizer or regularizer is - added. PPO's configured entropy/value terms and SAC's configured entropy - mechanism remain part of their respective native objectives. -- **Minimum data:** at least one complete episode (the current block's newly - completed episode) and at least one valid transition are required. -- **Buffer construction:** a fresh immutable snapshot is assembled at each - block boundary from the allowed prefix only. No shared/global replay buffer - is used. Transitions are never sampled from a future episode. -- **Update timing:** only after episode termination/truncation and before the - next reset; no updates at B15. The policy is constant during every episode. -- **Failure behavior:** any exception, non-finite loss/parameter/optimizer - value, invalid tensor, or shape change invalidates the replicate. The block is - rolled back before the failed replicate is recorded; execution does not - continue to another evaluation episode. -- **Diagnostics:** record pre/post SHA-256 model fingerprints, update seed, - visible episode indices, transition count, status, and global L2 parameter - delta. Zero delta is recorded as a non-mutating update, not successful - adaptation. - -The Fixed arm is an independent clone of the frozen checkpoint. It performs no -training or optimizer operation after the fork. Both arms use the same derived -post-shift episode seeds. A smoke run validates machinery only and is not -empirical evidence for the research hypothesis. - -## Implementation constraint - -Algorithm-specific recorded-data adapters are implemented for PPO and SAC. -The complete benchmark experiment runner, environment shift integration, -artifacts, and CLI are still required before the treatment may be run as an -Issue #265 experiment. The runner must fail explicitly for any configuration -that cannot provide the required PPO behavior quantities or recorded transition -data without extra environment interaction. # AdaptiveRL Treatment Card: Online Fine-Tuning Specification **Protocol Version**: 2.0
diff --git a/docs/research/adaptive_rl_hypothesis.md b/docs/research/adaptive_rl_hypothesis.md index b5442e7..74f5e15 100644 --- a/docs/research/adaptive_rl_hypothesis.md +++ b/docs/research/adaptive_rl_hypothesis.md @@ -9,24 +9,23 @@ This document is the normative, pre-specified experimental contract for evaluati > [!WARNING] > **Implementation Status** > -> **[VERIFIED-CODE]** `src/adaptive_rl/benchmarking/adaptation_runner.py` now implements the Issue #265 drone TEST-B lifecycle: train once on nominal parameters, share the pre-shift and shock episodes, fork independent PPO/SAC policies, update only Adaptive between episodes B5–B14, and compute recovery from the paired trajectories. `tests/test_adaptation_smoke.py` exercises the CI-sized PPO path; both PPO and SAC real-environment smoke paths have been executed. +> **[VERIFIED-CODE]** Current repository infrastructure trains on a nominal scenario, freezes the resulting policy, fingerprints it, and rejects any mutation during evaluation — see `src/adaptive_rl/experiments/shift_runner.py:180-205` at the pinned commit. > -> **[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Four non-drone primary cells do not have an Issue #265 shift environment/configuration in this checkout. They remain inconclusive; the six-cell family claim cannot be evaluated from the drone cell alone. +> **[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** The online Adaptive treatment defined in §4–§5. No online update/adaptation harness exists in the repository. > -> **[NO RESEARCH RESULT]** No full ten-replicate scientific benchmark has been collected. Smoke execution validates software behavior only and does not establish empirical superiority or statistical significance. +> **[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** The per-episode derived seed schedule of §14 is not yet executed by any runner; the current runner evaluates the config-declared scenario seed sets (`shift_runner.py:198-204`). > -> **PR #168 / Issue #98** introduced this protocol and its executable constants/seed/recovery/statistics mirror. Issue #265 adds the drone TEST-B harness. Nothing here claims a full research result or empirical validation. +> **PR #168 / Issue #98** is documentation plus this executable protocol mirror only. Nothing in this document should be read as claiming that any Adaptive cell is currently executable, that any result exists, or that the protocol has been validated on data. **Status tags used throughout** (a value is what its tag says, nothing more): | Tag | Meaning | |---|---| -| `[VERIFIED-CODE]` | Behavior verified in the current repository source; frozen scientific values remain pinned to the protocol version | -| `[SMOKE-VALIDATED]` | CI-sized or one-off smoke execution verified software behavior only; not research evidence | +| `[VERIFIED-CODE]` | Behavior verified by reading/reviewing repository source at the pinned commit | | `[VERIFIED-CONFIG]` | Value appears literally in the cited config file at the pinned commit | | `[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]` | Frozen requirement for a future harness; not present in the repository today | | `[FUTURE PROTOCOL VALUE]` | Pre-registered target that must be configured before that condition is executable | -| `[FUTURE DESIGN DECISION]` | Must be frozen in a signed-off artifact before that cell collects data | +| `[FUTURE DESIGN DECISION]` | Must be frozen in a signed-off artifact **before any data is collected** (see §5.3) | ## 2. Research Question @@ -66,7 +65,7 @@ The primary test is one-sided, consistent with H1 (§18.2). ### 4.2 Train-once, clone, fork design -**[SMOKE-VALIDATED]** The drone TEST-B runner executes each replicate as one shared segment followed by two arm segments: +**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Each replicate is executed as one shared segment followed by two arm segments: ``` TRAIN (seed = training_seed(i)) # one training run per replicate @@ -93,7 +92,7 @@ No running normalization statistics exist in the repository (no VecNormalize or ### 5.1 Block schedule -**[SMOKE-VALIDATED]** Exactly `N_update = 10` update blocks execute strictly between the termination of episode $k$ and the reset of episode $k+1$, for $k = 5, 6, \dots, 14$ (block identifiers $B_5, \dots, B_{14}$): +**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Exactly `N_update = 10` update blocks exist, executed strictly between the termination of episode $k$ and the reset of episode $k+1$, for $k = 5, 6, \dots, 14$ (block identifiers $B_5, \dots, B_{14}$): | Block | Executes between | Visible data (frozen) | |---|---|---| @@ -116,7 +115,7 @@ An update block completes (or fails) **before** the next episode reset. A failed ### 5.3 What is frozen vs. what must still be frozen before execution * **Frozen now**: schedule of §5.1, data visibility, freeze windows, identical-seed rule, logging requirements, all constants in §24. -* **[SMOKE-VALIDATED]** The adaptation algorithm, buffer construction, native PPO/SAC update details, inherited hyperparameters, and failure behavior are frozen in `docs/research/TREATMENT_CARD.md`. Artifacts record the Card SHA-256. Smoke tests are not evidence of outcome-based tuning or scientific validity. +* **[FUTURE DESIGN DECISION]** The adaptation algorithm itself (loss, epochs, learning rate, batch construction, buffer capacity) is not specified here because no harness exists to validate a specification against. Before any data is collected, a **Treatment Card** must be written and frozen (commit SHA recorded in the results artifact) that specifies these details under these constraints: (a) base-algorithm hyperparameters unchanged from the cell config except parameters declared in the Card; (b) no access to test/evaluation outcomes from episodes > k when executing $B_k$; (c) no seed other than the derived ones; (d) per-block logging of data used, parameter delta norm, and fingerprint. Executing without a frozen Card is a protocol violation. ## 6. Operational Definition of Recovery @@ -206,7 +205,7 @@ Pre-registered training budget and config pinned to §15; final-checkpoint rule ## 9. Baseline -**Fixed arm**: trained on the nominal distribution; policy frozen and fingerprinted immediately after training; evaluated under shift with weights locked at all times (`src/adaptive_rl/benchmarking/adaptation_runner.py` verifies the Fixed fingerprint against the frozen fingerprint `[SMOKE-VALIDATED]`). +**Fixed arm**: trained on the nominal distribution; policy frozen and fingerprinted immediately after training; evaluated under shift with weights locked at all times (`shift_runner.py:180-205` already enforces fingerprint invariance for the existing benchmark `[VERIFIED-CODE]`). ## 10. Evaluation Environments @@ -221,9 +220,9 @@ At the pinned commit `[VERIFIED-CODE src/adaptive_rl/environments/__init__.py:60 ## 11. Training Distribution -Issue #265 training uses only `environment.parameters` from `configs/drone_distribution_shift.yaml`; the distinct TEST-B `shift_parameters` are merged by the runner only after training and the shared pre-shift evaluation. Derived update/evaluation seeds are validated against the frozen configuration seed pools. The runner's separate configuration fields make the TEST-B values unavailable to the training environment constructor `[SMOKE-VALIDATED]`. +All training uses only the nominal configuration and only the TRAIN seed pool `[1000..1014]` `[VERIFIED-CONFIG]`. Test seeds and shift parameters must not leak into training; post-hoc scenario selection is prohibited. The existing runner enforces this via `TrainingDistributionWrapper` and the seed-containment audit (`shift_runner.py:143-177`) `[VERIFIED-CODE]`. -**[SMOKE-VALIDATED]** Each replicate trains with the selected preregistered `training_seed`; the trainer receives a per-replicate config copy and the source config is unchanged. +**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** Each replicate trains with `config.seed := training_seed(i)` (§14.1), replacing the config default (`42`/`123`). The containment audit must still pass afterwards; this has not been exercised. ## 12. Distribution Shifts @@ -324,7 +323,7 @@ Step 8 — CAUSAL RECOVERY MEASUREMENT adaptive_rl.protocol.recovery, not ad-hoc scripts. ``` -**Common-random-number property** `[SMOKE-VALIDATED]`: episodes are reseeded at every reset, and drone gust noise is drawn from the seeded environment RNG once per step. Both arms therefore receive the same exogenous noise stream for episode $j$ up to the point where episode length diverges (the number of RNG draws per episode is action-dependent). CRN alignment is exact for the shared segment and partial-by-construction for the arm segment; it is a variance-reduction property, not a claim of identical trajectories. +**Common-random-number property** `[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]`: episodes are reseeded at every reset, and environment exogenous noise is drawn from the seeded episode RNG in a way that does not depend on the action taken (e.g. drone gusts: `disturbed_drone.py:637,649` pass `rng=self.np_random`; `wind.py:85` draws `rng.normal(size=3)` per step) `[VERIFIED-CODE]`. Both arms therefore see the same exogenous noise stream for episode $j$ *up to the point where episode length diverges* (the number of RNG draws per episode is action-dependent). CRN alignment is exact for the shared segment and partial-by-construction for the arm segment; it is a variance-reduction property, not a claim of identical trajectories. ## 14. Seed Protocol @@ -340,7 +339,7 @@ TRAINING_SEEDS = [31001, 31002, 31003, 31004, 31005, 31006, 31007, 31008, 31009, ### 14.2 Derived episode/block seeds — exact specification -**[VERIFIED-CODE]** There are 400 derived values per schedule (10 seeds × (15 pre + 15 post + 10 update)). The derivation is SHA-256, **not** Python's `hash()` (which is salted per process by `PYTHONHASHSEED`): +**[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]** There are 400 derived values per schedule (10 seeds × (15 pre + 15 post + 10 update)). The derivation is SHA-256, **not** Python's `hash()` (which is salted per process by `PYTHONHASHSEED`): ``` payload = f"{training_seed}|{phase}|{index}".encode("utf-8") @@ -370,11 +369,11 @@ value = int.from_bytes(digest[:4], byteorder="big", signed=False) & 0x7FFFFFF schedule_fingerprint = 65939167572731c99599c382ac50cf3fddbba3cf758305764392f13b2e4efa67 ``` -The research artifact and each replicate record this fingerprint, shared by both arms; a mismatch against this document invalidates the replicate. Any change to seeds, indices, or phases changes the fingerprint and therefore fails `tests/test_protocol_doc_sync.py`. +Both arms record this fingerprint in their artifacts; a mismatch between arms (or against this document) invalidates the replicate. Any change to seeds, indices, or phases changes the fingerprint and therefore fails `tests/test_protocol_doc_sync.py`. ### 14.4 RNG initialization and train/test disjointness -Every pre/post episode reset uses `reset(seed = derived_value)` and every update block uses its corresponding derived update seed `[SMOKE-VALIDATED]`. Training-side seeding remains `random.seed` / `np.random.seed` / `torch.manual_seed` / CUDA seeds in `trainer.py`. Disjointness of the derived schedule from training seeds and config pools is enforced by `validate_schedule()` (§14.2). Determinism is **not** claimed across hardware/library versions (§25). +Every episode reset uses `reset(seed = derived_value)` so run order cannot leak state across episodes `[PROTOCOL REQUIREMENT — NOT IMPLEMENTED]`. Training-side seeding remains `random.seed` / `np.random.seed` / `torch.manual_seed` / CUDA seeds (`trainer.py:120-126`) `[VERIFIED-CODE]`. Train/test seed-set disjointness for the **config pools** is audited by `shift_runner.py:162-177` `[VERIFIED-CODE]`; disjointness of the **derived schedule** from those pools is enforced by `validate_schedule()` (§14.2). Determinism is **not** claimed across hardware/library versions (§25). ## 15. Training Budgets (Pinned) @@ -406,8 +405,8 @@ The deterministic pre-declared rule is the **final training checkpoint** at the |---|---|---| | 1 | `gridworld/ppo` | No — no shift config; moderate is design-reference only | | 2 | `traffic_signal/ppo` | No — moderate `[FUTURE PROTOCOL VALUE]` | -| 3 | `drone_disturbed/ppo` | Yes — runner and smoke path implemented; full ten-replicate data not collected | -| 4 | `drone_disturbed/sac` | Yes — runner and smoke path implemented; full ten-replicate data not collected | +| 3 | `drone_disturbed/ppo` | No — Adaptive harness absent (Fixed benchmark runnable) | +| 4 | `drone_disturbed/sac` | No — Adaptive harness absent (budget/config exist) | | 5 | `navigation_2d/ppo` | No — moderate `[FUTURE PROTOCOL VALUE]`; harness absent | | 6 | `navigation_2d/sac` | No — config absent; harness absent | @@ -492,11 +491,11 @@ Pairwise-complete exclusion in the primary analysis can create attrition bias if ## 20. Reporting Requirements -### 20.1 Historical Fixed-benchmark artifact (pinned protocol reference) +### 20.1 Existing artifact (Fixed benchmark) `[VERIFIED-CODE src/adaptive_rl/evaluation/shift_benchmark.py:523-557]` `ShiftBenchmarkReport` stores (actual field names): `schema_version`, `experiment_name`, `environment_name`, `algorithm_name`, `deterministic`, `total_training_timesteps`, `train_seeds`, `test_seeds`, **`scenarios`** (list of `ScenarioResult`), `recovery_definition`, `training_provenance`, `environment_provenance`, `config_sha256`, `metadata`. -The legacy Fixed-benchmark implementation at the pinned protocol revision stored per scenario: `scenario_name`, `role`, `seeds`, `environment_overrides`, `effective_environment_parameters`, `metrics`, `recovery`, **`episodes`** (list of `EpisodeBenchmarkRecord`), `gaps`, `policy_fingerprint`. That legacy module is not present in the current checkout; Issue #265 writes its own explicit adaptation schema. +Per scenario: `scenario_name`, `role`, `seeds`, `environment_overrides`, `effective_environment_parameters`, `metrics`, `recovery`, **`episodes`** (list of `EpisodeBenchmarkRecord`), `gaps`, `policy_fingerprint`. Per episode record (`shift_benchmark.py:466-490`): `seed`, `reward`, `length`, `success`, `collision`, `terminated`, `truncated`, `recovery_times`, `recovery_events`, `recovery_completed`, `recovery_censored`. @@ -504,9 +503,9 @@ Per episode record (`shift_benchmark.py:466-490`): `seed`, `reward`, `length`, ` ### 20.2 Derived research quantities -$P_{pre}$, $P_0$, $P(t)$, $R(t)$, $\tau$, $T_H$, status, δ, `delta_min`, and the update-block log are derived from episode records; the Issue #265 artifact stores them using `src/adaptive_rl/protocol/recovery.py` (never ad hoc equations). +$P_{pre}$, $P_0$, $P(t)$, $R(t)$, $\tau$, $T_H$, status, δ, `delta_min`, and the update-block log are **not native fields** of the current report; they must be derived from episode records by `src/adaptive_rl/protocol/recovery.py` (never ad hoc) and stored in a derived artifact. -### 20.3 Issue #265 artifact contents +### 20.3 Future Adaptive artifact (required contents) Per replicate × arm: training seed, all three phase-seed lists used, schedule fingerprint (§14.3), `PROTOCOL_VERSION`, pinned commit SHA, config SHA-256, pre-shift/shock/post-shift return vectors, per-episode success flags, $P_{pre}$, $P_0$, δ, `delta_min`, $P(t)$ and $R(t)$ trajectories, predicate vector, $\tau$, $T_H$, status; per block: block id, episode range, derived update seed, data-episode indices, parameter-delta norm, fingerprint before/after; plus environment/library provenance (`environment_provenance` already exists) including numpy version (bootstrap dependency, §18.3). @@ -521,13 +520,13 @@ Per replicate × arm: training seed, all three phase-seed lists used, schedule f | Threat | Why it matters | Mitigation | Remaining limitation | |---|---|---|---| -| Treatment-contaminated $P_0$ | Adaptive updating before $P_0$ is measured breaks normalization | Shared single execution of episodes 1–5 (§4.2); $B_5$ strictly after episode 5 | Full ten-replicate protocol execution remains unverified | +| Treatment-contaminated $P_0$ | Adaptive updating before $P_0$ is measured breaks normalization | Shared single execution of episodes 1–5 (§4.2); $B_5$ strictly after episode 5 | Depends on future harness honoring the ordering contract (§5.1–5.2) | | `no_degradation` convention | $T_H = 0$ anchors the distribution without adaptation evidence | Convention retained, status always labeled; arm-invariant so $D_i = 0$ exactly | Can still shift the *level* of $T_H$ vs other studies; comparisons must match conventions | | Finite-horizon truncation | $T_H = 15$ for non-recovery makes the estimand a truncated mean | Estimand explicitly finite-horizon; right-censoring labeled, never extrapolated | Not a claim about true recovery-time distributions | | Normality of $D_i$ (t-test) | Primary test assumes it; $n = 8..10$ cannot verify it | Sensitivities (§18.3) target median/pseudomedian/resampling; disagreement must be reported | Type I/II error may deviate from nominal if strongly violated | | `MIN_VALID_N = 8` threshold | A cell at exactly 8 has less power than one at 10 | Rule frozen pre-data; per-cell $N_{valid}$ always reported | Power is unquantified; no formal power analysis (§25) | | Failed-run attrition | Excluding crashes can bias the contrast | Pairwise exclusion; per-arm failure counts; two-sided $[0,H]$ imputation bounds | Counterfactual $T_H$ of crashed runs unobservable | -| Partial cell coverage | Only the drone PPO/SAC cells have the Issue #265 runner in this checkout | Missing cells remain explicitly inconclusive; no six-cell family claim | GridWorld, Traffic Signal, and Navigation cells are unimplemented | +| Adaptive harness absent | Nothing in the treatment sections is executable | Protocol preregistered before implementation; status tags | Actual adaptation behavior entirely untested | | Unequal compute | Adaptive adds online compute | Declared part of the intervention; Fixed budget unchanged | Deployment latency/cost unmeasured | | Training non-determinism | No cudnn/deterministic-algorithm flags exist (`trainer.py:120-126`) | Train-once design: one run serves both arms (§4.2) | Cross-machine retraining may not reproduce weights; recorded fingerprint detects it | | CRN partial alignment | Episode lengths are action-dependent | Reseeding per episode; alignment stated as partial-by-construction (§13) | Variance reduction weaker than exact pairing of noise | @@ -542,7 +541,7 @@ Per replicate × arm: training seed, all three phase-seed lists used, schedule f * [ ] Config YAMLs archived at the pinned commit; `config_sha256` recorded. * [ ] All 400 derived seeds logged; schedule fingerprint equals `65939167572731c99599c382ac50cf3fddbba3cf758305764392f13b2e4efa67` in both arms. * [ ] `validate_schedule()` run on the artifact's schedule with zero violations (uniqueness, range, disjointness). -* [ ] Training receives nominal parameters only; the runner withholds TEST-B parameters until after shared pre-shift evaluation, and `validate_schedule()` passes. +* [ ] Config-pool disjointness verified by the existing runner audit (`shift_runner.py:162-177`) **and** by `validate_schedule()`. * [ ] Train-once/fork design confirmed: one training fingerprint per replicate, both arms fork from it; Fixed arm fingerprint unchanged at end. * [ ] $P_0$ measured in the shared segment, pre-$B_5$; block ordering log shows $B_5$ strictly between episodes 5 and 6. * [ ] Per-block log complete for $B_5..B_{14}$ (10 blocks, no block after episode 15). @@ -550,7 +549,7 @@ Per replicate × arm: training seed, all three phase-seed lists used, schedule f * [ ] Primary test, sensitivities, Holm, and `decide_family()` executed from `adaptive_rl.protocol.statistics`. * [ ] Per-cell $N_{valid}$, failure counts by arm, and both imputation directions reported. * [ ] Doc-sync tests pass: `python -m pytest tests/test_protocol_seed_schedule.py tests/test_protocol_recovery.py tests/test_protocol_statistics.py tests/test_protocol_doc_sync.py`. -* [ ] Treatment Card (§5.3) frozen before a scientific run and its SHA recorded in the artifact. +* [ ] Treatment Card (§5.3) committed with SHA recorded before any data collection. ## 24. Executable Protocol Mirror diff --git a/docs/research/issue-265.md b/docs/research/issue-265.md deleted file mode 100644 index 7293cea..0000000 --- a/docs/research/issue-265.md +++ /dev/null @@ -1,105 +0,0 @@ -# Issue #265 implementation guide - -## What is implemented - -`adaptive-rl benchmark adaptation` runs the protocol's train-once, shared -pre-shift, shared shock, fork, and paired post-shift lifecycle for drone TEST-B -with PPO or SAC. The base environment receives only the nominal configuration -from `configs/drone_distribution_shift.yaml`; the TEST-B overrides are merged -after training and the pre-shift segment. The run checks the effective shift -values before proceeding. - -The online data boundary is represented by completed post-shift episode -records. Block Bk receives the ordered prefix 1..k only, and the builder rejects -missing, duplicated, out-of-order, wrong-seed, or future episode data. Blocks -B5 through B14 execute between episodes; B15 is rejected. PPO uses stored -behavior values/log-probabilities with its native clipped update. SAC uses a -new replay buffer containing only the allowed post-shift prefix. Both use the -block's derived update seed. Update failures roll model state back and fail the -replicate. - -The fork deep-copies both arms, verifies equal starting policy states, and gives -the Fixed arm a prediction-only interface. The runner verifies its final -fingerprint against the frozen fingerprint. Episode records include outcomes, -derived seeds, policy fingerprints, update association, and the complete -transition data used by adaptation. - -Recovery is computed only by -`adaptive_rl.protocol.recovery.compute_recovery()`. Paired analysis uses the -protocol's paired t-test, interval, dz, sign and Wilcoxon sensitivities, -bootstrap interval, failure-imputation bounds, and Holm correction. Since this -checkout implements the selected drone cell only, the other five primary cells are -reported as inconclusive in each single-cell artifact; the six-cell family -claim is therefore inconclusive until the remaining cells have data. - -## Running it - -Run a reduced, explicitly labeled machinery check: - -```bash -adaptive-rl benchmark adaptation \ - --config configs/drone_distribution_shift.yaml \ - --stochastic \ - --smoke \ - --output-dir artifacts/issue265_smoke_ppo -``` - -PPO adaptation requires stochastic behavior-policy actions: deterministic mean -actions do not form valid on-policy PPO rollout data. The runner rejects -deterministic PPO collection before training. This machinery check does not -resolve the separate conflict between that requirement and Issue #271's frozen -deterministic-evaluation rule; it is not a valid run of the preregistered study. - -Run the smoke path for SAC: - -```bash -adaptive-rl benchmark adaptation \ - --config configs/drone_distribution_shift.yaml \ - --algorithm sac \ - --smoke \ - --output-dir artifacts/issue265_smoke_sac -``` - -For an Issue #265 PPO run, explicitly select stochastic evaluation actions (the -same command accepts `--algorithm sac` for SAC): - -The stochastic PPO setting conflicts with Issue #271's frozen deterministic -evaluation rule. Treat this command as Issue #265 machinery/diagnostic output; -do not report it as a valid preregistered-study result without a prospective -protocol amendment. - -```bash -adaptive-rl benchmark adaptation \ - --config configs/drone_distribution_shift.yaml \ - --stochastic \ - --output-dir artifacts/issue265_ppo -``` - -`--training-seeds` may select a comma-separated subset of the frozen training -seeds. Unselected replicates remain missing and count as failures for -pair-complete analysis. `--deterministic` and `--stochastic` override the -configured evaluation action selection. The command writes -`adaptation.json` and `adaptation.csv`; it refuses to overwrite either file or -an existing per-seed training directory. - -The JSON contains the full trajectories, replicate failures and reasons, -protocol and schedule fingerprints, Treatment Card SHA-256, canonical and raw -configuration hashes, repository commit/dirty state, runtime versions, -recovery outputs, and paired analysis. CSV has one row per recorded episode. -Smoke output is labeled `run_type: smoke`, uses a reduced training budget and -episode length, and is never treated as a scientific result. - -## Reproducibility and status - -Adaptation choices are frozen in -[`TREATMENT_CARD.md`](TREATMENT_CARD.md). The JSON records its content hash, -the full derived schedule fingerprint, each replicate's exact phase seeds, -each update seed, and model fingerprints before and after every block. Failed -training, evaluation, or update work is represented in `failure_summary` and -the replicate record; a failed update stops that replicate before the next -episode. - -Both PPO and SAC smoke paths have been executed and checked for block order, -episode counts, seed alignment, Fixed immutability, and recovery output. Those -checks validate the harness only. No full ten-replicate benchmark has been run -and no empirical superiority or statistical significance is claimed. diff --git a/docs/research/issue-271.md b/docs/research/issue-271.md deleted file mode 100644 index 6008698..0000000 --- a/docs/research/issue-271.md +++ /dev/null @@ -1,187 +0,0 @@ -# Issue #271: preregistered Adaptive vs Fixed study - -## Scope and preregistration - -This execution targets only `drone_disturbed/ppo` under TEST-B. The normative -protocol remains [`adaptive_rl_hypothesis.md`](adaptive_rl_hypothesis.md); this -document does not change its hypothesis, thresholds, alpha, horizon, seeds, or -analysis plan. The treatment is specified by -[`TREATMENT_CARD.md`](TREATMENT_CARD.md). - -## Before implementation: gap analysis - -| Requirement | Before | Evidence | -|---|---|---| -| Train once on nominal parameters; share pre-shift and shock; fork and adapt only between episodes | PARTIAL | `src/adaptive_rl/benchmarking/adaptation_runner.py` implemented the Issue #265 lifecycle, but had no Issue #271 invariant record | -| Exact ten-seed schedule and per-block provenance | PARTIAL | `src/adaptive_rl/protocol/seeds.py` froze the schedule; `--training-seeds` still allowed a subset | -| Recovery endpoint and registered tests/sensitivities | EXISTS | `src/adaptive_rl/protocol/recovery.py`, `src/adaptive_rl/protocol/statistics.py`, `src/adaptive_rl/benchmarking/adaptation_statistics.py` | -| Immutable JSON/CSV plus complete checksummed manifest and clean-tree execution gate | PARTIAL | `src/adaptive_rl/benchmarking/adaptation_artifacts.py` refused JSON/CSV overwrite, but there was no manifest or clean-tree gate | -| Required mutation, integrity, and tidy study CSV checks | PARTIAL | Existing Issue #265 tests covered smoke execution; no manifest validation or invariant mutation tests | -| Full real ten-replicate execution and report | MISSING | `docs/research/issue-265.md` stated that no full run had been collected | - -## Implementation - -The `adaptive-rl benchmark adaptation --study prereg-v1 --run-id RUN_ID` -entrypoint runs all ten training seeds in preregistered order, rejects subsets -and smoke mode, requires a clean committed tree, and writes into an immutable -run directory. Before training, `study_manifest.json` records the canonical -configuration, the frozen treatment and seed schedule, source revision, and -runtime identity. Its stable JSON representation is SHA-256 hashed. Prereg-v1 -also rejects any config or treatment-card hash that differs from the values -frozen here. `--resume` recomputes that identity and accepts only terminal -per-replicate records whose digest, study hash, protocol hash, and seed identity -verify; it never trusts partial training directories. An interrupted seed -without a complete checkpoint is recorded as failed, and remaining unstarted -seeds continue. A present but malformed, stale, or mismatched checkpoint aborts -resume rather than becoming a failed replicate. Final manifest validation also -rejects files added to the completed run directory after its artifact set was -recorded. Replicate checkpoints bind the training files they rely on, and the -runner validates all checkpoint state before resuming any unfinished seed. -Repeating resume after finalization validates and returns the same -immutable result. The JSON stores -the raw trajectories, protocol analysis, seed schedule, outcomes, runtime -invariants, and run status. -The runner returns that same persisted JSON on initial completion and repeated -resume; its artifact paths are stable and relative to the run directory. -The CSV has one row per replicate and arm, with finite-horizon `T_H`, status, -per-episode return vectors, and seed vectors. `manifest.json` checksums every -file in the run directory, including the pre-execution study manifest and -replicate checkpoints; `validate_study_manifest()` detects missing, changed, -malformed, or path-escaping entries. Result files are written atomically and -are not overwritten. - -The fixed arm has a prediction-only interface. Exact equality of its initial, -per-episode, and final policy fingerprints establishes a zero weight delta; the -artifact records this as `fixed_parameter_delta_l2: 0.0` and -`fixed_weight_update_count: 0`. Adaptive block logs contain their update seed, -visible episode prefix, finite loss metrics, state fingerprints, and parameter -delta. The runner recomputes invariant checks before writing a successful -replicate. - -Right-censored recovery remains the preregistered finite endpoint `T_H = 15`. -Failed replicates are kept in the artifact and CSV; they are omitted pairwise -from the primary analysis and retained in both preregistered imputation bounds. -The statistical artifact includes standard error, t statistic, degrees of -freedom, one-sided p-value, 95% interval, Cohen's `d_z`, exact Wilcoxon and sign -tests, bootstrap interval, and failure-imputation bounds. - -## Ambiguity resolutions - -* The literal training seeds 31001–31010 identify the ten replicates. Each uses - the existing SHA-256-derived `pre`, `post`, and `update` phase seeds; the frozen - schedule fingerprint is recorded and rechecked. -* The shared shock window is the single execution of TEST-B post-shift episodes - 1–5, used by both arms before B5. The shared segment is represented once in - the raw artifact and referenced by identical per-arm return-vector hashes. -* B5 through B14 use the exact completed post-shift episode prefix required by - `protocol.adaptation.build_update_batch`. No block runs after episode 15. -* Censoring uses the preregistered finite-horizon value 15, not infinity. This - differs from the issue prompt's parenthetical `T_H = inf`; the preregistration - is the specified source of truth. - -## Dependency and repository status - -No local `ExperimentManifest` or roadmap issue 4/5 implementation was found in -the current checkout, so the study uses the minimal manifest described above. -The checked-out base was `fix/pr-259-review-hardening`, one commit ahead and -seven behind `origin/main`; implementation proceeds on -`feat/issue-271-preregistered-study`. GitHub issue/PR pages were unavailable to -the browsing environment, so the live status of PR #266 and roadmap issues 4/5 -could not be independently verified. No dependency on unmerged code is used. - -## Execution record - -Execution status: **COMPLETE** for this one-cell, ten-replicate study. The -preregistered six-cell family remains **INCONCLUSIVE** because five cells are -not executable in this checkout. These data do not support H1. - -The intended single execution command is: - -```bash -.venv/bin/adaptive-rl benchmark adaptation \ - --config configs/drone_distribution_shift.yaml \ - --output-dir artifacts/issue271 \ - --study prereg-v1 \ - --run-id issue271-prereg-v1-20260929-01 -``` - -The runner enables Torch deterministic algorithms in warn-only mode and cuDNN -deterministic settings for the study, while recording that cross-hardware and -cross-library bitwise reproducibility is not claimed. - -## Scientific validity audit finding - -The frozen config sets `evaluation.deterministic: true`, and -`evaluate_episode()` passes that setting to PPO while collecting the post-shift -transitions later supplied to PPO's native clipped update. PPO's update assumes -actions were sampled from the recorded behavior distribution. A deterministic -mean action with its Gaussian density recorded as `behavior_log_prob` does not -have that sampling distribution, so the stored rollout is not a valid on-policy -PPO sample. The current preregistration does not define an action-sampling rule -that resolves this conflict. No treatment or analysis change is made here; the -adaptation runner now rejects deterministic PPO collection before training, -including the frozen prereg-v1 configuration. Issue #265's general PPO CLI can -run a machinery check with explicit stochastic collection, but that does not -resolve the conflict with this study's deterministic-evaluation rule. The PPO -treatment must not be described as scientifically validated until a prospective -protocol amendment resolves the action-selection contract. -This limitation does not change the previously recorded artifact or its -descriptive statistics; its `COMPLETE` status describes harness execution, not -valid on-policy PPO evidence. - -* Run ID: `issue271-prereg-v1-20260929-01` -* Run status: `COMPLETE`; 10 completed, 0 failed, ordered seeds 31001–31010. -* Executed commit: `a5250ffa5efaedaf76ad88c398f9087eb5ef49ca`; clean tree. -* Wall time: approximately 17 minutes 50 seconds (run-directory creation to - manifest creation); summed PPO training time was 939.38 seconds. -* Hardware/runtime: Linux x86_64, 4 logical CPUs, Python 3.14.7, gymnasium - 1.3.0, stable-baselines3 2.9.0, Torch 2.14.0, NumPy 2.5.3. The platform - reported no processor model. -* Artifacts: `artifacts/issue271/issue271-prereg-v1-20260929-01/` - * `adaptive_vs_fixed.json` SHA-256: - `e7b681d875e8fdfdcc56acada44a34062ac05c92e8881b16c8a3354633b79265` - * `adaptive_vs_fixed.csv` SHA-256: - `4d2879f08cf6caa67585576bc67c2cf0d699b617ee8260849c10b7ae7e7ce098` - * `manifest.json` SHA-256: - `bb5c3e6f5150a329aecca114dfd47f50366d99ddb1c3e84c515f6fa7d5d480ec` -* Config SHA-256: `0039c298b5048254b2d211cc66967e9d3e1d71275575fcfe28736485c65937b5`. - Treatment Card SHA-256: `8383e736f02ff31393474b241b06d2dc92b036910c93c26081c1673c92a317e3`. -* Independent audit verified the exact seed list and frozen schedule fingerprint, - ten passing invariant records, all 100 B5–B14 blocks, 20 tidy CSV rows, - all manifest-listed checksums, and no `/home/aryan` path in the JSON. - -The run predates the follow-up safe-resume addition; it executed from the clean -source commit recorded by the manifest and was not resumed. The artifact's -`execution_command` is normalized to `adaptive-rl` because the entrypoint -records the command's CLI form; the exact shell invocation above includes the -virtual-environment path used to select the installed executable. - -## Result from the artifact - -For every seed, both arms had `T_H = 0`: 4 replicates were `no_degradation` and -6 were `degradation_below_resolution` in each arm. There were no recovered or -right-censored replicates and no failures. Thus the censoring count was zero; -the finite right-censor endpoint remains `T_H = 15` by protocol. - -The paired differences were ten zeros. The artifact reports `N_valid = 10`, -mean difference 0, standard error 0, one-sided paired t statistic 0 on 9 df, -primary p = 0.5, 95% t interval [0, 0], Cohen's `d_z = 0`, exact sign p = 1, -exact Wilcoxon p = 1, and bootstrap interval [0, 0]. The cell is evaluable but -not significant. The six-cell family decision is `INCONCLUSIVE`; no family -claim is made. - -## Final acceptance status - -| Acceptance area | Status | Evidence | -|---|---|---| -| Ten preregistered PPO replicates and seeds | PASS | JSON seed list, schedule fingerprint, 10 completed records | -| Shared segments, identical fork, fixed lock, ten causal update blocks | PASS | Every replicate's `invariants.all_passed`; mutation coverage in `tests/test_adaptation_invariants.py` | -| Recovery and preregistered paired analyses | PASS | Per-replicate recovery plus `paired_analysis` in JSON | -| Immutable JSON/CSV and validated checksums | PASS | 20-row CSV; `manifest.json`; `validate_study_manifest()` | -| Safe continuation after interruption | PASS in follow-up code | Digest-verified terminal replicate checkpoints; incomplete directories become recorded failures | -| Statistical superiority of Adaptive | FAIL | All observed paired differences are zero; p = 0.5 | -| Six-cell family claim / power ≥ 0.80 | NOT MET | Five cells unavailable; preregistration says power is unquantified at N = 10 | - -PR #266 and roadmap issues 4/5 remain unverified live because GitHub was not -available. The run does not depend on them. This is a one-cell study, not -evidence for the broader multi-environment claim. diff --git a/src/adaptive_rl/algorithms/__init__.py b/src/adaptive_rl/algorithms/__init__.py index d55f201..17fa5b7 100644 --- a/src/adaptive_rl/algorithms/__init__.py +++ b/src/adaptive_rl/algorithms/__init__.py @@ -1,11 +1,5 @@ """Algorithm abstraction layer for AdaptiveRL.""" -from adaptive_rl.algorithms.adaptation import ( - AdaptationUpdateLog, - PPOAdaptationAdapter, - SACAdaptationAdapter, - run_adaptation_update, -) from adaptive_rl.algorithms.base import BaseAlgorithm from adaptive_rl.algorithms.ppo import PPOAlgorithm from adaptive_rl.algorithms.random_policy import RandomPolicy @@ -27,12 +21,9 @@ "AlgorithmMetadata", "AlgorithmRegistry", "AlgorithmRegistryError", - "AdaptationUpdateLog", "BaseAlgorithm", "PPOAlgorithm", - "PPOAdaptationAdapter", "RandomPolicy", - "SACAdaptationAdapter", "SACAlgorithm", "algorithm_registry", "get_algorithm_factory", @@ -41,5 +32,4 @@ "list_all_algorithm_metadata", "load_algorithm_from_pretrained", "register_algorithm", - "run_adaptation_update", ] diff --git a/src/adaptive_rl/algorithms/adaptation.py b/src/adaptive_rl/algorithms/adaptation.py deleted file mode 100644 index ca8d3c6..0000000 --- a/src/adaptive_rl/algorithms/adaptation.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Recorded-data PPO and SAC update adapters for Issue #265.""" - -from __future__ import annotations - -import random -from contextlib import contextmanager -from dataclasses import asdict, dataclass -from typing import Any, Iterator - -import numpy as np -import torch -from stable_baselines3.common.buffers import ReplayBuffer, RolloutBuffer -from stable_baselines3.common.logger import Logger - -from adaptive_rl.protocol.adaptation import AdaptationAdapter, UpdateBatch, call_update_atomically -from adaptive_rl.protocol.fork import model_fingerprint, policy_state_tensors - - -@contextmanager -def _seeded_update(seed: int) -> Iterator[None]: - """Seed update-side RNGs and restore caller RNG state on exit.""" - py_state = random.getstate() - np_state = np.random.get_state() - torch_state = torch.random.get_rng_state() - cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None - try: - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - yield - finally: - random.setstate(py_state) - np.random.set_state(np_state) - torch.random.set_rng_state(torch_state) - if cuda_states is not None: - torch.cuda.set_rng_state_all(cuda_states) - - -def _model(algorithm: Any) -> Any: - model = getattr(algorithm, "model", None) - if model is None: - raise RuntimeError("Cannot adapt an uninitialized algorithm") - return model - - -@contextmanager -def _logger_ready(model: Any) -> Iterator[None]: - """Provide a silent SB3 logger when a model is adapted before its first fit.""" - had_logger = hasattr(model, "_logger") - old_logger = getattr(model, "_logger", None) - if not had_logger: - model.set_logger(Logger(folder=None, output_formats=[])) - logger = getattr(model, "logger", None) - old_values = dict(getattr(logger, "name_to_value", {})) - recorded_values = getattr(logger, "name_to_value", None) - if isinstance(recorded_values, dict): - recorded_values.clear() - try: - yield - finally: - if isinstance(recorded_values, dict): - recorded_values.clear() - recorded_values.update(old_values) - if had_logger: - model._logger = old_logger - else: - delattr(model, "_logger") - - -class PPOAdaptationAdapter: - """Use Stable-Baselines3 PPO's native clipped objective on stored rollouts.""" - - def __init__(self) -> None: - self.last_loss_metrics: dict[str, float] = {} - - def update(self, algorithm: Any, batch: UpdateBatch) -> None: - model = _model(algorithm) - if not hasattr(model, "rollout_buffer") or not hasattr(model, "n_epochs"): - raise TypeError("PPOAdaptationAdapter requires a Stable-Baselines3 PPO model") - if int(model.n_epochs) < 1 or int(model.batch_size) < 1: - raise ValueError("PPO adaptation requires positive configured epochs and batch size") - if any( - transition.behavior_log_prob is None or transition.behavior_value is None - for transition in batch.transitions - ): - raise ValueError( - "PPO adaptation requires behavior log-probability and value per transition" - ) - if any( - transition.truncated - and not transition.terminated - and transition.behavior_next_value is None - for transition in batch.transitions - ): - raise ValueError("PPO truncated transitions require their recorded behavior next-value") - - count = len(batch.transitions) - model.rollout_buffer = RolloutBuffer( - buffer_size=count, - observation_space=model.observation_space, - action_space=model.action_space, - device=model.device, - gae_lambda=model.gae_lambda, - gamma=model.gamma, - n_envs=1, - ) - model.rollout_buffer.reset() - observations: list[np.ndarray] = [] - - for transition in batch.transitions: - observation = np.asarray(transition.observation) - action = np.asarray(transition.action) - reward = float(transition.reward) - if transition.truncated and not transition.terminated: - assert transition.behavior_next_value is not None - reward += model.gamma * float(transition.behavior_next_value) - observations.append(observation) - model.rollout_buffer.add( - obs=observation.reshape((1, *observation.shape)), - action=action.reshape((1, -1)), - reward=np.asarray([reward], dtype=np.float32), - episode_start=np.asarray( - [ - len(observations) == 1 - or _previous_done(batch.transitions, len(observations) - 1) - ], - dtype=np.float32, - ), - value=torch.as_tensor( - [transition.behavior_value], dtype=torch.float32, device=model.device - ), - log_prob=torch.as_tensor( - [transition.behavior_log_prob], dtype=torch.float32, device=model.device - ), - ) - model.rollout_buffer.compute_returns_and_advantage( - last_values=torch.zeros(1, dtype=torch.float32, device=model.device), - dones=np.ones(1, dtype=np.float32), - ) - with _logger_ready(model), _seeded_update(batch.seed): - model.train() - self.last_loss_metrics = _loss_metrics(model) - model.policy.set_training_mode(False) - - -def _previous_done(transitions: tuple[Any, ...], previous_index: int) -> bool: - previous = transitions[previous_index - 1] - return bool(previous.terminated or previous.truncated) - - -class SACAdaptationAdapter: - """Train SAC from a fresh replay buffer containing only the visible batch.""" - - def __init__(self) -> None: - self.last_loss_metrics: dict[str, float] = {} - - def update(self, algorithm: Any, batch: UpdateBatch) -> None: - model = _model(algorithm) - if not hasattr(model, "critic_target") or not hasattr(model, "gradient_steps"): - raise TypeError("SACAdaptationAdapter requires a Stable-Baselines3 SAC model") - if int(model.gradient_steps) < 1 or int(model.batch_size) < 1: - raise ValueError( - "SAC adaptation requires positive configured gradient steps and batch size" - ) - count = len(batch.transitions) - # Never reuse the replay buffer containing nominal training experience. - replay = ReplayBuffer( - buffer_size=max(count, int(model.batch_size)), - observation_space=model.observation_space, - action_space=model.action_space, - device=model.device, - n_envs=1, - optimize_memory_usage=False, - handle_timeout_termination=True, - ) - for transition in batch.transitions: - observation = np.asarray(transition.observation).reshape( - (1, *np.asarray(transition.observation).shape) - ) - next_observation = np.asarray(transition.next_observation).reshape( - (1, *np.asarray(transition.next_observation).shape) - ) - action = np.asarray(transition.action).reshape((1, -1)) - done = bool(transition.terminated or transition.truncated) - replay.add( - observation, - next_observation, - action, - np.asarray([transition.reward], dtype=np.float32), - np.asarray([done], dtype=np.float32), - infos=[ - { - "TimeLimit.truncated": bool( - transition.truncated and not transition.terminated - ) - } - ], - ) - - prior_buffer = model.replay_buffer - model.replay_buffer = replay - try: - with _logger_ready(model), _seeded_update(batch.seed): - model.train( - gradient_steps=int(model.gradient_steps), batch_size=int(model.batch_size) - ) - self.last_loss_metrics = _loss_metrics(model) - finally: - model.replay_buffer = prior_buffer - model.policy.set_training_mode(False) - - -@dataclass(frozen=True) -class AdaptationUpdateLog: - block_episode: int - update_seed: int - visible_episode_indices: tuple[int, ...] - transition_count: int - fingerprint_before: str - fingerprint_after: str - parameter_delta_l2: float - status: str - loss_metrics: dict[str, float] - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -def _loss_metrics(model: Any) -> dict[str, float]: - logger = getattr(model, "logger", None) - recorded = getattr(logger, "name_to_value", {}) if logger is not None else {} - metrics = { - str(name): float(value) - for name, value in recorded.items() - if "loss" in str(name).lower() - and isinstance(value, (int, float, np.number)) - and np.isfinite(value) - } - if not metrics: - raise RuntimeError("adaptation update produced no finite loss metrics") - return metrics - - -def _parameters(model_or_wrapper: Any) -> dict[str, torch.Tensor]: - return { - name: value.detach().cpu().clone() - for name, value in policy_state_tensors(model_or_wrapper).items() - } - - -def _validate_update(model_or_wrapper: Any, before: dict[str, torch.Tensor]) -> float: - model = getattr(model_or_wrapper, "model", model_or_wrapper) - after = policy_state_tensors(model_or_wrapper) - policy = getattr(model, "policy", model) - parameter_names = {f"policy.{name}" for name, _ in policy.named_parameters()} - if isinstance(getattr(model, "log_ent_coef", None), torch.Tensor): - parameter_names.add("algorithm.log_ent_coef") - if after.keys() != before.keys(): - raise RuntimeError("Update changed the model state structure") - squared_delta = 0.0 - for name, tensor in after.items(): - if tensor.shape != before[name].shape or tensor.dtype != before[name].dtype: - raise RuntimeError(f"Update changed model tensor shape or dtype: {name}") - if not torch.isfinite(tensor).all(): - raise FloatingPointError(f"Update produced non-finite model state: {name}") - if name in parameter_names: - difference = tensor.detach().cpu().to(torch.float64) - before[name].to(torch.float64) - squared_delta += float(torch.sum(difference * difference)) - for name, optimizer in _optimizers(model): - _validate_finite_tree(optimizer.state_dict(), f"optimizer {name}") - for parameter, state in optimizer.state.items(): - for state_name, state_value in state.items(): - if isinstance(state_value, torch.Tensor): - if state_value.numel() > 1 and state_value.shape != parameter.shape: - raise RuntimeError( - f"Update corrupted optimizer state shape: {name}.{state_name}" - ) - return float(np.sqrt(squared_delta)) - - -def _optimizers(model: Any) -> list[tuple[str, Any]]: - result = [] - for name, value in vars(model).items(): - if isinstance(value, torch.optim.Optimizer): - result.append((name, value)) - policy_optimizer = getattr(getattr(model, "policy", None), "optimizer", None) - if isinstance(policy_optimizer, torch.optim.Optimizer): - result.append(("policy.optimizer", policy_optimizer)) - for owner_name in ("actor", "critic"): - owner = getattr(model, owner_name, None) - optimizer = getattr(owner, "optimizer", None) - if isinstance(optimizer, torch.optim.Optimizer): - result.append((f"{owner_name}.optimizer", optimizer)) - ent_optimizer = getattr(model, "ent_coef_optimizer", None) - if isinstance(ent_optimizer, torch.optim.Optimizer): - result.append(("ent_coef_optimizer", ent_optimizer)) - return result - - -def _validate_finite_tree(value: Any, path: str) -> None: - if isinstance(value, torch.Tensor): - if not torch.isfinite(value).all(): - raise FloatingPointError(f"Update produced non-finite {path}") - elif isinstance(value, dict): - for key, nested in value.items(): - _validate_finite_tree(nested, f"{path}.{key}") - elif isinstance(value, (list, tuple)): - for index, nested in enumerate(value): - _validate_finite_tree(nested, f"{path}[{index}]") - elif isinstance(value, (float, int)) and not np.isfinite(value): - raise FloatingPointError(f"Update produced non-finite {path}") - - -class _ValidatedAdapter: - def __init__(self, delegate: AdaptationAdapter, before: dict[str, torch.Tensor]) -> None: - self.delegate = delegate - self.before = before - - def update(self, algorithm: Any, batch: UpdateBatch) -> None: - self.delegate.update(algorithm, batch) - _validate_update(algorithm, self.before) - - -def run_adaptation_update( - algorithm: Any, adapter: AdaptationAdapter, batch: UpdateBatch -) -> AdaptationUpdateLog: - """Execute one seeded block atomically and return its auditable diagnostics.""" - before_state = _parameters(algorithm) - before_fingerprint = model_fingerprint(algorithm) - call_update_atomically(algorithm, _ValidatedAdapter(adapter, before_state), batch) - delta = _validate_update(algorithm, before_state) - after_fingerprint = model_fingerprint(algorithm) - return AdaptationUpdateLog( - block_episode=batch.block_episode, - update_seed=batch.seed, - visible_episode_indices=batch.visible_episode_indices, - transition_count=len(batch.transitions), - fingerprint_before=before_fingerprint, - fingerprint_after=after_fingerprint, - parameter_delta_l2=delta, - status="updated" if delta > 0.0 else "no_parameter_change", - loss_metrics=dict(getattr(adapter, "last_loss_metrics", {})), - ) - - -__all__ = [ - "AdaptationUpdateLog", - "PPOAdaptationAdapter", - "SACAdaptationAdapter", - "run_adaptation_update", -] diff --git a/src/adaptive_rl/benchmarking/__init__.py b/src/adaptive_rl/benchmarking/__init__.py index 7b5de84..707ef70 100644 --- a/src/adaptive_rl/benchmarking/__init__.py +++ b/src/adaptive_rl/benchmarking/__init__.py @@ -9,12 +9,6 @@ from importlib import import_module from typing import Any -from adaptive_rl.benchmarking.adaptation_artifacts import write_adaptation_artifacts -from adaptive_rl.benchmarking.adaptation_statistics import ( - PairedRecoveryAnalysis, - analyze_paired_recovery, - analyze_primary_cells, -) from adaptive_rl.benchmarking.learning_curve import ( BenchmarkRunError, LearningCurveBenchmarkResult, @@ -65,11 +59,7 @@ def __getattr__(name: str) -> Any: "BenchmarkRunError", "LearningCurveBenchmarkResult", "LearningCurvePoint", - "PairedRecoveryAnalysis", - "analyze_paired_recovery", - "analyze_primary_cells", "plot_learning_curve", "run_learning_curve_benchmark", "validate_budgets", - "write_adaptation_artifacts", ] diff --git a/src/adaptive_rl/benchmarking/adaptation_artifacts.py b/src/adaptive_rl/benchmarking/adaptation_artifacts.py deleted file mode 100644 index 2a9b386..0000000 --- a/src/adaptive_rl/benchmarking/adaptation_artifacts.py +++ /dev/null @@ -1,764 +0,0 @@ -"""JSON and analysis-friendly CSV serialization for Issue #265 runs.""" - -from __future__ import annotations - -import csv -import dataclasses -import hashlib -import importlib.metadata -import json -import os -import platform -import subprocess -import tempfile -from pathlib import Path -from typing import Any, Iterable, Mapping, Sequence, cast - -import numpy as np - -STUDY_MANIFEST_SCHEMA_VERSION = "1.0" -STUDY_ARTIFACT_SCHEMA_VERSION = "1.0" -REPLICATE_CHECKPOINT_SCHEMA_VERSION = "1.1" - -CSV_FIELDS = ( - "replicate_index", - "training_seed", - "arm", - "phase", - "episode_index", - "episode_seed", - "algorithm", - "environment", - "reward", - "length", - "success", - "collision", - "terminated", - "truncated", - "policy_fingerprint_start", - "policy_fingerprint_end", - "update_block", - "update_seed", - "update_status", - "parameter_delta_l2", -) - -STUDY_CSV_FIELDS = ( - "training_seed", - "arm", - "replicate_status", - "recovery_status", - "T_H", - "pre_returns", - "shock_returns", - "post_returns", - "pre_seeds", - "shock_seeds", - "post_seeds", - "update_seeds", - "failure_reason", - "json_trajectory_reference", -) - - -def _plain(value: Any) -> Any: - """Convert supported scientific data values into strict JSON primitives.""" - if dataclasses.is_dataclass(value) and not isinstance(value, type): - return _plain(dataclasses.asdict(value)) - if isinstance(value, np.ndarray): - return value.tolist() - if isinstance(value, np.generic): - return value.item() - if isinstance(value, Path): - return str(value) - if isinstance(value, Mapping): - if not all(isinstance(key, (str, int, float, bool)) for key in value): - raise TypeError("artifact mappings must have primitive keys") - return {str(key): _plain(nested) for key, nested in value.items()} - if isinstance(value, (list, tuple)): - return [_plain(nested) for nested in value] - if isinstance(value, str) and value.startswith("/"): - candidate = Path(value) - try: - return candidate.resolve().relative_to(Path.cwd().resolve()).as_posix() - except (OSError, ValueError): - return candidate.name - if value is None or isinstance(value, (str, int, float, bool)): - return value - raise TypeError(f"unsupported artifact value type: {type(value).__name__}") - - -def _episode_rows(artifact: Mapping[str, Any]) -> Iterable[dict[str, Any]]: - for replicate_index, replicate in enumerate(artifact.get("replicates", []), start=1): - common = { - "replicate_index": replicate_index, - "training_seed": replicate.get("training_seed"), - "algorithm": artifact.get("experiment", {}).get("algorithm"), - "environment": artifact.get("experiment", {}).get("environment"), - } - segments = ( - ("shared_pre_shift_episodes", "pre", "shared"), - ("shared_shock_episodes", "post", "shared"), - ("adaptive_episodes", "post", "adaptive"), - ("fixed_episodes", "post", "fixed"), - ) - for key, phase, arm in segments: - for episode in replicate.get(key, []): - row: dict[str, Any] = {field: None for field in CSV_FIELDS} - row.update(common) - row.update( - arm=arm, - phase=phase, - episode_index=episode.get("episode_index"), - episode_seed=episode.get("episode_seed"), - reward=episode.get("reward"), - length=episode.get("length"), - success=episode.get("success"), - collision=episode.get("collision"), - terminated=episode.get("terminated"), - truncated=episode.get("truncated"), - policy_fingerprint_start=episode.get("policy_fingerprint_start"), - policy_fingerprint_end=episode.get("policy_fingerprint_end"), - update_block=episode.get("update_block"), - update_seed=episode.get("update_seed"), - update_status=episode.get("update_status"), - parameter_delta_l2=episode.get("parameter_delta_l2"), - ) - yield row - - -def write_adaptation_artifacts( - artifact: Mapping[str, Any], - output_dir: str | Path, - *, - stem: str = "adaptation", -) -> tuple[Path, Path]: - """Write strict JSON and flattened CSV without overwriting prior results. - - Both payloads are first written to temporary files, then installed with - exclusive hard links. If either target already exists, no target is - overwritten and the call raises ``FileExistsError``. - """ - if not stem or Path(stem).name != stem: - raise ValueError("stem must be a non-empty filename component") - target_dir = Path(output_dir) - target_dir.mkdir(parents=True, exist_ok=True) - json_path = target_dir / f"{stem}.json" - csv_path = target_dir / f"{stem}.csv" - plain = _plain(artifact) - if not isinstance(plain, dict): - raise TypeError("artifact root must be a mapping") - - temp_paths: list[Path] = [] - try: - for suffix, writer in ( - (".json", lambda handle: json.dump(plain, handle, indent=2, allow_nan=False)), - (".csv", lambda handle: _write_csv(handle, plain)), - ): - fd, temp_name = tempfile.mkstemp(prefix=f".{stem}-", suffix=suffix, dir=target_dir) - temp_path = Path(temp_name) - temp_paths.append(temp_path) - with os.fdopen(fd, "w", newline="", encoding="utf-8") as handle: - writer(handle) - handle.flush() - os.fsync(handle.fileno()) - # link() is atomic and fails if the destination exists. - os.link(temp_paths[0], json_path) - try: - os.link(temp_paths[1], csv_path) - except BaseException: - json_path.unlink() - raise - finally: - for temp_path in temp_paths: - temp_path.unlink(missing_ok=True) - return json_path, csv_path - - -def write_adaptive_vs_fixed_artifacts( - artifact: Mapping[str, Any], output_dir: str | Path -) -> tuple[Path, Path]: - """Write study JSON plus the preregistered one-row-per-arm CSV.""" - target_dir = Path(output_dir) - target_dir.mkdir(parents=True, exist_ok=True) - json_path = target_dir / "adaptive_vs_fixed.json" - csv_path = target_dir / "adaptive_vs_fixed.csv" - plain = _plain(artifact) - temp_paths: list[Path] = [] - try: - for suffix, writer in ( - (".json", lambda handle: json.dump(plain, handle, indent=2, allow_nan=False)), - (".csv", lambda handle: _write_study_csv(handle, plain)), - ): - fd, temp_name = tempfile.mkstemp( - prefix=".adaptive-vs-fixed-", suffix=suffix, dir=target_dir - ) - temp_path = Path(temp_name) - temp_paths.append(temp_path) - with os.fdopen(fd, "w", newline="", encoding="utf-8") as handle: - writer(handle) - handle.flush() - os.fsync(handle.fileno()) - os.link(temp_paths[0], json_path) - try: - os.link(temp_paths[1], csv_path) - except BaseException: - json_path.unlink() - raise - finally: - for temp_path in temp_paths: - temp_path.unlink(missing_ok=True) - return json_path, csv_path - - -def _study_rows(artifact: Mapping[str, Any]) -> Iterable[dict[str, Any]]: - for replicate in artifact.get("replicates", []): - for arm in ("adaptive", "fixed"): - failed = replicate.get("status") != "completed" - recovery = replicate.get(f"{arm}_recovery") or {} - post = replicate.get("shared_shock_episodes", []) + replicate.get(f"{arm}_episodes", []) - seed_map = replicate.get("seeds", {}) - values = { - "training_seed": replicate.get("training_seed"), - "arm": arm, - "replicate_status": replicate.get("status"), - "recovery_status": None if failed else recovery.get("status"), - "T_H": None if failed else recovery.get("T_H"), - "pre_returns": _json_cell( - [item.get("reward") for item in replicate.get("shared_pre_shift_episodes", [])] - ), - "shock_returns": _json_cell( - [item.get("reward") for item in replicate.get("shared_shock_episodes", [])] - ), - "post_returns": _json_cell([item.get("reward") for item in post]), - "pre_seeds": _json_cell(seed_map.get("pre", [])), - "shock_seeds": _json_cell(seed_map.get("post", [])[:5]), - "post_seeds": _json_cell(seed_map.get("post", [])), - "update_seeds": _json_cell(seed_map.get("update", [])), - "failure_reason": replicate.get("failure_reason"), - "json_trajectory_reference": ( - f"replicates[training_seed={replicate.get('training_seed')}].{arm}_episodes" - ), - } - yield values - - -def _json_cell(value: Any) -> str: - return json.dumps(_plain(value), separators=(",", ":"), allow_nan=False) - - -def _write_study_csv(handle: Any, artifact: Mapping[str, Any]) -> None: - writer = csv.DictWriter(handle, fieldnames=STUDY_CSV_FIELDS, extrasaction="raise") - writer.writeheader() - writer.writerows(_study_rows(artifact)) - - -def sha256_file(path: str | Path) -> str: - digest = hashlib.sha256() - with Path(path).open("rb") as handle: - for block in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def canonical_json_bytes(value: Any) -> bytes: - """Serialize JSON data canonically for semantic study identity.""" - return json.dumps( - _plain(value), - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - allow_nan=False, - ).encode("ascii") - - -def make_study_manifest(inputs: Mapping[str, Any]) -> dict[str, Any]: - """Build the immutable, pre-execution study identity document.""" - plain_inputs = _plain(inputs) - if not isinstance(plain_inputs, dict): - raise TypeError("study manifest inputs must be a mapping") - return { - "schema_version": STUDY_MANIFEST_SCHEMA_VERSION, - "study_hash": hashlib.sha256(canonical_json_bytes(plain_inputs)).hexdigest(), - "inputs": plain_inputs, - } - - -def write_or_verify_study_manifest( - inputs: Mapping[str, Any], manifest_path: str | Path, *, resume: bool -) -> str: - """Persist the expected study identity or fail closed when resuming.""" - manifest_path = Path(manifest_path) - expected = make_study_manifest(inputs) - if resume: - try: - existing = json.loads(manifest_path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ValueError("cannot resume without a valid immutable study manifest") from exc - if ( - not isinstance(existing, dict) - or existing.get("schema_version") != STUDY_MANIFEST_SCHEMA_VERSION - or not isinstance(existing.get("inputs"), dict) - ): - raise ValueError("study manifest has an invalid structure") - actual_hash = hashlib.sha256(canonical_json_bytes(existing["inputs"])).hexdigest() - if existing.get("study_hash") != actual_hash: - raise ValueError("study manifest hash is invalid") - if actual_hash != expected["study_hash"]: - raise ValueError("resume study hash mismatch: execution inputs changed") - return actual_hash - if manifest_path.exists(): - raise FileExistsError(f"refusing to overwrite study manifest: {manifest_path}") - manifest_path.parent.mkdir(parents=True, exist_ok=True) - encoded = (json.dumps(expected, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( - "utf-8" - ) - fd, temp_name = tempfile.mkstemp(prefix=".study-manifest-", dir=manifest_path.parent) - temp_path = Path(temp_name) - try: - with os.fdopen(fd, "wb") as handle: - handle.write(encoded) - handle.flush() - os.fsync(handle.fileno()) - os.link(temp_path, manifest_path) - finally: - temp_path.unlink(missing_ok=True) - return str(expected["study_hash"]) - - -def write_study_manifest( - artifact_path: str | Path, - csv_path: str | Path, - manifest_path: str | Path, - *, - run_id: str, - command: str, -) -> dict[str, Any]: - """Write an immutable provenance manifest for a completed study attempt.""" - artifact_path = Path(artifact_path) - csv_path = Path(csv_path) - manifest_path = Path(manifest_path) - if manifest_path.exists(): - raise FileExistsError(f"refusing to overwrite manifest: {manifest_path}") - try: - commit = subprocess.check_output( - ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL - ).strip() - dirty = bool( - subprocess.check_output( - ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL - ).strip() - ) - diff_hash = None - if dirty: - diff = subprocess.check_output(["git", "diff", "HEAD", "--binary"]) - diff_hash = hashlib.sha256(diff).hexdigest() - except (OSError, subprocess.CalledProcessError): - commit, dirty, diff_hash = None, None, None - artifact_files: list[Path] = [] - for path in sorted(manifest_path.parent.rglob("*")): - if path.is_symlink(): - raise ValueError(f"run directory contains a symlinked artifact: {path.name}") - if path.is_file() and path != manifest_path: - artifact_files.append(path) - manifest = { - "schema_version": STUDY_MANIFEST_SCHEMA_VERSION, - "study": "adaptive-vs-fixed/prereg-v1", - "run_id": run_id, - "commit_sha": commit, - "working_tree_dirty": dirty, - "dirty_diff_sha256": diff_hash, - "execution_command": command, - "hardware": { - "python": platform.python_version(), - "platform": platform.platform(), - "machine": platform.machine(), - "processor": platform.processor() or None, - "cpu_count": os.cpu_count(), - }, - "package_versions": { - name: _distribution_version(name) - for name in ("adaptive-rl", "gymnasium", "stable-baselines3", "torch", "numpy") - }, - "determinism": { - "pythonhashseed_env": os.environ.get("PYTHONHASHSEED"), - "torch_deterministic_algorithms": _torch_deterministic_algorithms(), - "torch_cudnn_deterministic": _torch_cudnn_deterministic(), - "torch_num_threads": _torch_num_threads(), - "cuda_available": _torch_cuda_available(), - "cuda_device_count": _torch_cuda_device_count(), - "protocol_seed_schedule": "SHA-256 derived seeds; see adaptive_vs_fixed.json", - }, - "artifacts": { - str(path.relative_to(manifest_path.parent).as_posix()): sha256_file(path) - for path in artifact_files - }, - } - study_spec_path = manifest_path.parent / "study_manifest.json" - if study_spec_path.is_file(): - study_spec = json.loads(study_spec_path.read_text(encoding="utf-8")) - manifest["study_hash"] = study_spec.get("study_hash") - plain = _plain(manifest) - encoded = json.dumps(plain, indent=2, allow_nan=False) + "\n" - manifest_path.parent.mkdir(parents=True, exist_ok=True) - fd, temp_name = tempfile.mkstemp(prefix=".manifest-", dir=manifest_path.parent) - temp_path = Path(temp_name) - try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(encoded) - handle.flush() - os.fsync(handle.fileno()) - os.link(temp_path, manifest_path) - finally: - temp_path.unlink(missing_ok=True) - return dict(plain) - - -def validate_study_manifest(manifest_path: str | Path) -> None: - """Raise when a listed immutable run artifact is missing or has changed.""" - manifest_path = Path(manifest_path) - if manifest_path.is_symlink(): - raise ValueError("study artifact manifest cannot be a symlink") - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if ( - not isinstance(manifest, dict) - or manifest.get("schema_version") != STUDY_MANIFEST_SCHEMA_VERSION - ): - raise ValueError("unsupported or malformed study artifact manifest") - artifacts = manifest.get("artifacts") - if not isinstance(artifacts, dict) or not artifacts: - raise ValueError("manifest must list at least one artifact checksum") - root = manifest_path.parent.resolve() - for relative_path, expected in artifacts.items(): - if ( - not isinstance(relative_path, str) - or not isinstance(expected, str) - or len(expected) != 64 - or any(character not in "0123456789abcdef" for character in expected) - ): - raise ValueError("manifest contains an invalid artifact path or SHA-256 digest") - relative = Path(relative_path) - if relative.is_absolute() or ".." in relative.parts: - raise ValueError(f"manifest artifact path escapes the run directory: {relative_path}") - path = manifest_path.parent / relative - try: - path.resolve().relative_to(root) - except (OSError, ValueError) as exc: - raise ValueError( - f"manifest artifact path escapes the run directory: {relative_path}" - ) from exc - if path.is_symlink(): - raise ValueError(f"manifest artifact is a symlink: {relative_path}") - if not path.is_file(): - raise ValueError(f"manifest artifact is missing: {relative_path}") - actual = sha256_file(path) - if actual != expected: - raise ValueError(f"manifest checksum mismatch: {relative_path}") - actual_artifacts = { - path.relative_to(manifest_path.parent).as_posix() - for path in manifest_path.parent.rglob("*") - if path != manifest_path and (path.is_file() or path.is_symlink()) - } - unlisted_artifacts = actual_artifacts - set(artifacts) - if unlisted_artifacts: - names = ", ".join(sorted(unlisted_artifacts)) - raise ValueError(f"run directory contains unlisted artifacts: {names}") - if "study_hash" in manifest: - spec_path = manifest_path.parent / "study_manifest.json" - if not spec_path.is_file() or "study_manifest.json" not in artifacts: - raise ValueError("study artifact manifest does not bind its pre-execution study spec") - spec = json.loads(spec_path.read_text(encoding="utf-8")) - if not isinstance(spec, dict) or not isinstance(spec.get("inputs"), dict): - raise ValueError("pre-execution study manifest is malformed") - actual_study_hash = hashlib.sha256(canonical_json_bytes(spec["inputs"])).hexdigest() - if spec.get("study_hash") != actual_study_hash or manifest.get("study_hash") != actual_study_hash: - raise ValueError("study artifact manifest hash does not match its pre-execution spec") - - -def write_replicate_checkpoint( - replicate: Mapping[str, Any], - checkpoint_path: str | Path, - *, - study_hash: str, - protocol_hash: str, - artifact_root: str | Path | None = None, - artifact_directories: Sequence[str | Path] = (), -) -> None: - """Persist a terminal replicate bound to its study and replicate identity.""" - checkpoint_path = Path(checkpoint_path) - if replicate.get("status") not in {"completed", "failed"}: - raise ValueError("only terminal replicate states may be checkpointed") - if artifact_directories and artifact_root is None: - raise ValueError("artifact_root is required when checkpoint artifacts are declared") - plain = _plain(dict(replicate)) - artifact_integrity = ( - _snapshot_artifact_directories(artifact_root, artifact_directories) - if artifact_root is not None - else [] - ) - envelope = { - "schema_version": REPLICATE_CHECKPOINT_SCHEMA_VERSION, - "study_hash": study_hash, - "protocol_hash": protocol_hash, - "replicate_id": plain.get("training_seed"), - "artifact_integrity": artifact_integrity, - "replicate": plain, - } - encoded = (json.dumps(envelope, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( - "utf-8" - ) - checkpoint_path.parent.mkdir(parents=True, exist_ok=True) - fd, temp_name = tempfile.mkstemp(prefix=".replicate-", dir=checkpoint_path.parent) - temp_path = Path(temp_name) - digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") - installed_checkpoint = False - try: - with os.fdopen(fd, "wb") as handle: - handle.write(encoded) - handle.flush() - os.fsync(handle.fileno()) - os.link(temp_path, checkpoint_path) - installed_checkpoint = True - digest = hashlib.sha256(encoded).hexdigest() - digest_temp_fd, digest_temp_name = tempfile.mkstemp( - prefix=".replicate-digest-", dir=checkpoint_path.parent - ) - digest_temp_path = Path(digest_temp_name) - try: - with os.fdopen(digest_temp_fd, "w", encoding="ascii") as handle: - handle.write(f"{digest}\n") - handle.flush() - os.fsync(handle.fileno()) - os.link(digest_temp_path, digest_path) - finally: - digest_temp_path.unlink(missing_ok=True) - except BaseException: - if installed_checkpoint: - checkpoint_path.unlink(missing_ok=True) - raise - finally: - temp_path.unlink(missing_ok=True) - - -def read_replicate_checkpoint( - checkpoint_path: str | Path, - *, - study_hash: str, - protocol_hash: str, - training_seed: int, -) -> dict[str, Any]: - """Read a terminal checkpoint after digest, study, protocol, and ID checks.""" - checkpoint_path = Path(checkpoint_path) - digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") - payload = checkpoint_path.read_bytes() - if not digest_path.is_file(): - raise ValueError(f"replicate checkpoint digest is missing: {checkpoint_path.name}") - expected = digest_path.read_text(encoding="ascii").strip() - actual = hashlib.sha256(payload).hexdigest() - if actual != expected: - raise ValueError(f"replicate checkpoint checksum mismatch: {checkpoint_path.name}") - envelope = json.loads(payload) - if ( - not isinstance(envelope, dict) - or envelope.get("schema_version") != REPLICATE_CHECKPOINT_SCHEMA_VERSION - or not isinstance(envelope.get("replicate"), dict) - ): - raise ValueError("replicate checkpoint envelope is malformed") - if envelope.get("study_hash") != study_hash: - raise ValueError("replicate checkpoint study hash mismatch") - if envelope.get("protocol_hash") != protocol_hash: - raise ValueError("replicate checkpoint protocol hash mismatch") - replicate = envelope["replicate"] - _validate_checkpoint_artifacts( - checkpoint_path.parent.parent, - envelope.get("artifact_integrity"), - ) - if envelope.get("replicate_id") != training_seed or replicate.get("training_seed") != training_seed: - raise ValueError("replicate checkpoint identity mismatch") - result = replicate - if result.get("status") not in {"completed", "failed"}: - raise ValueError("replicate checkpoint is not terminal") - return cast(dict[str, Any], result) - - -def _snapshot_artifact_directories( - artifact_root: str | Path, - artifact_directories: Sequence[str | Path], -) -> list[dict[str, Any]]: - root = Path(artifact_root).resolve() - snapshots: list[dict[str, Any]] = [] - seen_directories: set[str] = set() - for directory_value in artifact_directories: - directory = Path(directory_value) - if not directory.is_absolute(): - directory = root / directory - if directory.is_symlink(): - raise ValueError(f"checkpoint artifact directory is a symlink: {directory}") - resolved = directory.resolve() - try: - relative_directory = resolved.relative_to(root).as_posix() - except ValueError as exc: - raise ValueError(f"checkpoint artifact directory escapes its run: {directory}") from exc - if relative_directory in seen_directories: - raise ValueError(f"duplicate checkpoint artifact directory: {relative_directory}") - seen_directories.add(relative_directory) - exists = directory.exists() - if exists and not directory.is_dir(): - raise ValueError(f"checkpoint artifact path is not a directory: {directory}") - files: dict[str, str] = {} - if exists: - for path in sorted(directory.rglob("*")): - if path.is_symlink(): - raise ValueError(f"checkpoint artifact contains a symlink: {path}") - if path.is_file(): - relative_path = path.resolve().relative_to(root).as_posix() - files[relative_path] = sha256_file(path) - snapshots.append( - {"path": relative_directory, "exists": exists, "files": files} - ) - return snapshots - - -def _validate_checkpoint_artifacts(artifact_root: str | Path, snapshots: Any) -> None: - if not isinstance(snapshots, list): - raise ValueError("replicate checkpoint artifact integrity is malformed") - root = Path(artifact_root).resolve() - seen_directories: set[str] = set() - for snapshot in snapshots: - if not isinstance(snapshot, dict): - raise ValueError("replicate checkpoint artifact directory record is malformed") - relative_directory = snapshot.get("path") - exists = snapshot.get("exists") - files = snapshot.get("files") - if ( - not isinstance(relative_directory, str) - or not relative_directory - or Path(relative_directory).is_absolute() - or ".." in Path(relative_directory).parts - or not isinstance(exists, bool) - or not isinstance(files, dict) - or relative_directory in seen_directories - ): - raise ValueError("replicate checkpoint artifact directory record is invalid") - seen_directories.add(relative_directory) - directory = root / relative_directory - try: - directory.resolve().relative_to(root) - except (OSError, ValueError) as exc: - raise ValueError("replicate checkpoint artifact path escapes its run") from exc - if not exists: - if directory.exists() or directory.is_symlink(): - raise ValueError("checkpoint artifact directory appeared after checkpointing") - if files: - raise ValueError("absent checkpoint artifact directory has recorded files") - continue - if directory.is_symlink() or not directory.is_dir(): - raise ValueError("checkpoint artifact directory is missing or changed type") - expected_files: set[str] = set() - for relative_path, expected_hash in files.items(): - relative = Path(relative_path) if isinstance(relative_path, str) else Path("/") - if ( - not isinstance(relative_path, str) - or relative.is_absolute() - or ".." in relative.parts - or not isinstance(expected_hash, str) - or len(expected_hash) != 64 - or any(character not in "0123456789abcdef" for character in expected_hash) - ): - raise ValueError("checkpoint artifact file record is invalid") - if relative.parts[: len(Path(relative_directory).parts)] != Path( - relative_directory - ).parts: - raise ValueError("checkpoint artifact file is outside its recorded directory") - path = root / relative - try: - path.resolve().relative_to(root) - except (OSError, ValueError) as exc: - raise ValueError("checkpoint artifact file escapes its run") from exc - if path.is_symlink() or not path.is_file(): - raise ValueError(f"checkpoint artifact file is missing or changed: {relative_path}") - if sha256_file(path) != expected_hash: - raise ValueError(f"checkpoint artifact checksum mismatch: {relative_path}") - expected_files.add(relative_path) - actual_files: set[str] = set() - for path in directory.rglob("*"): - if path.is_symlink(): - raise ValueError(f"checkpoint artifact directory contains a symlink: {path}") - if path.is_file(): - actual_files.add(path.resolve().relative_to(root).as_posix()) - if actual_files != expected_files: - raise ValueError("checkpoint artifact file set changed after checkpointing") - - -def _distribution_version(name: str) -> str | None: - try: - return importlib.metadata.version(name) - except importlib.metadata.PackageNotFoundError: - return None - - -def _torch_deterministic_algorithms() -> bool | None: - try: - import torch - - return bool(torch.are_deterministic_algorithms_enabled()) - except ImportError: - return None - - -def _torch_cudnn_deterministic() -> bool | None: - try: - import torch - - return bool(torch.backends.cudnn.deterministic) - except ImportError: - return None - - -def _torch_num_threads() -> int | None: - try: - import torch - - return int(torch.get_num_threads()) - except ImportError: - return None - - -def _torch_cuda_available() -> bool | None: - try: - import torch - - return bool(torch.cuda.is_available()) - except ImportError: - return None - - -def _torch_cuda_device_count() -> int | None: - try: - import torch - - return int(torch.cuda.device_count()) - except ImportError: - return None - - -def _write_csv(handle: Any, artifact: Mapping[str, Any]) -> None: - writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS, extrasaction="raise") - writer.writeheader() - writer.writerows(_episode_rows(artifact)) - - -__all__ = [ - "CSV_FIELDS", - "STUDY_CSV_FIELDS", - "canonical_json_bytes", - "make_study_manifest", - "sha256_file", - "read_replicate_checkpoint", - "validate_study_manifest", - "write_adaptation_artifacts", - "write_adaptive_vs_fixed_artifacts", - "write_replicate_checkpoint", - "write_or_verify_study_manifest", - "write_study_manifest", -] diff --git a/src/adaptive_rl/benchmarking/adaptation_runner.py b/src/adaptive_rl/benchmarking/adaptation_runner.py deleted file mode 100644 index 27cf2b8..0000000 --- a/src/adaptive_rl/benchmarking/adaptation_runner.py +++ /dev/null @@ -1,1053 +0,0 @@ -"""Protocol-ordered Issue #265 train/freeze/share/fork experiment runner.""" - -from __future__ import annotations - -import hashlib -import importlib.metadata -import json -import logging -import os -import platform -import subprocess -import sys -import time -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Any, Callable, Optional, Sequence - -import gymnasium as gym -import numpy as np -import torch - -from adaptive_rl.algorithms.adaptation import ( - PPOAdaptationAdapter, - SACAdaptationAdapter, - run_adaptation_update, -) -from adaptive_rl.benchmarking.adaptation_artifacts import ( - REPLICATE_CHECKPOINT_SCHEMA_VERSION, - STUDY_ARTIFACT_SCHEMA_VERSION, - STUDY_MANIFEST_SCHEMA_VERSION, - canonical_json_bytes, - read_replicate_checkpoint, - validate_study_manifest, - write_adaptation_artifacts, - write_adaptive_vs_fixed_artifacts, - write_or_verify_study_manifest, - write_replicate_checkpoint, - write_study_manifest, -) -from adaptive_rl.benchmarking.adaptation_runtime import EpisodeRecord, evaluate_episode -from adaptive_rl.benchmarking.adaptation_statistics import analyze_primary_cells -from adaptive_rl.config import ExperimentConfig, compute_config_sha256 -from adaptive_rl.environments.registry import make_env -from adaptive_rl.protocol.adaptation import ( - AdaptationAdapter, - Transition, - build_update_batch, - validate_block_sequence, -) -from adaptive_rl.protocol.constants import ( - ISSUE271_CONFIG_SHA256, - ISSUE271_TREATMENT_CARD_SHA256, - K_PRE, - N_POST, - PRIMARY_CELLS, - TRAINING_SEEDS, -) -from adaptive_rl.protocol.fork import fork_adaptive_and_fixed, model_fingerprint -from adaptive_rl.protocol.recovery import compute_recovery -from adaptive_rl.protocol.seeds import frozen_schedule, schedule_fingerprint -from adaptive_rl.protocol.statistics import decide_family -from adaptive_rl.training.trainer import get_trainer - -logger = logging.getLogger(__name__) - -EnvironmentFactory = Callable[..., gym.Env] -TrainerFactory = Callable[..., Any] - - -def _repository_metadata() -> dict[str, Any]: - commit: str | None - dirty: bool | None - try: - commit = subprocess.check_output( - ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL - ).strip() - dirty = bool( - subprocess.check_output( - ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL - ).strip() - ) - except (OSError, subprocess.CalledProcessError): - commit, dirty = None, None - versions: dict[str, str | None] = { - "python": sys.version.split()[0], - "platform": platform.platform(), - } - for distribution in ("adaptive-rl", "gymnasium", "stable-baselines3", "torch", "numpy"): - try: - versions[distribution] = importlib.metadata.version(distribution) - except importlib.metadata.PackageNotFoundError: - versions[distribution] = None - runtime_settings: dict[str, Any] = { - name: os.environ.get(name) - for name in ( - "PYTHONHASHSEED", - "CUBLAS_WORKSPACE_CONFIG", - "CUDA_VISIBLE_DEVICES", - "OMP_NUM_THREADS", - "MKL_NUM_THREADS", - "OPENBLAS_NUM_THREADS", - ) - } - runtime_settings.update( - { - "torch_num_threads": torch.get_num_threads(), - "cuda_available": torch.cuda.is_available(), - "cuda_device_count": torch.cuda.device_count(), - "cuda_device_names": [ - torch.cuda.get_device_name(index) - for index in range(torch.cuda.device_count()) - ], - } - ) - return { - "repository_commit": commit, - "working_tree_dirty": dirty, - "runtime_versions": versions, - "runtime_settings": runtime_settings, - } - - -def _sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for block in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - -def _card_path() -> Path: - return Path(__file__).resolve().parents[3] / "docs" / "research" / "TREATMENT_CARD.md" - - -def _new_env( - config: ExperimentConfig, - *, - shifted: bool, - environment_factory: EnvironmentFactory, - max_steps: Optional[int] = None, -) -> gym.Env: - parameters = dict(config.environment.parameters) - parameters["max_steps"] = int(max_steps or config.environment.max_steps) - if shifted: - benchmark = config.adaptation_benchmark - if benchmark is None: - raise ValueError("configuration does not declare the Issue #265 adaptation cell") - parameters.update(benchmark.shift_parameters) - return environment_factory(config.environment.name, **parameters) - - -def _adapter_for(algorithm_name: str) -> AdaptationAdapter: - if algorithm_name == "ppo": - return PPOAdaptationAdapter() - if algorithm_name == "sac": - return SACAdaptationAdapter() - raise ValueError(f"Issue #265 supports PPO and SAC, got {algorithm_name!r}") - - -def _train_once( - config: ExperimentConfig, - training_seed: int, - training_dir: Path, - *, - trainer_factory: TrainerFactory, - environment_factory: EnvironmentFactory, - smoke: bool, -) -> tuple[Any, dict[str, Any]]: - if config.training is None: - raise ValueError("Issue #265 requires a training configuration") - if training_dir.exists(): - raise FileExistsError(f"training output already exists: {training_dir}") - training_dir.mkdir(parents=True) - - effective = config.model_copy(deep=True) - training = effective.training - if training is None: - raise ValueError("Issue #265 requires a training configuration") - effective.seed = int(training_seed) - effective.name = f"{config.name}_seed_{training_seed}" - effective.output_dir = training_dir - effective.log_dir = training_dir / "logs" - # The training component does not receive the TEST-B scenario payload. - effective.adaptation_benchmark = None - algorithm_config = effective.algorithm.model_copy(deep=True) - algorithm_parameters = dict(algorithm_config.parameters) - algorithm_parameters["seed"] = int(training_seed) - if smoke: - training.total_timesteps = 32 - if algorithm_config.name.lower() == "ppo": - algorithm_parameters.update({"n_steps": 16, "n_epochs": 1}) - algorithm_config.batch_size = min(8, algorithm_config.batch_size) - else: - algorithm_parameters.update( - {"learning_starts": 1, "gradient_steps": 1, "buffer_size": 64} - ) - algorithm_config.batch_size = min(8, algorithm_config.batch_size) - algorithm_config.parameters = algorithm_parameters - effective.algorithm = algorithm_config - training_env = _new_env( - effective, - shifted=False, - environment_factory=environment_factory, - max_steps=8 if smoke else None, - ) - trainer = None - started = time.perf_counter() - try: - trainer = trainer_factory(config=effective, env=training_env) - result = trainer.fit() - algorithm = trainer.algorithm - except BaseException: - if trainer is not None: - trainer.close() - else: - training_env.close() - raise - else: - training_seconds = float(time.perf_counter() - started) - trainer.close() - return algorithm, { - "training_seed": training_seed, - "total_timesteps_requested": training.total_timesteps, - "total_timesteps_completed": int(getattr(algorithm, "num_timesteps", 0)), - "training_time_seconds": training_seconds, - "model_path": str(result.final_model_path), - "training_metadata_path": str(result.metadata_path) if result.metadata_path else None, - "episodes_completed": int(result.episodes_completed), - "mean_reward": float(result.mean_reward), - "smoke_override": bool(smoke), - "effective_config": effective.model_dump(mode="json"), - } - - -def _run_evaluation_segment( - *, - algorithm: Any, - env: gym.Env, - training_seed: int, - algorithm_name: str, - environment_name: str, - phase: str, - indices: Sequence[int], - seeds: Sequence[int], - deterministic: bool, - arm: str, - initial_update_log: Any = None, -) -> list[EpisodeRecord]: - if len(indices) != len(seeds): - raise ValueError("episode indices and seeds must have matching lengths") - records: list[EpisodeRecord] = [] - update_log = initial_update_log - for episode_index, episode_seed in zip(indices, seeds): - records.append( - evaluate_episode( - algorithm=algorithm, - env=env, - training_seed=training_seed, - phase=phase, - episode_index=episode_index, - episode_seed=episode_seed, - algorithm_name=algorithm_name, - environment_name=environment_name, - deterministic=deterministic, - arm=arm, - update_log=update_log, - ) - ) - update_log = None - return records - - -def _recover(pre: Sequence[EpisodeRecord], post: Sequence[EpisodeRecord]) -> dict[str, Any]: - result = compute_recovery( - [episode.reward for episode in pre], [episode.reward for episode in post] - ) - output = asdict(result) - output["T_H"] = result.truncated_recovery_time - return output - - -def _return_vector_fingerprint(episodes: Sequence[dict[str, Any]]) -> str: - payload = json.dumps([float(item["reward"]) for item in episodes], separators=(",", ":")) - return hashlib.sha256(payload.encode("ascii")).hexdigest() - - -def _enable_study_determinism() -> dict[str, Any]: - """Enable deterministic Torch behavior where supported and report limits.""" - torch.use_deterministic_algorithms(True, warn_only=True) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - return { - "torch_deterministic_algorithms": torch.are_deterministic_algorithms_enabled(), - "torch_deterministic_warn_only": True, - "cudnn_deterministic": bool(torch.backends.cudnn.deterministic), - "cudnn_benchmark": bool(torch.backends.cudnn.benchmark), - "limitations": [ - "warn-only Torch operations may remain nondeterministic", - "cross-hardware and cross-library-version bitwise identity is not claimed", - "training and update RNGs are reseeded from the recorded protocol seeds", - ], - } - - -def _audit_replicate_invariants( - replicate: dict[str, Any], schedule: dict[int, dict[str, list[int]]] -) -> dict[str, Any]: - """Recompute the execution invariants from one serialized replicate record.""" - blocks = replicate["update_blocks"] - arms = replicate["arm_input_fingerprints"] - pre_hash = _return_vector_fingerprint(replicate["shared_pre_shift_episodes"]) - shock_hash = _return_vector_fingerprint(replicate["shared_shock_episodes"]) - checks = { - "pre_shift_shared_once": len(replicate["shared_pre_shift_episodes"]) == K_PRE, - "pre_shift_identical_across_arms": ( - arms["adaptive"]["pre"] == pre_hash == arms["fixed"]["pre"] - ), - "shock_shared_once": len(replicate["shared_shock_episodes"]) == 5, - "shock_returns_identical_across_arms": ( - arms["adaptive"]["shock"] == shock_hash == arms["fixed"]["shock"] - ), - "fixed_zero_updates": replicate["fixed_weight_update_count"] == 0, - "fixed_l2_delta_zero": replicate["fixed_parameter_delta_l2"] == 0.0, - "fixed_fingerprint_unchanged": ( - replicate["fixed_final_fingerprint"] == replicate["frozen_fingerprint"] - and all( - episode["policy_fingerprint_start"] == replicate["frozen_fingerprint"] - and episode["policy_fingerprint_end"] == replicate["frozen_fingerprint"] - for episode in replicate["fixed_episodes"] - ) - ), - "adaptive_exactly_ten_blocks": len(blocks) == 10, - "adaptive_block_schedule": [b["block_episode"] for b in blocks] == list(range(5, 15)), - "no_future_data": all( - list(b["visible_episode_indices"]) == list(range(1, b["block_episode"] + 1)) - for b in blocks - ), - "adaptive_update_seeds_match_schedule": [b["update_seed"] for b in blocks] - == replicate["seeds"]["update"], - "adaptive_blocks_have_loss_metrics": all(b["loss_metrics"] for b in blocks), - "adaptive_updates_between_episodes": [ - episode["update_block"] for episode in replicate["adaptive_episodes"] - ] - == list(range(5, 15)), - "fork_fingerprint_identical": replicate["fork_fingerprint"] - == replicate["frozen_fingerprint"], - "all_seeds_match_schedule": replicate["seeds"] - == {phase: list(values) for phase, values in schedule[replicate["training_seed"]].items()}, - } - return {"all_passed": all(checks.values()), **checks} - - -@dataclass -class ReplicateResult: - training_seed: int - status: str - schedule_fingerprint: Optional[str] = None - training_provenance: dict[str, Any] = field(default_factory=dict) - frozen_fingerprint: Optional[str] = None - fork_fingerprint: Optional[str] = None - fixed_final_fingerprint: Optional[str] = None - pre_shift_performance: Optional[float] = None - shock_performance: Optional[float] = None - shared_pre_shift_episodes: list[EpisodeRecord] = field(default_factory=list) - shared_shock_episodes: list[EpisodeRecord] = field(default_factory=list) - adaptive_episodes: list[EpisodeRecord] = field(default_factory=list) - fixed_episodes: list[EpisodeRecord] = field(default_factory=list) - update_blocks: list[dict[str, Any]] = field(default_factory=list) - adaptive_recovery: Optional[dict[str, Any]] = None - fixed_recovery: Optional[dict[str, Any]] = None - seeds: dict[str, list[int]] = field(default_factory=dict) - arm_input_fingerprints: dict[str, dict[str, str]] = field(default_factory=dict) - fixed_weight_update_count: int = 0 - fixed_parameter_delta_l2: float = 0.0 - effective_nominal_parameters: dict[str, Any] = field(default_factory=dict) - effective_shift_parameters: dict[str, Any] = field(default_factory=dict) - failure_reason: Optional[str] = None - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -def _restore_replicate(data: dict[str, Any]) -> ReplicateResult: - """Restore only records already accepted by the checkpoint digest check.""" - episode_fields = ( - "shared_pre_shift_episodes", - "shared_shock_episodes", - "adaptive_episodes", - "fixed_episodes", - ) - restored = dict(data) - for field_name in episode_fields: - episodes = [] - for episode_data in restored.get(field_name, []): - episode = dict(episode_data) - episode["transitions"] = tuple( - Transition(**transition) for transition in episode.get("transitions", []) - ) - episodes.append(EpisodeRecord(**episode)) - restored[field_name] = episodes - return ReplicateResult(**restored) - - -def _resume_replicate_checkpoint( - checkpoint_path: Path, - *, - study_hash: str, - protocol_hash: str, - training_seed: int, -) -> ReplicateResult: - """Restore one authenticated terminal replicate or fail the whole resume.""" - try: - checkpoint = read_replicate_checkpoint( - checkpoint_path, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seed=training_seed, - ) - return _restore_replicate(checkpoint) - except (OSError, TypeError, ValueError, KeyError) as exc: - raise ValueError( - f"resume rejected invalid checkpoint for seed {training_seed}: {exc}" - ) from exc - - -def _load_resume_replicates( - state_root: Path, - *, - study_hash: str, - protocol_hash: str, - training_seeds: Sequence[int], -) -> dict[int, ReplicateResult]: - """Validate every stored checkpoint before resuming any unfinished seed.""" - if state_root.is_symlink(): - raise ValueError("resume replicate state path is not a regular directory") - if not state_root.exists(): - return {} - if not state_root.is_dir(): - raise ValueError("resume replicate state path is not a regular directory") - allowed_names = { - name - for seed in training_seeds - for name in (f"seed_{seed}.json", f"seed_{seed}.json.sha256") - } - for path in state_root.iterdir(): - if path.name not in allowed_names or path.is_symlink() or not path.is_file(): - raise ValueError(f"resume replicate state contains an unexpected entry: {path.name}") - - restored: dict[int, ReplicateResult] = {} - for seed in training_seeds: - checkpoint_path = state_root / f"seed_{seed}.json" - digest_path = checkpoint_path.with_suffix(checkpoint_path.suffix + ".sha256") - if checkpoint_path.exists() != digest_path.exists(): - raise ValueError(f"resume checkpoint or digest is missing for seed {seed}") - if checkpoint_path.exists(): - restored[seed] = _resume_replicate_checkpoint( - checkpoint_path, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seed=seed, - ) - return restored - - -def _read_completed_study_artifact( - artifact_path: Path, - manifest_path: Path, - *, - run_id: str, - study_hash: str, -) -> dict[str, Any]: - """Return the persisted result only after validating its immutable envelope.""" - validate_study_manifest(manifest_path) - try: - artifact = json.loads(artifact_path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ValueError("completed study artifact is malformed") from exc - artifact_paths = artifact.get("artifact_paths") if isinstance(artifact, dict) else None - if ( - not isinstance(artifact, dict) - or artifact.get("schema_version") != STUDY_ARTIFACT_SCHEMA_VERSION - or artifact.get("run_id") != run_id - or artifact.get("study_hash") != study_hash - or artifact.get("run_status") not in {"COMPLETE", "PARTIAL"} - or artifact_paths - != { - "json": "adaptive_vs_fixed.json", - "csv": "adaptive_vs_fixed.csv", - "manifest": "manifest.json", - } - ): - raise ValueError("completed study artifact has an invalid schema or study identity") - return artifact - - -def _run_replicate( - config: ExperimentConfig, - training_seed: int, - training_dir: Path, - *, - schedule: dict[int, dict[str, list[int]]], - smoke: bool, - trainer_factory: TrainerFactory, - environment_factory: EnvironmentFactory, -) -> ReplicateResult: - result = ReplicateResult(training_seed=training_seed, status="failed") - try: - benchmark = config.adaptation_benchmark - if benchmark is None: - raise ValueError("configuration lacks an Issue #265 adaptation cell") - result.schedule_fingerprint = schedule_fingerprint(schedule) - algorithm, training_provenance = _train_once( - config, - training_seed, - training_dir, - trainer_factory=trainer_factory, - environment_factory=environment_factory, - smoke=smoke, - ) - result.training_provenance = training_provenance - frozen_fingerprint = model_fingerprint(algorithm) - result.frozen_fingerprint = frozen_fingerprint - algorithm.model.policy.set_training_mode(False) - - nominal_params = dict(config.environment.parameters) - nominal_params["max_steps"] = 8 if smoke else config.environment.max_steps - shifted_params = dict(nominal_params) - shifted_params.update(benchmark.shift_parameters) - result.effective_nominal_parameters = dict(nominal_params) - result.effective_shift_parameters = dict(shifted_params) - result.seeds = {phase: list(values) for phase, values in schedule[training_seed].items()} - - nominal_env = _new_env( - config, - shifted=False, - environment_factory=environment_factory, - max_steps=8 if smoke else None, - ) - try: - result.shared_pre_shift_episodes = _run_evaluation_segment( - algorithm=algorithm, - env=nominal_env, - training_seed=training_seed, - algorithm_name=config.algorithm.name.lower(), - environment_name=config.environment.name, - phase="pre", - indices=range(1, K_PRE + 1), - seeds=schedule[training_seed]["pre"], - deterministic=config.evaluation.deterministic, - arm="shared", - ) - finally: - nominal_env.close() - if model_fingerprint(algorithm) != frozen_fingerprint: - raise RuntimeError("shared pre-shift evaluation mutated the frozen policy") - result.pre_shift_performance = float( - np.mean([episode.reward for episode in result.shared_pre_shift_episodes]) - ) - - # Shift introduction occurs only after training and the one shared pre segment. - shift_before = model_fingerprint(algorithm) - shock_env = _new_env( - config, - shifted=True, - environment_factory=environment_factory, - max_steps=8 if smoke else None, - ) - try: - get_effective = getattr(shock_env, "get_effective_parameters", None) - effective = dict(get_effective()) if callable(get_effective) else shifted_params - for key, expected in benchmark.shift_parameters.items(): - if effective.get(key) != expected: - raise RuntimeError( - f"TEST-B parameter {key!r} did not apply: expected {expected!r}, " - f"got {effective.get(key)!r}" - ) - result.effective_shift_parameters = effective - result.shared_shock_episodes = _run_evaluation_segment( - algorithm=algorithm, - env=shock_env, - training_seed=training_seed, - algorithm_name=config.algorithm.name.lower(), - environment_name=config.environment.name, - phase="post", - indices=range(1, 6), - seeds=schedule[training_seed]["post"][:5], - deterministic=config.evaluation.deterministic, - arm="shared", - ) - finally: - shock_env.close() - if model_fingerprint(algorithm) != shift_before: - raise RuntimeError("shift introduction or shared shock evaluation mutated the policy") - result.shock_performance = float( - np.mean([episode.reward for episode in result.shared_shock_episodes]) - ) - - adaptive, fixed, fork_fingerprint = fork_adaptive_and_fixed(algorithm) - result.fork_fingerprint = fork_fingerprint - if fork_fingerprint != frozen_fingerprint: - raise RuntimeError("Adaptive/Fixed forks did not originate at the frozen fingerprint") - adapter = _adapter_for(config.algorithm.name.lower()) - post_history = [episode.post_shift_data() for episode in result.shared_shock_episodes] - adaptive_env = _new_env( - config, - shifted=True, - environment_factory=environment_factory, - max_steps=8 if smoke else None, - ) - try: - for episode_index in range(6, N_POST + 1): - boundary = episode_index - 1 - batch = build_update_batch(training_seed, post_history, block_episode=boundary) - update_log = run_adaptation_update(adaptive, adapter, batch) - result.update_blocks.append(update_log.to_dict()) - record = _run_evaluation_segment( - algorithm=adaptive, - env=adaptive_env, - training_seed=training_seed, - algorithm_name=config.algorithm.name.lower(), - environment_name=config.environment.name, - phase="post", - indices=[episode_index], - seeds=[schedule[training_seed]["post"][episode_index - 1]], - deterministic=config.evaluation.deterministic, - arm="adaptive", - initial_update_log=update_log, - )[0] - result.adaptive_episodes.append(record) - post_history.append(record.post_shift_data()) - finally: - adaptive_env.close() - validate_block_sequence([block["block_episode"] for block in result.update_blocks]) - - fixed_env = _new_env( - config, - shifted=True, - environment_factory=environment_factory, - max_steps=8 if smoke else None, - ) - try: - result.fixed_episodes = _run_evaluation_segment( - algorithm=fixed, - env=fixed_env, - training_seed=training_seed, - algorithm_name=config.algorithm.name.lower(), - environment_name=config.environment.name, - phase="post", - indices=range(6, N_POST + 1), - seeds=schedule[training_seed]["post"][5:], - deterministic=config.evaluation.deterministic, - arm="fixed", - ) - finally: - fixed_env.close() - result.fixed_final_fingerprint = fixed.fingerprint - if result.fixed_final_fingerprint != frozen_fingerprint: - raise RuntimeError("Fixed policy fingerprint changed during its evaluation arm") - - adaptive_post = result.shared_shock_episodes + result.adaptive_episodes - fixed_post = result.shared_shock_episodes + result.fixed_episodes - result.adaptive_recovery = _recover(result.shared_pre_shift_episodes, adaptive_post) - result.fixed_recovery = _recover(result.shared_pre_shift_episodes, fixed_post) - pre_hash = _return_vector_fingerprint( - [{"reward": episode.reward} for episode in result.shared_pre_shift_episodes] - ) - shock_hash = _return_vector_fingerprint( - [{"reward": episode.reward} for episode in result.shared_shock_episodes] - ) - result.arm_input_fingerprints = { - arm: {"pre": pre_hash, "shock": shock_hash} for arm in ("adaptive", "fixed") - } - result.status = "completed" - except Exception as exc: - result.failure_reason = f"{type(exc).__name__}: {exc}" - logger.exception("Issue #265 replicate failed for training seed %s", training_seed) - return result - - -def run_adaptation_benchmark( - config: ExperimentConfig, - *, - output_dir: str | Path | None = None, - training_seeds: Optional[Sequence[int]] = None, - smoke: bool = False, - trainer_factory: TrainerFactory = get_trainer, - environment_factory: EnvironmentFactory = make_env, - config_path: str | Path | None = None, - study_run_id: str | None = None, - resume: bool = False, -) -> dict[str, Any]: - """Run the selected preregistered replicates and write JSON/CSV artifacts. - - A smoke run is visibly marked and uses one preregistered seed, a tiny - training budget, and short episodes. It validates execution only; its - outputs are not empirical research data. - """ - if config.adaptation_benchmark is None: - raise ValueError("configuration must include the Issue #265 adaptation_benchmark section") - benchmark = config.adaptation_benchmark - if config.algorithm.name.strip().lower() not in {"ppo", "sac"}: - raise ValueError("Issue #265 supports only PPO and SAC") - if config.training is None: - raise ValueError("Issue #265 requires a training section") - card = _card_path() - if not card.is_file(): - raise FileNotFoundError(f"Treatment Card is required before experiment execution: {card}") - card_sha = _sha256_file(card) - repository_root = Path.cwd().resolve() - config_arg = ( - Path(config_path).as_posix() - if config_path is not None - else "configs/drone_distribution_shift.yaml" - ) - schedule = frozen_schedule() - schedule_fp = schedule_fingerprint(schedule) - if training_seeds is None: - selected_seeds = [TRAINING_SEEDS[0]] if smoke else list(TRAINING_SEEDS) - else: - selected_seeds = [int(seed) for seed in training_seeds] - if not selected_seeds or len(set(selected_seeds)) != len(selected_seeds): - raise ValueError("training_seeds must be non-empty and unique") - if not set(selected_seeds).issubset(TRAINING_SEEDS): - raise ValueError("all selected training seeds must come from TRAINING_SEEDS") - if smoke and len(selected_seeds) != 1: - raise ValueError("smoke mode runs exactly one preregistered training seed") - if study_run_id is not None: - if not study_run_id or Path(study_run_id).name != study_run_id: - raise ValueError("study_run_id must be a non-empty filename-safe component") - if smoke or selected_seeds != list(TRAINING_SEEDS): - raise ValueError("prereg-v1 requires one non-smoke attempt of all ten seeds in order") - if output_dir is not None and Path(output_dir).is_absolute(): - raise ValueError("prereg-v1 artifact output_dir must be relative to the repository") - selected_output_dir = Path(output_dir) if output_dir is not None else Path(config.output_dir) - try: - selected_output_dir.resolve().relative_to(repository_root) - except ValueError as exc: - raise ValueError("prereg-v1 artifact output_dir must stay inside the repository") from exc - if Path(config_arg).is_absolute(): - raise ValueError("prereg-v1 config path must be repository-relative") - try: - (repository_root / config_arg).resolve().relative_to(repository_root) - except ValueError as exc: - raise ValueError("prereg-v1 config path must stay inside the repository") from exc - if config.algorithm.name.strip().lower() != "ppo": - raise ValueError("prereg-v1 is frozen to the drone_disturbed/ppo cell") - if config.environment.name != "drone_disturbed" or benchmark.scenario != "TEST-B": - raise ValueError("prereg-v1 is frozen to drone_disturbed under TEST-B") - if compute_config_sha256(config) != ISSUE271_CONFIG_SHA256: - raise ValueError("prereg-v1 config differs from the frozen Issue #271 configuration") - if card_sha != ISSUE271_TREATMENT_CARD_SHA256: - raise ValueError("prereg-v1 Treatment Card differs from the frozen treatment") - if ( - config.algorithm.name.strip().lower() == "ppo" - and config.evaluation.deterministic - ): - raise ValueError( - "PPO adaptation requires stochastic behavior-policy action sampling; " - "deterministic mean actions are not valid on-policy rollout data" - ) - if study_run_id is not None: - try: - dirty = subprocess.check_output( - ["git", "status", "--porcelain"], text=True, stderr=subprocess.DEVNULL - ).strip() - except (OSError, subprocess.CalledProcessError) as exc: - raise RuntimeError( - "cannot verify clean working tree before prereg-v1 execution" - ) from exc - if dirty: - raise RuntimeError("prereg-v1 execution requires a clean, committed working tree") - determinism = _enable_study_determinism() if study_run_id is not None else None - - config_file = Path(config_path).resolve() if config_path is not None else None - config_file_sha = _sha256_file(config_file) if config_file is not None else None - - target_dir = Path(output_dir) if output_dir is not None else Path(config.output_dir) - if study_run_id is not None: - target_dir = target_dir / study_run_id - target_dir.mkdir(parents=True, exist_ok=True) - stem = "adaptive_vs_fixed" if study_run_id is not None else "adaptation" - suffixes = (f"{stem}.json", f"{stem}.csv") - study_hash: str | None = None - protocol_hash: str | None = None - if study_run_id is not None: - runtime_identity = _repository_metadata() - study_inputs = { - "study": "adaptive-vs-fixed/prereg-v1", - "artifact_schema_version": STUDY_ARTIFACT_SCHEMA_VERSION, - "manifest_schema_version": STUDY_MANIFEST_SCHEMA_VERSION, - "replicate_checkpoint_schema_version": REPLICATE_CHECKPOINT_SCHEMA_VERSION, - "study_config": config.model_dump( - mode="json", exclude={"output_dir", "log_dir"} - ), - "canonical_config_sha256": compute_config_sha256(config), - "treatment_card_sha256": card_sha, - "protocol_version": benchmark.protocol_version, - "protocol_constants": { - "pre_episodes": K_PRE, - "post_episodes": N_POST, - "training_seeds": list(TRAINING_SEEDS), - "schedule": schedule, - "schedule_fingerprint": schedule_fp, - }, - "source_identity": runtime_identity, - "runtime_determinism": determinism, - } - study_hash = write_or_verify_study_manifest( - study_inputs, target_dir / "study_manifest.json", resume=resume - ) - protocol_hash = hashlib.sha256( - canonical_json_bytes( - { - "protocol_version": benchmark.protocol_version, - "treatment_card_sha256": card_sha, - "protocol_constants": study_inputs["protocol_constants"], - } - ) - ).hexdigest() - final_paths = [target_dir / suffix for suffix in (*suffixes, "manifest.json")] - if resume and all(path.is_file() for path in final_paths): - return _read_completed_study_artifact( - final_paths[0], - final_paths[-1], - run_id=study_run_id, - study_hash=study_hash, - ) - for path in final_paths: - if path.exists(): - raise FileExistsError( - f"refusing to overwrite existing artifact or partial completion: {path}" - ) - else: - for suffix in (*suffixes, "manifest.json"): - if (target_dir / suffix).exists(): - raise FileExistsError(f"refusing to overwrite existing artifact: {target_dir / suffix}") - training_root = target_dir / "training" - state_root = target_dir / "replicate_state" - if study_run_id is not None and state_root.exists() and not resume: - raise FileExistsError("run state already exists; pass --resume to use hashed replicates") - resumed_replicates: dict[int, ReplicateResult] = {} - if study_run_id is not None and resume: - assert study_hash is not None and protocol_hash is not None - resumed_replicates = _load_resume_replicates( - state_root, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seeds=selected_seeds, - ) - if study_run_id is None: - for seed in selected_seeds: - if (training_root / f"seed_{seed}").exists(): - raise FileExistsError(f"refusing to overwrite training output for seed {seed}") - - results: list[ReplicateResult] = [] - for seed in selected_seeds: - training_dir = training_root / f"seed_{seed}" - if seed in resumed_replicates: - results.append(resumed_replicates[seed]) - continue - if study_run_id is not None and resume and training_dir.exists(): - results.append( - ReplicateResult( - training_seed=seed, - status="failed", - failure_reason=( - "interrupted replicate has no complete hashed checkpoint; " - "partial training state was not trusted" - ), - ) - ) - continue - replicate = _run_replicate( - config, - seed, - training_dir, - schedule=schedule, - smoke=smoke, - trainer_factory=trainer_factory, - environment_factory=environment_factory, - ) - results.append(replicate) - if study_run_id is not None: - assert study_hash is not None and protocol_hash is not None - write_replicate_checkpoint( - replicate.to_dict(), - state_root / f"seed_{seed}.json", - study_hash=study_hash, - protocol_hash=protocol_hash, - artifact_root=target_dir, - artifact_directories=(training_dir,), - ) - - vectors: dict[str, tuple[list[Optional[float]], list[Optional[float]]]] = {} - for cell in PRIMARY_CELLS: - vectors[cell] = ([None] * len(TRAINING_SEEDS), [None] * len(TRAINING_SEEDS)) - cell_name = f"drone_disturbed/{config.algorithm.name.strip().lower()}" - if cell_name not in vectors: - raise ValueError(f"configured Issue #265 cell {cell_name!r} is not preregistered") - fixed_vector, adaptive_vector = vectors[cell_name] - for replicate in results: - if replicate.status != "completed": - continue - position = TRAINING_SEEDS.index(replicate.training_seed) - assert replicate.fixed_recovery is not None - assert replicate.adaptive_recovery is not None - fixed_vector[position] = float(replicate.fixed_recovery["truncated_recovery_time"]) - adaptive_vector[position] = float(replicate.adaptive_recovery["truncated_recovery_time"]) - analysis_vectors: dict[str, tuple[Sequence[Optional[float]], Sequence[Optional[float]]]] = { - cell: (fixed, adaptive) for cell, (fixed, adaptive) in vectors.items() - } - paired = analyze_primary_cells(analysis_vectors) - family_decision = decide_family( - {cell: analysis.primary_p_value for cell, analysis in paired.items()} - ) - - provenance = _repository_metadata() - run_status = "PARTIAL" if any(rep.status != "completed" for rep in results) else "COMPLETE" - - def outcome_count(arm: str, status: str) -> int: - count = 0 - for replicate_result in results: - if replicate_result.status != "completed": - continue - recovery = ( - replicate_result.adaptive_recovery - if arm == "adaptive" - else replicate_result.fixed_recovery - ) - if recovery is not None and recovery.get("status") == status: - count += 1 - return count - - outcome_summary = { - arm: { - **{ - status: outcome_count(arm, status) - for status in ( - "recovered", - "right_censored", - "no_degradation", - "degradation_below_resolution", - ) - }, - "failed_replicates": sum(rep.status != "completed" for rep in results), - } - for arm in ("adaptive", "fixed") - } - artifact: dict[str, Any] = { - "schema_version": STUDY_ARTIFACT_SCHEMA_VERSION, - "run_id": study_run_id, - "study_hash": study_hash, - "run_status": run_status if study_run_id is not None else None, - "protocol_version": benchmark.protocol_version, - "issue": "271" if study_run_id is not None else "265", - "run_type": ( - "smoke" - if smoke - else "prereg-v1" - if study_run_id is not None - else "full_or_selected_research_run" - ), - "determinism": determinism, - "treatment_card_sha256": card_sha, - "schedule_fingerprint": schedule_fp, - "experiment": { - "name": config.name, - "algorithm": config.algorithm.name.strip().lower(), - "environment": config.environment.name, - "scenario": benchmark.scenario, - "planned_replicates": len(TRAINING_SEEDS), - "selected_training_seeds": selected_seeds, - "config_sha256": compute_config_sha256(config), - "config": config.model_dump(mode="json"), - }, - "replicates": [replicate.to_dict() for replicate in results], - "paired_analysis": {cell: result.to_dict() for cell, result in paired.items()}, - "family_decision": family_decision, - "failure_summary": { - "failed_replicates": [ - {"training_seed": replicate.training_seed, "reason": replicate.failure_reason} - for replicate in results - if replicate.status != "completed" - ], - "completed_replicates": sum(rep.status == "completed" for rep in results), - "valid_pairs": {cell: result.valid_n for cell, result in paired.items()}, - }, - "outcome_summary": outcome_summary, - "provenance": { - **provenance, - "treatment_card_path": str(card), - "config_path": str(config_file) if config_file else None, - "config_file_sha256": config_file_sha, - }, - "scientific_claim": "Harness execution alone does not establish empirical superiority.", - } - for replicate in artifact["replicates"]: - if replicate["status"] != "completed": - replicate["invariants"] = {"all_passed": False, "failed_replicate": True} - continue - replicate["invariants"] = _audit_replicate_invariants(replicate, schedule) - if not replicate["invariants"]["all_passed"]: - raise RuntimeError( - f"runtime invariant failed for seed {replicate['training_seed']}: " - f"{replicate['invariants']}" - ) - artifact["run_status"] = ( - "PARTIAL" - if study_run_id is not None - and any(rep["status"] != "completed" for rep in artifact["replicates"]) - else run_status - ) - if study_run_id is not None: - artifact["artifact_paths"] = { - "json": f"{stem}.json", - "csv": f"{stem}.csv", - "manifest": "manifest.json", - } - if study_run_id is not None: - json_path, csv_path = write_adaptive_vs_fixed_artifacts(artifact, target_dir) - else: - json_path, csv_path = write_adaptation_artifacts(artifact, target_dir, stem=stem) - if study_run_id is not None: - output_arg = f"--output-dir {Path(output_dir)} " if output_dir is not None else "" - executable = Path(sys.argv[0]) - try: - executable_arg = executable.resolve().relative_to(Path.cwd().resolve()).as_posix() - except (OSError, ValueError): - executable_arg = executable.name - command = ( - f"{executable_arg} benchmark adaptation " - f"--config {config_arg} {output_arg}" - f"--study prereg-v1 --run-id {study_run_id}" - ) - write_study_manifest( - json_path, - csv_path, - target_dir / "manifest.json", - run_id=study_run_id, - command=command, - ) - assert study_hash is not None - return _read_completed_study_artifact( - json_path, - target_dir / "manifest.json", - run_id=study_run_id, - study_hash=study_hash, - ) - artifact["artifact_paths"] = {"json": str(json_path), "csv": str(csv_path)} - return artifact - - -__all__ = ["ReplicateResult", "run_adaptation_benchmark"] diff --git a/src/adaptive_rl/benchmarking/adaptation_runtime.py b/src/adaptive_rl/benchmarking/adaptation_runtime.py deleted file mode 100644 index 21c768e..0000000 --- a/src/adaptive_rl/benchmarking/adaptation_runtime.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Deterministic episode collection for the Issue #265 fork protocol.""" - -from __future__ import annotations - -import random -from contextlib import contextmanager -from dataclasses import asdict, dataclass -from typing import Any, Iterator, Optional - -import gymnasium as gym -import numpy as np -import torch - -from adaptive_rl.protocol.adaptation import PostShiftEpisode, Transition -from adaptive_rl.protocol.fork import model_fingerprint - - -def _fingerprint(algorithm: Any) -> str: - fixed_fingerprint = getattr(algorithm, "fingerprint", None) - if isinstance(fixed_fingerprint, str): - return fixed_fingerprint - return model_fingerprint(algorithm) - - -@contextmanager -def _seed_episode(seed: int) -> Iterator[None]: - """Seed episode-side stochastic actions and restore the caller RNG states.""" - python_state = random.getstate() - numpy_state = np.random.get_state() - torch_state = torch.random.get_rng_state() - cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None - try: - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - if cuda_states is not None: - torch.cuda.manual_seed_all(seed) - yield - finally: - random.setstate(python_state) - np.random.set_state(numpy_state) - torch.random.set_rng_state(torch_state) - if cuda_states is not None: - torch.cuda.set_rng_state_all(cuda_states) - - -def _action_and_behavior( - algorithm: Any, observation: Any, deterministic: bool -) -> tuple[np.ndarray, np.ndarray, float | None, float | None]: - model = getattr(algorithm, "model", None) - policy = getattr(model, "policy", None) if model is not None else None - if policy is None: - environment_action, _ = algorithm.predict(observation, deterministic=deterministic) - environment_action = np.asarray(environment_action) - return environment_action, environment_action, None, None - if hasattr(model, "rollout_buffer"): - assert model is not None - obs_tensor, _ = policy.obs_to_tensor(observation) - with torch.no_grad(): - native_action, value, log_prob = policy(obs_tensor, deterministic=deterministic) - native_array = native_action.detach().cpu().numpy().reshape(-1) - if getattr(policy, "squash_output", False): - environment_action = policy.unscale_action(native_array) - else: - environment_action = np.clip( - native_array, - np.asarray(model.action_space.low), - np.asarray(model.action_space.high), - ) - return ( - np.asarray(native_array), - np.asarray(environment_action), - float(log_prob.reshape(-1)[0].cpu()), - float(value.reshape(-1)[0].cpu()), - ) - - environment_action, _ = algorithm.predict(observation, deterministic=deterministic) - environment_action = np.asarray(environment_action) - if getattr(policy, "squash_output", False): - native_action = policy.scale_action(environment_action) - else: - native_action = environment_action - return np.asarray(native_action), environment_action, None, None - - -def _behavior_value(algorithm: Any, observation: Any) -> float | None: - model = getattr(algorithm, "model", None) - if model is None or not hasattr(model, "rollout_buffer"): - return None - obs_tensor, _ = model.policy.obs_to_tensor(observation) - with torch.no_grad(): - value = model.policy.predict_values(obs_tensor) - return float(value.reshape(-1)[0].cpu()) - - -@dataclass(frozen=True) -class EpisodeRecord: - """Episode metrics plus complete transition data for later adaptation.""" - - training_seed: int - arm: str - algorithm: str - environment: str - phase: str - episode_index: int - episode_seed: int - reward: float - length: int - success: Optional[bool] - collision: Optional[bool] - terminated: bool - truncated: bool - policy_fingerprint_start: str - policy_fingerprint_end: str - update_block: Optional[int] - update_seed: Optional[int] - update_status: Optional[str] - parameter_delta_l2: Optional[float] - final_info: dict[str, Any] - transitions: tuple[Transition, ...] - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - def post_shift_data(self) -> PostShiftEpisode: - if self.phase != "post": - raise ValueError("only completed post-shift episodes can enter adaptation data") - return PostShiftEpisode( - index=self.episode_index, - seed=self.episode_seed, - transitions=self.transitions, - ) - - -def evaluate_episode( - *, - algorithm: Any, - env: gym.Env, - training_seed: int, - phase: str, - episode_index: int, - episode_seed: int, - algorithm_name: str, - environment_name: str, - deterministic: bool, - arm: str, - update_log: Any = None, -) -> EpisodeRecord: - """Run and record one episode; no model update is reachable in this function.""" - if phase not in {"pre", "post"}: - raise ValueError("phase must be 'pre' or 'post'") - model = getattr(algorithm, "model", None) - if model is None and not isinstance(getattr(algorithm, "fingerprint", None), str): - raise RuntimeError("Cannot evaluate an uninitialized algorithm") - if model is not None: - model.policy.set_training_mode(False) - fingerprint_start = _fingerprint(algorithm) - transitions: list[Transition] = [] - rewards: list[float] = [] - success_values: list[bool] = [] - collisions: list[bool] = [] - observation, info = env.reset(seed=int(episode_seed)) - terminated = False - truncated = False - last_info: dict[str, Any] = dict(info) - - with _seed_episode(int(episode_seed)): - while not (terminated or truncated): - native_action, environment_action, behavior_log_prob, behavior_value = ( - _action_and_behavior(algorithm, observation, deterministic) - ) - next_observation, reward, terminated, truncated, step_info = env.step( - environment_action - ) - behavior_next_value = ( - _behavior_value(algorithm, next_observation) - if truncated and not terminated - else None - ) - transition = Transition( - observation=np.asarray(observation), - action=native_action, - environment_action=np.asarray(environment_action), - reward=float(reward), - next_observation=np.asarray(next_observation), - terminated=bool(terminated), - truncated=bool(truncated), - behavior_log_prob=behavior_log_prob, - behavior_value=behavior_value, - behavior_next_value=behavior_next_value, - ) - transitions.append(transition) - rewards.append(float(reward)) - if isinstance(step_info.get("success"), (bool, np.bool_)): - success_values.append(bool(step_info["success"])) - if isinstance(step_info.get("collision"), (bool, np.bool_)): - collisions.append(bool(step_info["collision"])) - last_info = dict(step_info) - observation = next_observation - - fingerprint_end = _fingerprint(algorithm) - if fingerprint_end != fingerprint_start: - raise RuntimeError("Policy parameters changed during an evaluation episode") - if not rewards: - raise RuntimeError("Environment returned an empty episode") - safe_info: dict[str, Any] = {} - for key, value in last_info.items(): - if isinstance(value, np.ndarray): - safe_info[str(key)] = value if np.isfinite(value).all() else None - elif isinstance(value, np.generic): - native = value.item() - safe_info[str(key)] = ( - native if not isinstance(native, float) or np.isfinite(native) else None - ) - elif value is None or isinstance(value, (str, bool, int)): - safe_info[str(key)] = value - elif isinstance(value, float): - safe_info[str(key)] = value if np.isfinite(value) else None - if any(isinstance(value, float) and not np.isfinite(value) for value in rewards): - raise FloatingPointError("episode produced a non-finite reward") - return EpisodeRecord( - training_seed=training_seed, - arm=arm, - algorithm=algorithm_name, - environment=environment_name, - phase=phase, - episode_index=episode_index, - episode_seed=int(episode_seed), - reward=float(sum(rewards)), - length=len(rewards), - success=success_values[-1] if success_values else None, - collision=any(collisions) if collisions else None, - terminated=bool(terminated), - truncated=bool(truncated), - policy_fingerprint_start=fingerprint_start, - policy_fingerprint_end=fingerprint_end, - update_block=getattr(update_log, "block_episode", None), - update_seed=getattr(update_log, "update_seed", None), - update_status=getattr(update_log, "status", None), - parameter_delta_l2=getattr(update_log, "parameter_delta_l2", None), - final_info=safe_info, - transitions=tuple(transitions), - ) - - -__all__ = ["EpisodeRecord", "evaluate_episode"] diff --git a/src/adaptive_rl/benchmarking/adaptation_statistics.py b/src/adaptive_rl/benchmarking/adaptation_statistics.py deleted file mode 100644 index d8ab715..0000000 --- a/src/adaptive_rl/benchmarking/adaptation_statistics.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Paired recovery-time and six-cell multiplicity analysis for Issue #265.""" - -from __future__ import annotations - -from dataclasses import asdict, dataclass, replace -from typing import Any, Optional, Sequence - -from adaptive_rl.protocol.constants import HORIZON, MIN_VALID_N, PLANNED_N, PRIMARY_CELLS -from adaptive_rl.protocol.statistics import ( - bootstrap_percentile_ci, - cohen_dz, - exact_sign_test, - exact_wilcoxon_signed_rank, - holm_adjust, - impute_differences, - paired_differences, - paired_t_interval, - paired_t_test, -) - - -@dataclass(frozen=True) -class PairedRecoveryAnalysis: - planned_n: int - completed_n: int - failed_n: int - valid_n: int - differences: list[Optional[float]] - mean_difference: Optional[float] - standard_deviation: Optional[float] - standard_error: Optional[float] - t_statistic: Optional[float] - degrees_of_freedom: Optional[int] - primary_p_value: Optional[float] - holm_adjusted_p_value: Optional[float] - interval_95: Optional[list[float]] - cohen_dz: Optional[float] - exact_sign_p_value: Optional[float] - exact_wilcoxon_p_value: Optional[float] - bootstrap_interval_95: Optional[list[float]] - failure_imputation_bounds: Optional[dict[str, dict[str, Any]]] - status: str - - def to_dict(self) -> dict[str, object]: - return asdict(self) - - -def analyze_paired_recovery( - fixed_t_h: Sequence[Optional[float]], - adaptive_t_h: Sequence[Optional[float]], - *, - planned_n: int = PLANNED_N, - horizon: float = HORIZON, - min_valid_n: int = MIN_VALID_N, -) -> PairedRecoveryAnalysis: - """Analyze one cell using the frozen paired tests and sensitivity methods.""" - if len(fixed_t_h) != len(adaptive_t_h): - raise ValueError("Fixed and Adaptive T_H vectors must have equal length.") - if len(fixed_t_h) != planned_n: - raise ValueError(f"Expected {planned_n} planned pairs, got {len(fixed_t_h)}.") - - completed_n = sum(f is not None and a is not None for f, a in zip(fixed_t_h, adaptive_t_h)) - failed_n = planned_n - completed_n - paired = paired_differences(fixed_t_h, adaptive_t_h) - differences = [ - None if fixed is None or adaptive is None else float(adaptive) - float(fixed) - for fixed, adaptive in zip(fixed_t_h, adaptive_t_h) - ] - worst = impute_differences(fixed_t_h, adaptive_t_h, horizon, "worst_for_adaptive") - best = impute_differences(fixed_t_h, adaptive_t_h, horizon, "best_for_adaptive") - bounds = { - "worst_for_adaptive": {"mean_difference": sum(worst) / planned_n, "per_replicate": worst}, - "best_for_adaptive": {"mean_difference": sum(best) / planned_n, "per_replicate": best}, - } - if len(paired) < min_valid_n: - return PairedRecoveryAnalysis( - planned_n=planned_n, - completed_n=completed_n, - failed_n=failed_n, - valid_n=len(paired), - differences=differences, - mean_difference=None, - standard_deviation=None, - standard_error=None, - t_statistic=None, - degrees_of_freedom=None, - primary_p_value=None, - holm_adjusted_p_value=None, - interval_95=None, - cohen_dz=None, - exact_sign_p_value=None, - exact_wilcoxon_p_value=None, - bootstrap_interval_95=None, - failure_imputation_bounds=bounds, - status="inconclusive", - ) - - primary = paired_t_test(paired, min_valid_n=min_valid_n) - return PairedRecoveryAnalysis( - planned_n=planned_n, - completed_n=completed_n, - failed_n=failed_n, - valid_n=len(paired), - differences=differences, - mean_difference=primary.mean, - standard_deviation=primary.std_dev, - standard_error=primary.standard_error, - t_statistic=primary.t_statistic, - degrees_of_freedom=primary.degrees_of_freedom, - primary_p_value=primary.p_value, - holm_adjusted_p_value=None, - interval_95=list(paired_t_interval(paired, min_valid_n=min_valid_n)), - cohen_dz=cohen_dz(paired), - exact_sign_p_value=exact_sign_test(paired, min_valid_n=min_valid_n).p_value, - exact_wilcoxon_p_value=exact_wilcoxon_signed_rank(paired, min_valid_n=min_valid_n).p_value, - bootstrap_interval_95=list(bootstrap_percentile_ci(paired, min_valid_n=min_valid_n)), - failure_imputation_bounds=bounds, - status="evaluable", - ) - - -def analyze_primary_cells( - outcomes: dict[str, tuple[Sequence[Optional[float]], Sequence[Optional[float]]]], -) -> dict[str, PairedRecoveryAnalysis]: - """Analyze the six frozen cells and apply Holm across the full cell family. - - Missing or invalid cells remain explicitly inconclusive. Their unavailable - p-values are represented as 1.0 during family correction, and remain null - in their own output record. - """ - if set(outcomes) != set(PRIMARY_CELLS): - raise ValueError("outcomes must contain exactly the preregistered PRIMARY_CELLS") - results = {cell: analyze_paired_recovery(*outcomes[cell]) for cell in PRIMARY_CELLS} - p_values: list[float] = [] - for cell in PRIMARY_CELLS: - p_value = results[cell].primary_p_value - p_values.append(p_value if p_value is not None else 1.0) - adjusted = holm_adjust(p_values) - return { - cell: replace(results[cell], holm_adjusted_p_value=adjusted[index]) - if results[cell].primary_p_value is not None - else results[cell] - for index, cell in enumerate(PRIMARY_CELLS) - } - - -__all__ = ["PairedRecoveryAnalysis", "analyze_paired_recovery", "analyze_primary_cells"] diff --git a/src/adaptive_rl/cli.py b/src/adaptive_rl/cli.py index fa669d4..876eed9 100644 --- a/src/adaptive_rl/cli.py +++ b/src/adaptive_rl/cli.py @@ -28,7 +28,6 @@ benchmark_app = typer.Typer( name="benchmark", - help="Benchmark commands for learning curves and online adaptation.", help="Benchmarking and comparative evaluation commands.", no_args_is_help=True, ) @@ -538,136 +537,6 @@ def benchmark_budgets( @benchmark_app.command(name="adaptation") -def benchmark_adaptation( - config: Path = typer.Option( - Path("configs/drone_distribution_shift.yaml"), - "--config", - "-c", - help="Issue #265 nominal training and TEST-B configuration", - ), - algorithm: Optional[str] = typer.Option( - None, "--algorithm", help="Algorithm cell: ppo or sac (defaults to config value)" - ), - training_seeds: Optional[str] = typer.Option( - None, - "--training-seeds", - help="Comma-separated preregistered training seeds; defaults to all ten", - ), - output_dir: Optional[Path] = typer.Option( - None, "--output-dir", help="Directory for Issue #265 JSON/CSV and training artifacts" - ), - study: Optional[str] = typer.Option( - None, "--study", help="Run the immutable full protocol study (currently prereg-v1)" - ), - run_id: Optional[str] = typer.Option( - None, "--run-id", help="Unique immutable output directory name required with --study" - ), - resume: bool = typer.Option( - False, - "--resume", - help="Reuse only complete hashed replicate checkpoints for an unfinished study run", - ), - deterministic: Optional[bool] = typer.Option( - None, - "--deterministic/--stochastic", - help="Override evaluation action selection; PPO adaptation requires stochastic actions", - ), - smoke: bool = typer.Option( - False, - "--smoke", - help="Run one explicitly labeled, reduced-size machinery check (not research data)", - ), -) -> None: - """Run the preregistered train-once, forked Adaptive-vs-Fixed experiment.""" - try: - exp_config = load_config(config) - if algorithm is not None: - selected_algorithm = algorithm.strip().lower() - if selected_algorithm not in {"ppo", "sac"}: - raise ValueError("--algorithm must be 'ppo' or 'sac'") - algorithm_config = exp_config.algorithm.model_copy(deep=True) - if selected_algorithm != algorithm_config.name.strip().lower(): - algorithm_config.name = selected_algorithm - if selected_algorithm == "sac": - algorithm_config.parameters = { - "buffer_size": 100_000, - "learning_starts": 100, - "train_freq": 1, - "gradient_steps": 1, - "tau": 0.005, - "ent_coef": "auto", - } - else: - algorithm_config.parameters = { - "n_steps": 1024, - "n_epochs": 10, - "clip_range": 0.2, - "ent_coef": 0.01, - } - exp_config = exp_config.model_copy( - update={"algorithm": algorithm_config}, deep=True - ) - if deterministic is not None: - evaluation_config = exp_config.evaluation.model_copy(deep=True) - evaluation_config.deterministic = deterministic - exp_config = exp_config.model_copy(update={"evaluation": evaluation_config}, deep=True) - - selected_seeds = None - if training_seeds is not None: - tokens = [token.strip() for token in training_seeds.split(",")] - if not tokens or any(not token for token in tokens): - raise ValueError("--training-seeds expects comma-separated integers") - try: - selected_seeds = [int(token) for token in tokens] - except ValueError as exc: - raise ValueError("--training-seeds expects comma-separated integers") from exc - - from adaptive_rl.benchmarking.adaptation_runner import run_adaptation_benchmark - - if study not in {None, "prereg-v1"}: - raise ValueError("--study currently supports only prereg-v1") - if (study is None) != (run_id is None): - raise ValueError("--study and --run-id must be supplied together") - if resume and study is None: - raise ValueError("--resume requires --study prereg-v1 and --run-id") - - artifact = run_adaptation_benchmark( - exp_config, - output_dir=output_dir, - training_seeds=selected_seeds, - smoke=smoke, - config_path=config, - study_run_id=run_id if study is not None else None, - resume=resume, - ) - except Exception as err: - console.print(f"[bold red]Issue #265 benchmark failed:[/bold red] {err}") - raise typer.Exit(code=1) - - failed = artifact["failure_summary"]["failed_replicates"] - completed = artifact["failure_summary"]["completed_replicates"] - json_artifact_path = Path(artifact["artifact_paths"]["json"]) - csv_artifact_path = Path(artifact["artifact_paths"]["csv"]) - if study is not None and run_id is not None: - study_dir = (output_dir or exp_config.output_dir) / run_id - json_artifact_path = study_dir / json_artifact_path - csv_artifact_path = study_dir / csv_artifact_path - console.print( - Panel.fit( - f"[bold]{'Smoke check' if smoke else 'Issue #265 benchmark'} finished[/bold]\n\n" - f"• [bold]Run type:[/bold] {artifact['run_type']}\n" - f"• [bold]Algorithm:[/bold] {artifact['experiment']['algorithm']}\n" - f"• [bold]Completed replicates:[/bold] {completed}\n" - f"• [bold]Failed replicates:[/bold] {len(failed)}\n" - f"• [bold]JSON:[/bold] {json_artifact_path}\n" - f"• [bold]CSV:[/bold] {csv_artifact_path}\n" - f"• [bold]Scientific result:[/bold] not established by harness execution", - title="Online Adaptation Benchmark", - border_style="yellow" if smoke or failed else "green", - ) - ) - if failed: - raise typer.Exit(code=1) @app.command(name="benchmark-adaptation") def benchmark_adaptation( seeds: Optional[str] = typer.Option( diff --git a/src/adaptive_rl/config.py b/src/adaptive_rl/config.py index ee1f3e6..0c44bbf 100644 --- a/src/adaptive_rl/config.py +++ b/src/adaptive_rl/config.py @@ -150,29 +150,6 @@ def _validate_evaluation_seeds(cls, values: list[int]) -> list[int]: return values -class AdaptationBenchmarkConfig(BaseModel): - """Frozen TEST-B cell declaration for Issue #265.""" - - model_config = ConfigDict(extra="forbid") - - protocol_version: str = "2.0" - scenario: str = "TEST-B" - shift_parameters: Dict[str, Any] = Field( - default_factory=lambda: {"num_obstacles": 12, "wind_speed": 4.0, "gust_sigma": 0.6} - ) - - @model_validator(mode="after") - def _validate_test_b(self) -> "AdaptationBenchmarkConfig": - expected = {"num_obstacles": 12, "wind_speed": 4.0, "gust_sigma": 0.6} - if self.protocol_version != "2.0": - raise ValueError("Issue #265 requires protocol_version '2.0'") - if self.scenario != "TEST-B" or self.shift_parameters != expected: - raise ValueError( - "Issue #265's primary cell is exactly TEST-B (12 obstacles, wind 4.0, gust 0.6)" - ) - return self - - class ExperimentConfig(BaseModel): """Top-level configuration schema for an AdaptiveRL experiment.""" @@ -196,10 +173,6 @@ class ExperimentConfig(BaseModel): default=None, description="Optional benchmark settings for training-budget learning curves.", ) - adaptation_benchmark: Optional[AdaptationBenchmarkConfig] = Field( - default=None, - description="Frozen Issue #265 TEST-B online-adaptation cell.", - ) @model_validator(mode="before") @classmethod diff --git a/src/adaptive_rl/environments/__init__.py b/src/adaptive_rl/environments/__init__.py index f65b00e..6490614 100644 --- a/src/adaptive_rl/environments/__init__.py +++ b/src/adaptive_rl/environments/__init__.py @@ -58,19 +58,6 @@ def register_default_environments() -> None: ), ) - if "drone_disturbed" not in list_environments(): - register( - "drone_disturbed", - lambda **kwargs: DroneNavigation3DEnv(**kwargs), - metadata=EnvironmentMetadata( - name="drone_disturbed", - description="Drone navigation with deterministic steady-wind and OU gust parameters.", - observation_type="box", - action_type="continuous", - version="0.2.0", - max_episode_steps=200, - reward_range=(-100.0, 100.0), - tags=["continuous", "drone", "distribution-shift", "wind", "gust"], if "drone-6dof" not in list_environments(): from adaptive_rl.environments.drone_6dof import Drone6DOFEnv diff --git a/src/adaptive_rl/environments/drone.py b/src/adaptive_rl/environments/drone.py index 3697748..3864b3a 100644 --- a/src/adaptive_rl/environments/drone.py +++ b/src/adaptive_rl/environments/drone.py @@ -130,6 +130,7 @@ def step(self, action_acceleration: np.ndarray) -> Tuple[np.ndarray, np.ndarray] raw_acc = np.asarray(action_acceleration, dtype=np.float64) if raw_acc.shape != (3,): raise ValueError(f"Acceleration command must have shape (3,), got {raw_acc.shape}") + clamped_acc = np.clip(raw_acc, -self.max_acceleration, self.max_acceleration) self.state.acceleration = clamped_acc @@ -431,10 +432,6 @@ def __init__( terminate_on_collision: bool = True, render_mode: Optional[str] = None, split: Optional[str] = None, - wind_speed: float = 0.0, - gust_sigma: float = 0.0, - gust_theta: float = 0.15, - wind_direction: Tuple[float, float, float] = (1.5, 0.5, 0.0), ) -> None: super().__init__() if any(b <= 0.0 for b in bounds): @@ -445,19 +442,6 @@ def __init__( raise ValueError(f"target_radius must be positive, got {target_radius}") if collision_radius <= 0.0: raise ValueError(f"collision_radius must be positive, got {collision_radius}") - if not np.isfinite(wind_speed) or wind_speed < 0.0: - raise ValueError(f"wind_speed must be finite and non-negative, got {wind_speed}") - if not np.isfinite(gust_sigma) or gust_sigma < 0.0: - raise ValueError(f"gust_sigma must be finite and non-negative, got {gust_sigma}") - if not np.isfinite(gust_theta) or gust_theta <= 0.0: - raise ValueError(f"gust_theta must be finite and positive, got {gust_theta}") - direction = np.asarray(wind_direction, dtype=np.float64) - if ( - direction.shape != (3,) - or not np.isfinite(direction).all() - or np.linalg.norm(direction) == 0 - ): - raise ValueError("wind_direction must be a finite, non-zero 3-vector") if lidar_noise_std < 0.0: raise ValueError(f"lidar_noise_std cannot be negative, got {lidar_noise_std}") if not (0.0 <= lidar_dropout_prob <= 1.0): @@ -513,11 +497,6 @@ def __init__( self.action_penalty_weight = float(action_penalty_weight) self.terminate_on_collision = terminate_on_collision self.render_mode = render_mode - self.wind_speed = float(wind_speed) - self.gust_sigma = float(gust_sigma) - self.gust_theta = float(gust_theta) - self.wind_direction = direction / np.linalg.norm(direction) - self._gust_velocity = np.zeros(3, dtype=np.float64) self.max_diagonal = float(np.linalg.norm(self.bounds)) @@ -621,7 +600,6 @@ def _get_info(self) -> Dict[str, Any]: "num_obstacles": len(self._obstacles), "min_obstacle_distance": min_obs_dist if self._obstacles else float("inf"), "altitude": float(self._position[2]), - "wind_velocity": self._wind_velocity(self._position).copy(), "lidar_noise_std": self.lidar_noise_std, "lidar_dropout_prob": self.lidar_dropout_prob, "lidar_min_range": self.lidar_min_range, @@ -632,27 +610,6 @@ def _get_info(self) -> Dict[str, Any]: info["split_seed"] = self._last_split_seed return info - def _wind_velocity(self, position: np.ndarray) -> np.ndarray: - altitude_factor = 1.0 + 0.02 * max(0.0, float(position[2])) - return self.wind_direction * self.wind_speed * altitude_factor + self._gust_velocity - - def get_effective_parameters(self) -> Dict[str, Any]: - """Return the physical environment parameters used by this instance.""" - return { - "bounds": list(self.bounds), - "num_obstacles": self.num_obstacles, - "obstacle_radius": self.obstacle_radius, - "wind_speed": self.wind_speed, - "gust_sigma": self.gust_sigma, - "gust_theta": self.gust_theta, - "wind_direction": self.wind_direction.tolist(), - "wind_altitude_shear": 0.02, - "max_gust": 4.0, - "linear_damping": self.kinematics.linear_damping, - "max_acceleration": self.kinematics.max_acceleration, - "max_steps": self.max_steps, - } - @property def drone_state(self) -> DroneState3D: """Convenience property exposing the underlying drone state.""" @@ -741,7 +698,6 @@ def reset( self._position = self.default_start.copy() self._velocity = np.zeros(3, dtype=np.float64) self._goal = self.default_goal.copy() - self._gust_velocity = np.zeros(3, dtype=np.float64) self.kinematics.reset(self._position, self._velocity) @@ -773,19 +729,7 @@ def step( self._current_step += 1 - # Exact discrete OU update: stationary per-axis gust standard deviation - # approaches sigma/sqrt(2*theta), as specified by the shift config. - if self.gust_sigma > 0.0: - noise = self.np_random.normal(size=3) - self._gust_velocity += ( - -self.gust_theta * self._gust_velocity * self.kinematics.dt - + self.gust_sigma * np.sqrt(self.kinematics.dt) * noise - ) - gust_magnitude = float(np.linalg.norm(self._gust_velocity)) - if gust_magnitude > 4.0: - self._gust_velocity *= 4.0 / gust_magnitude - wind_acceleration = self.kinematics.linear_damping * self._wind_velocity(self._position) - acc_command = act_arr * self.kinematics.max_acceleration + wind_acceleration + acc_command = act_arr * self.kinematics.max_acceleration new_pos, new_vel = self.kinematics.step(acc_command) self._position = new_pos self._velocity = new_vel diff --git a/src/adaptive_rl/protocol/__init__.py b/src/adaptive_rl/protocol/__init__.py index f0128e6..964ffa7 100644 --- a/src/adaptive_rl/protocol/__init__.py +++ b/src/adaptive_rl/protocol/__init__.py @@ -9,15 +9,6 @@ ``shift_runner.py``, and do not make any experimental cell executable. """ -from adaptive_rl.protocol.adaptation import ( - AdaptationAdapter, - PostShiftEpisode, - Transition, - UpdateBatch, - build_update_batch, - call_update_atomically, - validate_block_sequence, -) from adaptive_rl.protocol.constants import ( ALPHA, BOOTSTRAP_REPS, @@ -83,7 +74,6 @@ ) __all__ = [ - "AdaptationAdapter", "ALPHA", "BOOTSTRAP_REPS", "BOOTSTRAP_SEED", @@ -105,7 +95,6 @@ "PROTOCOL_VERSION", "PairedTTest", "PERSISTENCE", - "PostShiftEpisode", "RECOVERY_STATUSES", "RECOVERY_THRESHOLD", "RecoveryResult", @@ -116,14 +105,10 @@ "STATUS_RIGHT_CENSORED", "SignTestResult", "TRAINING_SEEDS", - "Transition", - "UpdateBatch", "WINDOW", "WilcoxonResult", "bootstrap_percentile_ci", "build_schedule", - "build_update_batch", - "call_update_atomically", "cohen_dz", "compute_recovery", "decide_family", @@ -143,6 +128,5 @@ "student_t_cdf", "student_t_ppf", "trailing_window_means", - "validate_block_sequence", "validate_schedule", ] diff --git a/src/adaptive_rl/protocol/adaptation.py b/src/adaptive_rl/protocol/adaptation.py deleted file mode 100644 index bbb088f..0000000 --- a/src/adaptive_rl/protocol/adaptation.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Episode-bounded data boundary for Issue #265 online updates.""" - -from __future__ import annotations - -import copy -from dataclasses import dataclass -from typing import Any, Protocol, Sequence - -import numpy as np - -from adaptive_rl.protocol.constants import N_POST, N_UPDATE -from adaptive_rl.protocol.seeds import derive_seed - - -@dataclass(frozen=True) -class Transition: - """One recorded environment transition, detached from live environment state.""" - - observation: np.ndarray - action: np.ndarray - reward: float - next_observation: np.ndarray - terminated: bool - truncated: bool - environment_action: np.ndarray | None = None - behavior_log_prob: float | None = None - behavior_value: float | None = None - behavior_next_value: float | None = None - - def __post_init__(self) -> None: - for name in ("observation", "action", "next_observation"): - value = np.array(getattr(self, name), copy=True) - if value.dtype.kind not in "biuf": - raise ValueError(f"{name} must have a numeric dtype") - if not np.isfinite(value).all(): - raise ValueError(f"{name} must contain only finite values") - value.setflags(write=False) - object.__setattr__(self, name, value) - if self.environment_action is not None: - value = np.array(self.environment_action, copy=True) - if value.dtype.kind not in "biuf" or not np.isfinite(value).all(): - raise ValueError("environment_action must be a finite numeric array") - value.setflags(write=False) - object.__setattr__(self, "environment_action", value) - if not np.isfinite(float(self.reward)): - raise ValueError("reward must be finite") - object.__setattr__(self, "reward", float(self.reward)) - if not isinstance(self.terminated, bool) or not isinstance(self.truncated, bool): - raise ValueError("terminated and truncated must be booleans") - for name in ("behavior_log_prob", "behavior_value", "behavior_next_value"): - value = getattr(self, name) - if value is not None: - if not np.isfinite(float(value)): - raise ValueError(f"{name} must be finite when provided") - object.__setattr__(self, name, float(value)) - - -@dataclass(frozen=True) -class PostShiftEpisode: - """Completed post-shift episode with its protocol index and derived seed.""" - - index: int - seed: int - transitions: tuple[Transition, ...] - - def __post_init__(self) -> None: - if not 1 <= self.index <= N_POST: - raise ValueError(f"post-shift episode index must be in [1, {N_POST}]") - if not self.transitions: - raise ValueError("a completed episode must contain at least one transition") - - -@dataclass(frozen=True) -class UpdateBatch: - """Cumulative post-shift experience visible at exactly one update boundary.""" - - block_episode: int - seed: int - visible_episode_indices: tuple[int, ...] - transitions: tuple[Transition, ...] - - -class AdaptationAdapter(Protocol): - """Algorithm-specific adapter interface; called only at episode boundaries.""" - - def update(self, algorithm: Any, batch: UpdateBatch) -> None: ... - - -def build_update_batch( - training_seed: int, - completed_episodes: Sequence[PostShiftEpisode], - *, - block_episode: int, -) -> UpdateBatch: - """Build Bk from precisely post-shift episodes 1..k. - - The builder requires the full ordered prefix. It has no parameter for - pre-shift or future data, preventing accidental cross-phase exposure. - """ - if not 5 <= block_episode <= 14: - raise ValueError("Update blocks exist only after post-shift episodes 5 through 14") - if len(completed_episodes) != block_episode: - raise ValueError( - f"B{block_episode} requires exactly episodes 1..{block_episode}; " - f"received {len(completed_episodes)} episodes" - ) - expected_indices = tuple(range(1, block_episode + 1)) - actual_indices = tuple(episode.index for episode in completed_episodes) - if actual_indices != expected_indices: - raise ValueError( - f"B{block_episode} can see only ordered completed episodes " - f"1..{block_episode}; got {actual_indices}" - ) - expected_seeds = tuple(derive_seed(training_seed, "post", index) for index in expected_indices) - actual_seeds = tuple(episode.seed for episode in completed_episodes) - if actual_seeds != expected_seeds: - raise ValueError("post-shift episode seeds do not match the preregistered schedule") - - transitions = tuple( - transition for episode in completed_episodes for transition in episode.transitions - ) - return UpdateBatch( - block_episode=block_episode, - seed=derive_seed(training_seed, "update", block_episode - 5), - visible_episode_indices=expected_indices, - transitions=transitions, - ) - - -def validate_block_sequence(block_episodes: Sequence[int]) -> None: - """Require the complete preregistered B5..B14 ordering (never B15).""" - expected = tuple(range(5, 5 + N_UPDATE)) - actual = tuple(block_episodes) - if actual != expected: - raise ValueError(f"update ordering must be exactly {expected}, got {actual}") - - -def call_update_atomically(algorithm: Any, adapter: AdaptationAdapter, batch: UpdateBatch) -> None: - """Run an update with wrapper-state rollback if it raises. - - A failed block remains a failure at the caller; rollback only prevents a - partially-mutated policy from leaking into later evaluation. - """ - if not isinstance(getattr(algorithm, "__dict__", None), dict): - raise TypeError("algorithm adapter target must expose instance state") - snapshot = copy.deepcopy(algorithm.__dict__) - try: - adapter.update(algorithm, batch) - except BaseException: - algorithm.__dict__.clear() - algorithm.__dict__.update(snapshot) - raise - - -__all__ = [ - "AdaptationAdapter", - "PostShiftEpisode", - "Transition", - "UpdateBatch", - "build_update_batch", - "call_update_atomically", - "validate_block_sequence", -] diff --git a/src/adaptive_rl/protocol/constants.py b/src/adaptive_rl/protocol/constants.py index 54f8931..b1edf20 100644 --- a/src/adaptive_rl/protocol/constants.py +++ b/src/adaptive_rl/protocol/constants.py @@ -101,14 +101,6 @@ #: Number of planned replicate pairs per cell (len(TRAINING_SEEDS)). PLANNED_N: int = len(TRAINING_SEEDS) -# Issue #271 prereg-v1 freezes one PPO cell and its semantic config/treatment. -# These hashes are declared in docs/research/issue-271.md and require a new -# study version if either scientific input changes. -ISSUE271_CONFIG_SHA256: str = "0039c298b5048254b2d211cc66967e9d3e1d71275575fcfe28736485c65937b5" -ISSUE271_TREATMENT_CARD_SHA256: str = ( - "8383e736f02ff31393474b241b06d2dc92b036910c93c26081c1673c92a317e3" -) - __all__ = [ "ALPHA", "BOOTSTRAP_REPS", @@ -116,8 +108,6 @@ "CONFIG_TEST_POOL", "CONFIG_TRAIN_POOL", "HORIZON", - "ISSUE271_CONFIG_SHA256", - "ISSUE271_TREATMENT_CARD_SHA256", "K_PRE", "MIN_DEGRADATION_SE_MULTIPLIER", "MIN_VALID_N", diff --git a/src/adaptive_rl/protocol/fork.py b/src/adaptive_rl/protocol/fork.py deleted file mode 100644 index 4d76e7d..0000000 --- a/src/adaptive_rl/protocol/fork.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Model-state hashing and independent policy forks for paired experiments.""" - -from __future__ import annotations - -import copy -import hashlib -import struct -from typing import Any - -import torch - - -def _state_module(model_or_wrapper: Any) -> Any: - model = getattr(model_or_wrapper, "model", model_or_wrapper) - if callable(getattr(model, "state_dict", None)): - return model - policy = getattr(model, "policy", None) - if policy is not None and callable(getattr(policy, "state_dict", None)): - return policy - raise TypeError("Expected a model or algorithm wrapper with a state_dict().") - - -def policy_state_tensors(model_or_wrapper: Any) -> dict[str, Any]: - model = getattr(model_or_wrapper, "model", model_or_wrapper) - state = { - f"policy.{key}": value - for key, value in _state_module(model_or_wrapper).state_dict().items() - } - # SAC's automatic entropy coefficient is a trainable model parameter stored - # on the algorithm object rather than inside SACPolicy. - log_ent_coef = getattr(model, "log_ent_coef", None) - if isinstance(log_ent_coef, torch.Tensor): - state["algorithm.log_ent_coef"] = log_ent_coef - return state - - -def model_fingerprint(model_or_wrapper: Any) -> str: - """Return a stable SHA-256 fingerprint of a model's complete state dict. - - Stable-Baselines3 wrappers expose their model as ``.model``. Hashing the - state dict includes parameters and persistent buffers, with names, shapes, - and dtypes included to distinguish structurally different models. - """ - digest = hashlib.sha256() - for name, tensor in sorted(policy_state_tensors(model_or_wrapper).items()): - if not hasattr(tensor, "detach"): - raise TypeError(f"Model state entry {name!r} is not a tensor.") - value = tensor.detach().cpu().contiguous() - digest.update(name.encode("utf-8")) - digest.update(b"\0") - digest.update(str(value.dtype).encode("ascii")) - digest.update(b"\0") - digest.update(struct.pack("!I", value.ndim)) - for dimension in value.shape: - digest.update(struct.pack("!Q", int(dimension))) - digest.update(value.view(torch.uint8).numpy().tobytes(order="C")) - return digest.hexdigest() - - -def clone_algorithm(algorithm: Any) -> Any: - """Deep-clone an algorithm wrapper and reject clones with shared tensors. - - ``deepcopy`` is intentional: SB3 algorithms contain mutable optimizers, - buffers, counters, and RNG state in addition to policy parameters. - """ - clone = copy.deepcopy(algorithm) - original_model = getattr(algorithm, "model", None) - clone_model = getattr(clone, "model", None) - if original_model is None or clone_model is None: - raise TypeError("Algorithm and clone must both have initialized models.") - original_state = policy_state_tensors(algorithm) - clone_state = policy_state_tensors(clone) - if original_state.keys() != clone_state.keys(): - raise RuntimeError("Cloned model state has a different structure.") - for name in original_state: - left = original_state[name] - right = clone_state[name] - if left.shape != right.shape or left.dtype != right.dtype: - raise RuntimeError(f"Cloned model state {name!r} has a different shape or dtype.") - if left.data_ptr() == right.data_ptr(): - raise RuntimeError(f"Cloned model state {name!r} shares storage with its source.") - if not left.equal(right): - raise RuntimeError(f"Cloned model state {name!r} differs from its source.") - return clone - - -class FrozenPolicy: - """Prediction-only facade that exposes no training or model mutation API.""" - - __slots__ = ("__algorithm",) - - def __init__(self, algorithm: Any) -> None: - if not callable(getattr(algorithm, "predict", None)): - raise TypeError("Frozen policy requires an algorithm with predict().") - self.__algorithm = algorithm - - def predict(self, observation: Any, deterministic: bool = True) -> Any: - return self.__algorithm.predict(observation, deterministic=deterministic) - - @property - def fingerprint(self) -> str: - return model_fingerprint(self.__algorithm) - - -def fork_adaptive_and_fixed(algorithm: Any) -> tuple[Any, FrozenPolicy, str]: - """Return an independent adaptive clone and prediction-only fixed arm.""" - expected = model_fingerprint(algorithm) - adaptive = clone_algorithm(algorithm) - fixed = FrozenPolicy(clone_algorithm(algorithm)) - if model_fingerprint(adaptive) != expected or fixed.fingerprint != expected: - raise RuntimeError("Forked policies do not match the frozen model fingerprint.") - return adaptive, fixed, expected - - -__all__ = [ - "FrozenPolicy", - "clone_algorithm", - "fork_adaptive_and_fixed", - "model_fingerprint", - "policy_state_tensors", -] diff --git a/src/adaptive_rl/training/trainer.py b/src/adaptive_rl/training/trainer.py index cbcde6a..39c512d 100644 --- a/src/adaptive_rl/training/trainer.py +++ b/src/adaptive_rl/training/trainer.py @@ -59,8 +59,6 @@ class TrainingResult: training_time_seconds: float = 0.0 -class AlgorithmTrainer: - """Trainer orchestrating supported Stable-Baselines3 algorithm learning.""" class RLTrainer: """Trainer orchestrating reinforcement learning policy learning on the drone navigation environment.""" @@ -99,18 +97,6 @@ def __init__( gamma = algo_params.pop("gamma", self.config.algorithm.gamma) batch_size = algo_params.pop("batch_size", self.config.algorithm.batch_size) seed = algo_params.pop("seed", self.config.seed) - algorithm_name = self.config.algorithm.name.strip().lower() - algorithm_types = {"ppo": PPOAlgorithm, "sac": SACAlgorithm} - if algorithm_name not in algorithm_types: - raise ValueError(f"Unsupported training algorithm: {self.config.algorithm.name!r}") - self.algorithm = algorithm_types[algorithm_name]( - env=self.env, - learning_rate=lr, - gamma=gamma, - batch_size=batch_size, - seed=seed, - **algo_params, - ) algo_name = self.config.algorithm.name.lower() self.algorithm: BaseAlgorithm @@ -262,12 +248,6 @@ def close(self) -> None: ) -class PPOTrainer(AlgorithmTrainer): - """Trainer for PPO; retained as the explicit PPO-facing public class.""" - - -class SACTrainer(AlgorithmTrainer): - """Trainer for SAC using the same callbacks and artifact lifecycle.""" PPOTrainer = RLTrainer @@ -275,13 +255,6 @@ def get_trainer( config: ExperimentConfig, env: Optional[gym.Env] = None, callbacks: Optional[List[BaseCallback]] = None, -) -> AlgorithmTrainer: - """Factory for the configured PPO or SAC algorithm trainer.""" - trainer_types = {"ppo": PPOTrainer, "sac": SACTrainer} - algorithm_name = config.algorithm.name.strip().lower() - if algorithm_name not in trainer_types: - raise ValueError(f"Unsupported training algorithm: {config.algorithm.name!r}") - return trainer_types[algorithm_name](config=config, env=env, callbacks=callbacks) ) -> RLTrainer: """Factory returning the trainer based on configuration.""" return RLTrainer(config=config, env=env, callbacks=callbacks) diff --git a/tests/test_adaptation_artifacts.py b/tests/test_adaptation_artifacts.py deleted file mode 100644 index 028483c..0000000 --- a/tests/test_adaptation_artifacts.py +++ /dev/null @@ -1,405 +0,0 @@ -"""Structured artifact serialization and no-overwrite guarantees.""" - -from __future__ import annotations - -import csv -import json - -import numpy as np -import pytest - -from adaptive_rl.benchmarking.adaptation_artifacts import ( - canonical_json_bytes, - make_study_manifest, - read_replicate_checkpoint, - sha256_file, - validate_study_manifest, - write_adaptation_artifacts, - write_adaptive_vs_fixed_artifacts, - write_or_verify_study_manifest, - write_replicate_checkpoint, - write_study_manifest, -) -from adaptive_rl.config import compute_config_sha256, load_config -from adaptive_rl.protocol.constants import ( - ISSUE271_CONFIG_SHA256, - ISSUE271_TREATMENT_CARD_SHA256, -) - - -def _artifact(): - return { - "schema_version": "1.0", - "protocol_version": "2.0", - "issue": "265", - "experiment": {"algorithm": "ppo", "environment": "drone"}, - "replicates": [ - { - "training_seed": 31001, - "shared_pre_shift_episodes": [ - { - "episode_index": 1, - "episode_seed": 123, - "reward": np.float64(2.0), - "length": 15, - "success": True, - "collision": False, - "terminated": True, - "truncated": False, - "transitions": [{"observation": np.asarray([0.1, 0.2])}], - } - ], - "shared_shock_episodes": [], - "adaptive_episodes": [], - "fixed_episodes": [], - } - ], - } - - -def test_json_and_flattened_csv_are_written_without_opaque_objects(tmp_path) -> None: - json_path, csv_path = write_adaptation_artifacts(_artifact(), tmp_path) - payload = json.loads(json_path.read_text(encoding="utf-8")) - assert payload["replicates"][0]["shared_pre_shift_episodes"][0]["transitions"][0][ - "observation" - ] == [0.1, 0.2] - with csv_path.open(newline="", encoding="utf-8") as handle: - rows = list(csv.DictReader(handle)) - assert len(rows) == 1 - assert rows[0]["arm"] == "shared" - assert rows[0]["phase"] == "pre" - assert rows[0]["episode_seed"] == "123" - - -def test_existing_artifacts_are_never_overwritten(tmp_path) -> None: - json_path, csv_path = write_adaptation_artifacts(_artifact(), tmp_path) - before_json = json_path.read_bytes() - before_csv = csv_path.read_bytes() - with pytest.raises(FileExistsError): - write_adaptation_artifacts(_artifact(), tmp_path) - assert json_path.read_bytes() == before_json - assert csv_path.read_bytes() == before_csv - - -def test_nonfinite_values_are_rejected_for_strict_json(tmp_path) -> None: - data = _artifact() - data["value"] = float("nan") - with pytest.raises(ValueError, match="JSON compliant"): - write_adaptation_artifacts(data, tmp_path) - - -def test_manifest_hashes_all_artifacts_and_detects_tampering(tmp_path) -> None: - json_path, csv_path = write_adaptation_artifacts( - _artifact(), tmp_path, stem="adaptive_vs_fixed" - ) - training_artifact = tmp_path / "training" / "seed_31001" / "weights.zip" - training_artifact.parent.mkdir(parents=True) - training_artifact.write_bytes(b"real-artifact-bytes") - manifest_path = tmp_path / "manifest.json" - manifest = write_study_manifest( - json_path, - csv_path, - manifest_path, - run_id="test-run", - command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id test-run", - ) - assert manifest["artifacts"][json_path.name] == sha256_file(json_path) - assert manifest["artifacts"][csv_path.name] == sha256_file(csv_path) - assert manifest["artifacts"]["training/seed_31001/weights.zip"] == sha256_file( - training_artifact - ) - validate_study_manifest(manifest_path) - with pytest.raises(FileExistsError): - write_study_manifest( - json_path, - csv_path, - manifest_path, - run_id="test-run", - command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id test-run", - ) - training_artifact.write_bytes(b"tampered") - assert manifest["artifacts"]["training/seed_31001/weights.zip"] != sha256_file( - training_artifact - ) - with pytest.raises(ValueError, match="checksum mismatch"): - validate_study_manifest(manifest_path) - - -def test_manifest_rejects_unlisted_files_added_after_completion(tmp_path) -> None: - json_path, csv_path = write_adaptation_artifacts( - _artifact(), tmp_path, stem="adaptive_vs_fixed" - ) - manifest_path = tmp_path / "manifest.json" - write_study_manifest( - json_path, - csv_path, - manifest_path, - run_id="test-run", - command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id test-run", - ) - validate_study_manifest(manifest_path) - - (tmp_path / "unexpected.json").write_text("{}\n", encoding="utf-8") - with pytest.raises(ValueError, match="unlisted artifacts: unexpected.json"): - validate_study_manifest(manifest_path) - - -def test_completed_manifest_binds_preexecution_study_hash(tmp_path) -> None: - spec_path = tmp_path / "study_manifest.json" - study_hash = write_or_verify_study_manifest( - {"protocol": "prereg-v1", "seed": 31001}, spec_path, resume=False - ) - json_path, csv_path = write_adaptation_artifacts( - _artifact(), tmp_path, stem="adaptive_vs_fixed" - ) - manifest_path = tmp_path / "manifest.json" - write_study_manifest( - json_path, - csv_path, - manifest_path, - run_id="bound-study", - command="adaptive-rl benchmark adaptation --study prereg-v1", - ) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - assert manifest["study_hash"] == study_hash - validate_study_manifest(manifest_path) - - changed_spec = json.loads(spec_path.read_text(encoding="utf-8")) - changed_spec["inputs"]["seed"] = 31002 - spec_path.write_text(json.dumps(changed_spec), encoding="utf-8") - manifest["artifacts"]["study_manifest.json"] = sha256_file(spec_path) - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - with pytest.raises(ValueError, match="does not match its pre-execution spec"): - validate_study_manifest(manifest_path) - - -def test_manifest_rejects_path_escape_and_unsupported_schema(tmp_path) -> None: - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text( - json.dumps({"schema_version": "1.0", "artifacts": {"../outside": "0" * 64}}), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="escapes the run directory"): - validate_study_manifest(manifest_path) - manifest_path.write_text( - json.dumps({"schema_version": "2.0", "artifacts": {"x": "0" * 64}}), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="unsupported or malformed"): - validate_study_manifest(manifest_path) - - -def test_manifest_rejects_symlinked_artifact_outside_run_directory(tmp_path) -> None: - outside = tmp_path.parent / f"{tmp_path.name}-outside" - outside.write_bytes(b"outside") - link = tmp_path / "external.bin" - link.symlink_to(outside) - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text( - json.dumps( - { - "schema_version": "1.0", - "artifacts": {"external.bin": sha256_file(outside)}, - } - ), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="escapes the run directory"): - validate_study_manifest(manifest_path) - - -def test_manifest_rejects_internal_symlinked_artifact(tmp_path) -> None: - target = tmp_path / "target.bin" - target.write_bytes(b"bound-bytes") - link = tmp_path / "alias.bin" - link.symlink_to(target) - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text( - json.dumps( - { - "schema_version": "1.0", - "artifacts": {"alias.bin": sha256_file(target)}, - } - ), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="artifact is a symlink"): - validate_study_manifest(manifest_path) - - -def test_study_csv_has_one_row_per_arm_and_preserves_finite_censoring(tmp_path) -> None: - data = { - "replicates": [ - { - "training_seed": 31001, - "status": "completed", - "shared_pre_shift_episodes": [{"reward": 10.0}], - "shared_shock_episodes": [{"reward": 2.0}], - "adaptive_episodes": [{"reward": 3.0}], - "fixed_episodes": [{"reward": 2.5}], - "adaptive_recovery": {"status": "right_censored", "T_H": 15}, - "fixed_recovery": {"status": "right_censored", "T_H": 15}, - "seeds": {"pre": [11], "post": [12, 13], "update": [14]}, - } - ] - } - _, csv_path = write_adaptive_vs_fixed_artifacts(data, tmp_path) - with csv_path.open(newline="", encoding="utf-8") as handle: - rows = list(csv.DictReader(handle)) - assert len(rows) == 2 - assert {row["arm"] for row in rows} == {"adaptive", "fixed"} - assert [row["T_H"] for row in rows] == ["15", "15"] - assert [row["recovery_status"] for row in rows] == ["right_censored"] * 2 - assert json.loads(rows[0]["post_returns"]) == [2.0, 3.0] - - -def test_replicate_checkpoint_is_terminal_hashed_and_tamper_evident(tmp_path) -> None: - checkpoint = tmp_path / "replicate_state" / "seed_31001.json" - study_hash = "study-hash" - protocol_hash = "protocol-hash" - replicate = {"training_seed": 31001, "status": "failed", "failure_reason": "crash"} - write_replicate_checkpoint( - replicate, - checkpoint, - study_hash=study_hash, - protocol_hash=protocol_hash, - ) - assert read_replicate_checkpoint( - checkpoint, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seed=31001, - )["failure_reason"] == "crash" - original = checkpoint.read_bytes() - with pytest.raises(FileExistsError): - write_replicate_checkpoint( - {"training_seed": 31001, "status": "failed", "failure_reason": "other"}, - checkpoint, - study_hash=study_hash, - protocol_hash=protocol_hash, - ) - assert checkpoint.read_bytes() == original - with pytest.raises(ValueError, match="study hash mismatch"): - read_replicate_checkpoint( - checkpoint, - study_hash="different-study", - protocol_hash=protocol_hash, - training_seed=31001, - ) - with pytest.raises(ValueError, match="protocol hash mismatch"): - read_replicate_checkpoint( - checkpoint, - study_hash=study_hash, - protocol_hash="different-protocol", - training_seed=31001, - ) - with pytest.raises(ValueError, match="identity mismatch"): - read_replicate_checkpoint( - checkpoint, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seed=31002, - ) - checkpoint.write_text('{"training_seed":31001,"status":"completed"}', encoding="utf-8") - with pytest.raises(ValueError, match="checksum mismatch"): - read_replicate_checkpoint( - checkpoint, - study_hash=study_hash, - protocol_hash=protocol_hash, - training_seed=31001, - ) - - -def test_replicate_checkpoint_binds_training_artifact_files(tmp_path) -> None: - run_root = tmp_path / "run" - training_dir = run_root / "training" / "seed_31001" - training_dir.mkdir(parents=True) - model_path = training_dir / "model.zip" - model_path.write_bytes(b"frozen-model") - checkpoint_path = run_root / "replicate_state" / "seed_31001.json" - write_replicate_checkpoint( - {"training_seed": 31001, "status": "completed"}, - checkpoint_path, - study_hash="study-hash", - protocol_hash="protocol-hash", - artifact_root=run_root, - artifact_directories=(training_dir,), - ) - - read_replicate_checkpoint( - checkpoint_path, - study_hash="study-hash", - protocol_hash="protocol-hash", - training_seed=31001, - ) - model_path.write_bytes(b"changed-model") - with pytest.raises(ValueError, match="artifact checksum mismatch"): - read_replicate_checkpoint( - checkpoint_path, - study_hash="study-hash", - protocol_hash="protocol-hash", - training_seed=31001, - ) - model_path.write_bytes(b"frozen-model") - (training_dir / "unexpected.json").write_text("{}\n", encoding="utf-8") - with pytest.raises(ValueError, match="artifact file set changed"): - read_replicate_checkpoint( - checkpoint_path, - study_hash="study-hash", - protocol_hash="protocol-hash", - training_seed=31001, - ) - - -def test_canonical_study_hash_ignores_mapping_order_and_json_formatting(tmp_path) -> None: - left = {"config": {"seed": 31001, "lr": 0.001}, "schedule": [1, 2, 3]} - right = {"schedule": [1, 2, 3], "config": {"lr": 0.001, "seed": 31001}} - assert canonical_json_bytes(left) == canonical_json_bytes(right) - assert make_study_manifest(left)["study_hash"] == make_study_manifest(right)["study_hash"] - - path = tmp_path / "study_manifest.json" - digest = write_or_verify_study_manifest(left, path, resume=False) - path.write_text(json.dumps(json.loads(path.read_text()), indent=4), encoding="utf-8") - assert write_or_verify_study_manifest(right, path, resume=True) == digest - - -def test_study_hash_binds_artifact_schema_versions() -> None: - base = { - "protocol": "prereg-v1", - "artifact_schema_version": "1.0", - "manifest_schema_version": "1.0", - "replicate_checkpoint_schema_version": "1.1", - } - changed = {**base, "artifact_schema_version": "2.0"} - assert make_study_manifest(base)["study_hash"] != make_study_manifest(changed)["study_hash"] - - -def test_study_manifest_material_change_and_tampered_hash_fail_resume(tmp_path) -> None: - path = tmp_path / "study_manifest.json" - inputs = {"config": {"gamma": 0.99}, "seeds": [1, 2]} - write_or_verify_study_manifest(inputs, path, resume=False) - with pytest.raises(ValueError, match="study hash mismatch"): - write_or_verify_study_manifest( - {"config": {"gamma": 0.98}, "seeds": [1, 2]}, path, resume=True - ) - - document = json.loads(path.read_text(encoding="utf-8")) - document["study_hash"] = "0" * 64 - path.write_text(json.dumps(document), encoding="utf-8") - with pytest.raises(ValueError, match="hash is invalid"): - write_or_verify_study_manifest(inputs, path, resume=True) - - -def test_resume_rejects_missing_or_malformed_study_manifest(tmp_path) -> None: - path = tmp_path / "study_manifest.json" - with pytest.raises(ValueError, match="valid immutable study manifest"): - write_or_verify_study_manifest({"a": 1}, path, resume=True) - path.write_text("{broken", encoding="utf-8") - with pytest.raises(ValueError, match="valid immutable study manifest"): - write_or_verify_study_manifest({"a": 1}, path, resume=True) - - -def test_issue271_preregistered_inputs_match_frozen_hashes() -> None: - config = load_config("configs/drone_distribution_shift.yaml") - assert compute_config_sha256(config) == ISSUE271_CONFIG_SHA256 - assert sha256_file("docs/research/TREATMENT_CARD.md") == ISSUE271_TREATMENT_CARD_SHA256 diff --git a/tests/test_adaptation_environment.py b/tests/test_adaptation_environment.py deleted file mode 100644 index 153cdaf..0000000 --- a/tests/test_adaptation_environment.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Deterministic wind shift support in the existing drone environment.""" - -from __future__ import annotations - -import numpy as np - -from adaptive_rl.config import load_config -from adaptive_rl.environments.drone import DroneNavigation3DEnv - - -def test_issue_265_config_freezes_nominal_and_test_b_parameters() -> None: - config = load_config("configs/drone_distribution_shift.yaml") - assert config.adaptation_benchmark is not None - assert config.adaptation_benchmark.scenario == "TEST-B" - assert config.environment.parameters["num_obstacles"] == 8 - assert config.environment.parameters["wind_speed"] == 0.5 - assert config.adaptation_benchmark.shift_parameters == { - "num_obstacles": 12, - "wind_speed": 4.0, - "gust_sigma": 0.6, - } - - -def test_steady_wind_is_an_external_force_and_keeps_action_limit() -> None: - env = DroneNavigation3DEnv( - bounds=(30.0, 30.0, 15.0), - num_obstacles=0, - max_steps=2, - wind_speed=4.0, - gust_sigma=0.0, - linear_damping=0.05, - ) - try: - env.reset(seed=17) - env.step(np.zeros(3, dtype=np.float32)) - state = env.drone_state - assert state.velocity[0] > 0.0 - assert np.isclose(np.linalg.norm(state.acceleration), 0.22, atol=1e-7) - assert env.action_space.contains(np.zeros(3, dtype=np.float32)) - finally: - env.close() - - -def test_ou_gust_stream_repeats_for_same_episode_seed() -> None: - environments = [ - DroneNavigation3DEnv( - bounds=(30.0, 30.0, 15.0), - num_obstacles=0, - max_steps=3, - wind_speed=4.0, - gust_sigma=0.6, - gust_theta=0.15, - ) - for _ in range(2) - ] - try: - for env in environments: - env.reset(seed=791) - for _ in range(3): - outputs = [env.step(np.zeros(3, dtype=np.float32)) for env in environments] - np.testing.assert_array_equal(outputs[0][0], outputs[1][0]) - np.testing.assert_array_equal( - outputs[0][4]["wind_velocity"], outputs[1][4]["wind_velocity"] - ) - finally: - for env in environments: - env.close() diff --git a/tests/test_adaptation_invariants.py b/tests/test_adaptation_invariants.py deleted file mode 100644 index 0f22f7d..0000000 --- a/tests/test_adaptation_invariants.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Mutation checks for prereg-v1 runtime invariant auditing.""" - -from __future__ import annotations - -from copy import deepcopy - -import pytest - -from adaptive_rl.benchmarking.adaptation_runner import ( - _audit_replicate_invariants, - _restore_replicate, - _return_vector_fingerprint, -) -from adaptive_rl.protocol.constants import TRAINING_SEEDS -from adaptive_rl.protocol.seeds import frozen_schedule - - -@pytest.fixture -def valid_replicate(): - schedule = frozen_schedule() - seed = TRAINING_SEEDS[0] - fingerprint = "frozen" - pre = [{"reward": 2.0} for _ in range(15)] - shock = [{"reward": 0.0} for _ in range(5)] - pre_hash = _return_vector_fingerprint(pre) - shock_hash = _return_vector_fingerprint(shock) - replicate = { - "training_seed": seed, - "shared_pre_shift_episodes": pre, - "shared_shock_episodes": shock, - "adaptive_episodes": [{"update_block": block} for block in range(5, 15)], - "fixed_episodes": [ - {"policy_fingerprint_start": fingerprint, "policy_fingerprint_end": fingerprint} - for _ in range(10) - ], - "update_blocks": [ - { - "block_episode": block, - "visible_episode_indices": list(range(1, block + 1)), - "update_seed": schedule[seed]["update"][block - 5], - "loss_metrics": {"train/loss": 1.0}, - } - for block in range(5, 15) - ], - "seeds": schedule[seed], - "arm_input_fingerprints": { - arm: {"pre": pre_hash, "shock": shock_hash} for arm in ("adaptive", "fixed") - }, - "fixed_weight_update_count": 0, - "fixed_parameter_delta_l2": 0.0, - "fixed_final_fingerprint": fingerprint, - "frozen_fingerprint": fingerprint, - "fork_fingerprint": fingerprint, - } - return replicate, schedule - - -@pytest.mark.parametrize( - ("mutation", "failed_check"), - [ - ("pre", "pre_shift_identical_across_arms"), - ("shock", "shock_returns_identical_across_arms"), - ("fixed", "fixed_zero_updates"), - ("leak", "no_future_data"), - ("fork", "fork_fingerprint_identical"), - ], -) -def test_each_preregistered_invariant_violation_fails_audit( - valid_replicate, mutation, failed_check -): - replicate, schedule = valid_replicate - replicate = deepcopy(replicate) - if mutation == "pre": - replicate["arm_input_fingerprints"]["fixed"]["pre"] = "different" - elif mutation == "shock": - replicate["arm_input_fingerprints"]["adaptive"]["shock"] = "different" - elif mutation == "fixed": - replicate["fixed_weight_update_count"] = 1 - elif mutation == "leak": - replicate["update_blocks"][0]["visible_episode_indices"].append(6) - else: - replicate["fork_fingerprint"] = "different" - - audit = _audit_replicate_invariants(replicate, schedule) - assert audit[failed_check] is False - assert audit["all_passed"] is False - - -def test_replicate_restore_reconstructs_terminal_state() -> None: - restored = _restore_replicate( - { - "training_seed": 31001, - "status": "failed", - "failure_reason": "recorded failure", - } - ) - assert restored.training_seed == 31001 - assert restored.status == "failed" - assert restored.failure_reason == "recorded failure" diff --git a/tests/test_adaptation_smoke.py b/tests/test_adaptation_smoke.py deleted file mode 100644 index 0bdfee7..0000000 --- a/tests/test_adaptation_smoke.py +++ /dev/null @@ -1,257 +0,0 @@ -"""CI-sized end-to-end Issue #265 protocol smoke test.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest -from typer.testing import CliRunner - -from adaptive_rl.benchmarking.adaptation_artifacts import ( - write_adaptive_vs_fixed_artifacts, - write_or_verify_study_manifest, - write_replicate_checkpoint, - write_study_manifest, -) -from adaptive_rl.benchmarking.adaptation_runner import ( - _load_resume_replicates, - _read_completed_study_artifact, - _resume_replicate_checkpoint, - run_adaptation_benchmark, -) -from adaptive_rl.cli import app -from adaptive_rl.config import load_config -from adaptive_rl.protocol.constants import TRAINING_SEEDS -from adaptive_rl.protocol.seeds import frozen_schedule, schedule_fingerprint - - -def test_cli_adaptation_smoke_runs_complete_protocol_and_writes_artifacts(tmp_path: Path) -> None: - output_dir = tmp_path / "adaptation-smoke" - result = CliRunner().invoke( - app, - [ - "benchmark", - "adaptation", - "--smoke", - "--stochastic", - "--output-dir", - str(output_dir), - ], - ) - assert result.exit_code == 0, result.output - artifact = json.loads((output_dir / "adaptation.json").read_text(encoding="utf-8")) - assert artifact["run_type"] == "smoke" - replicate = artifact["replicates"][0] - assert replicate["training_seed"] == TRAINING_SEEDS[0] - assert replicate["status"] == "completed" - train_config = replicate["training_provenance"]["effective_config"] - assert train_config["adaptation_benchmark"] is None - assert train_config["environment"]["parameters"]["num_obstacles"] == 8 - assert train_config["environment"]["parameters"]["wind_speed"] == 0.5 - assert train_config["environment"]["parameters"]["gust_sigma"] == 0.15 - assert replicate["schedule_fingerprint"] == schedule_fingerprint(frozen_schedule()) - assert len(replicate["shared_pre_shift_episodes"]) == 15 - assert len(replicate["shared_shock_episodes"]) == 5 - assert len(replicate["adaptive_episodes"]) == 10 - assert len(replicate["fixed_episodes"]) == 10 - assert [block["block_episode"] for block in replicate["update_blocks"]] == list(range(5, 15)) - assert [block["visible_episode_indices"] for block in replicate["update_blocks"]] == [ - list(range(1, boundary + 1)) for boundary in range(5, 15) - ] - assert [e["episode_seed"] for e in replicate["adaptive_episodes"]] == [ - e["episode_seed"] for e in replicate["fixed_episodes"] - ] - assert replicate["fixed_final_fingerprint"] == replicate["frozen_fingerprint"] - assert artifact["paired_analysis"]["drone_disturbed/ppo"]["status"] == "inconclusive" - assert (output_dir / "adaptation.csv").is_file() - - -def test_cli_adaptation_rejects_deterministic_ppo_before_creating_outputs( - tmp_path: Path, -) -> None: - output_dir = tmp_path / "invalid-deterministic-ppo" - result = CliRunner().invoke( - app, - ["benchmark", "adaptation", "--smoke", "--output-dir", str(output_dir)], - ) - assert result.exit_code == 1 - assert "not valid on-policy rollout data" in result.output - assert not output_dir.exists() - - -@pytest.mark.parametrize( - ("args", "message"), - [ - (["--algorithm", "dqn"], "--algorithm must be 'ppo' or 'sac'"), - (["--resume"], "--resume requires --study prereg-v1 and --run-id"), - ( - ["--study", "prereg-v1", "--run-id", "hash-mismatch", "--stochastic"], - "frozen Issue #271 configuration", - ), - ], -) -def test_cli_adaptation_rejects_invalid_or_mismatched_requests( - args: list[str], message: str -) -> None: - result = CliRunner().invoke(app, ["benchmark", "adaptation", *args]) - assert result.exit_code == 1 - assert message in " ".join(result.output.split()) - - -def test_preregistered_study_rejects_subset_before_training(tmp_path: Path) -> None: - result = CliRunner().invoke( - app, - [ - "benchmark", - "adaptation", - "--study", - "prereg-v1", - "--run-id", - "subset-rejection", - "--training-seeds", - "31001", - "--output-dir", - str(tmp_path), - ], - ) - assert result.exit_code == 1 - assert "all ten seeds in order" in " ".join(result.output.split()) - - -def test_preregistered_study_rejects_changed_scientific_config_before_training() -> None: - config = load_config("configs/drone_distribution_shift.yaml") - algorithm = config.algorithm.model_copy(update={"learning_rate": 0.0002}, deep=True) - changed = config.model_copy(update={"algorithm": algorithm}, deep=True) - with pytest.raises(ValueError, match="frozen Issue #271 configuration"): - run_adaptation_benchmark( - changed, - study_run_id="changed-config", - training_seeds=TRAINING_SEEDS, - ) - - -def test_preregistered_study_rejects_deterministic_ppo_rollout_before_training() -> None: - with pytest.raises(ValueError, match="not valid on-policy rollout data"): - run_adaptation_benchmark( - load_config("configs/drone_distribution_shift.yaml"), - study_run_id="deterministic-ppo-rejection", - training_seeds=TRAINING_SEEDS, - ) - - -def test_issue265_rejects_deterministic_ppo_rollout_before_creating_outputs( - tmp_path: Path, -) -> None: - output_dir = tmp_path / "must-not-be-created" - with pytest.raises(ValueError, match="not valid on-policy rollout data"): - run_adaptation_benchmark( - load_config("configs/drone_distribution_shift.yaml"), - output_dir=output_dir, - training_seeds=TRAINING_SEEDS[:1], - ) - assert not output_dir.exists() - - -def test_checkpoint_resume_helper_rejects_corrupted_terminal_record( - tmp_path: Path, -) -> None: - checkpoint_path = tmp_path / "seed_31001.json" - write_replicate_checkpoint( - {"training_seed": TRAINING_SEEDS[0], "status": "completed"}, - checkpoint_path, - study_hash="expected-study", - protocol_hash="expected-protocol", - ) - checkpoint_path.write_text("truncated", encoding="utf-8") - - with pytest.raises(ValueError, match="resume rejected invalid checkpoint"): - _resume_replicate_checkpoint( - checkpoint_path, - study_hash="expected-study", - protocol_hash="expected-protocol", - training_seed=TRAINING_SEEDS[0], - ) - - -def test_resume_preflight_rejects_orphan_checkpoint_digest_before_training( - tmp_path: Path, -) -> None: - state_root = tmp_path / "replicate_state" - state_root.mkdir() - (state_root / "seed_31001.json.sha256").write_text("0" * 64, encoding="ascii") - - with pytest.raises(ValueError, match="checkpoint or digest is missing for seed 31001"): - _load_resume_replicates( - state_root, - study_hash="study", - protocol_hash="protocol", - training_seeds=(TRAINING_SEEDS[0], TRAINING_SEEDS[1]), - ) - - -def test_completed_study_reader_returns_stable_persisted_artifact( - tmp_path: Path, -) -> None: - study_hash = write_or_verify_study_manifest( - {"protocol": "test", "seed": TRAINING_SEEDS[0]}, - tmp_path / "study_manifest.json", - resume=False, - ) - artifact = { - "schema_version": "1.0", - "run_id": "idempotent-run", - "study_hash": study_hash, - "run_status": "COMPLETE", - "replicates": [], - "artifact_paths": { - "json": "adaptive_vs_fixed.json", - "csv": "adaptive_vs_fixed.csv", - "manifest": "manifest.json", - }, - } - json_path, csv_path = write_adaptive_vs_fixed_artifacts(artifact, tmp_path) - manifest_path = tmp_path / "manifest.json" - write_study_manifest( - json_path, - csv_path, - manifest_path, - run_id="idempotent-run", - command="adaptive-rl benchmark adaptation --study prereg-v1 --run-id idempotent-run", - ) - - first_result = _read_completed_study_artifact( - json_path, - manifest_path, - run_id="idempotent-run", - study_hash=study_hash, - ) - resumed_result = _read_completed_study_artifact( - json_path, - manifest_path, - run_id="idempotent-run", - study_hash=study_hash, - ) - assert first_result == resumed_result == json.loads(json_path.read_text(encoding="utf-8")) - - -@pytest.mark.parametrize( - ("kwargs", "message"), - [ - ({"output_dir": Path("/tmp/absolute-study-output")}, "must be relative"), - ( - {"config_path": Path.cwd() / "configs/drone_distribution_shift.yaml"}, - "config path must be repository-relative", - ), - ], -) -def test_preregistered_study_rejects_nonreproducible_paths_before_training( - kwargs: dict[str, Path], message: str -) -> None: - with pytest.raises(ValueError, match=message): - run_adaptation_benchmark( - load_config("configs/drone_distribution_shift.yaml"), - study_run_id="invalid-path", - training_seeds=TRAINING_SEEDS, - **kwargs, - ) diff --git a/tests/test_adaptation_statistics.py b/tests/test_adaptation_statistics.py deleted file mode 100644 index 4c1d7c3..0000000 --- a/tests/test_adaptation_statistics.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Tests for paired Issue #265 statistical reporting and Holm correction.""" - -from __future__ import annotations - -import pytest - -from adaptive_rl.benchmarking.adaptation_statistics import ( - analyze_paired_recovery, - analyze_primary_cells, -) -from adaptive_rl.protocol.constants import PRIMARY_CELLS - - -def test_inconclusive_cell_suppresses_inference_below_minimum_valid_n() -> None: - result = analyze_paired_recovery( - fixed_t_h=[15.0] * 7 + [None, None, None], - adaptive_t_h=[10.0] * 7 + [None, None, None], - ) - assert result.valid_n == 7 - assert result.failed_n == 3 - assert result.status == "inconclusive" - assert result.primary_p_value is None - assert result.holm_adjusted_p_value is None - assert result.failure_imputation_bounds is not None - - -def test_evaluable_cell_contains_paired_analysis_and_sensitivity_outputs() -> None: - result = analyze_paired_recovery( - fixed_t_h=[15.0] * 10, - adaptive_t_h=[6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 15.0, 15.0], - ) - assert result.status == "evaluable" - assert result.valid_n == 10 - assert result.mean_difference is not None and result.mean_difference < 0.0 - assert result.primary_p_value is not None - assert result.interval_95 is not None - assert result.exact_sign_p_value is not None - assert result.exact_wilcoxon_p_value is not None - assert result.bootstrap_interval_95 is not None - - -def test_holm_adjustment_covers_all_preregistered_cells_and_invalid_is_null() -> None: - outcomes = {cell: ([15.0] * 10, [6.0] * 10) for cell in PRIMARY_CELLS} - outcomes[PRIMARY_CELLS[-1]] = ([15.0] * 7 + [None] * 3, [6.0] * 7 + [None] * 3) - results = analyze_primary_cells(outcomes) - assert tuple(results) == PRIMARY_CELLS - assert results[PRIMARY_CELLS[0]].holm_adjusted_p_value is not None - assert results[PRIMARY_CELLS[-1]].status == "inconclusive" - assert results[PRIMARY_CELLS[-1]].primary_p_value is None - assert results[PRIMARY_CELLS[-1]].holm_adjusted_p_value is None - - -def test_cell_mapping_must_match_the_frozen_family() -> None: - with pytest.raises(ValueError, match="PRIMARY_CELLS"): - analyze_primary_cells({}) diff --git a/tests/test_algorithm_adaptation.py b/tests/test_algorithm_adaptation.py deleted file mode 100644 index 5ab8ce9..0000000 --- a/tests/test_algorithm_adaptation.py +++ /dev/null @@ -1,439 +0,0 @@ -"""Native PPO/SAC adaptation uses only recorded post-shift data.""" - -from __future__ import annotations - -import random - -import gymnasium as gym -import numpy as np -import pytest -import torch -from gymnasium import spaces -from stable_baselines3.common.logger import Logger - -from adaptive_rl.algorithms.adaptation import ( - PPOAdaptationAdapter, - SACAdaptationAdapter, - run_adaptation_update, -) -from adaptive_rl.algorithms.ppo import PPOAlgorithm -from adaptive_rl.algorithms.sac import SACAlgorithm -from adaptive_rl.benchmarking.adaptation_runtime import evaluate_episode -from adaptive_rl.protocol.adaptation import ( - PostShiftEpisode, - Transition, - UpdateBatch, - build_update_batch, -) -from adaptive_rl.protocol.fork import model_fingerprint -from adaptive_rl.protocol.seeds import derive_seed - - -class _OneStepEnv(gym.Env): - def __init__(self) -> None: - self.observation_space = spaces.Box(-1.0, 1.0, shape=(3,), dtype=np.float32) - self.action_space = spaces.Box(-1.0, 1.0, shape=(1,), dtype=np.float32) - - def reset(self, *, seed=None, options=None): - super().reset(seed=seed) - self.reset_seed = seed - return np.zeros(3, dtype=np.float32), {} - - def step(self, action): - assert self.action_space.contains(np.asarray(action, dtype=np.float32)) - return ( - np.zeros(3, dtype=np.float32), - 1.0, - False, - True, - {"success": True, "collision": False}, - ) - - -def _batch(algorithm, training_seed: int = 31001): - model = algorithm.model - assert model is not None - episodes = [] - for episode_index in range(1, 6): - transitions = [] - for step in range(3): - observation = np.full( - model.observation_space.shape, - episode_index * 0.01 + step * 0.1, - dtype=np.float32, - ) - next_observation = observation + 0.05 - if isinstance(algorithm, PPOAlgorithm): - obs_tensor, _ = model.policy.obs_to_tensor(observation) - with torch.no_grad(): - action_tensor, value_tensor, log_prob_tensor = model.policy(obs_tensor) - next_obs_tensor, _ = model.policy.obs_to_tensor(next_observation) - next_value_tensor = model.policy.predict_values(next_obs_tensor) - action = action_tensor.cpu().numpy().reshape(-1) - behavior_value = float(value_tensor.reshape(-1)[0].cpu()) - behavior_log_prob = float(log_prob_tensor.reshape(-1)[0].cpu()) - behavior_next_value = float(next_value_tensor.reshape(-1)[0].cpu()) - else: - action = np.zeros(model.action_space.shape, dtype=np.float32) - behavior_value = None - behavior_log_prob = None - behavior_next_value = None - transitions.append( - Transition( - observation=observation, - action=action, - reward=float(episode_index + step), - next_observation=next_observation, - terminated=False, - truncated=step == 2, - behavior_log_prob=behavior_log_prob, - behavior_value=behavior_value, - behavior_next_value=behavior_next_value, - ) - ) - episodes.append( - PostShiftEpisode( - index=episode_index, - seed=derive_seed(training_seed, "post", episode_index), - transitions=tuple(transitions), - ) - ) - return build_update_batch(training_seed, episodes, block_episode=5) - - -def _static_ppo_batch(seed: int) -> UpdateBatch: - transitions = tuple( - Transition( - observation=np.full(3, index / 10.0, dtype=np.float32), - action=np.zeros(1, dtype=np.float32), - reward=float(index % 4), - next_observation=np.full(3, (index + 1) / 10.0, dtype=np.float32), - terminated=index % 4 == 3, - truncated=False, - behavior_log_prob=-0.5, - behavior_value=0.0, - ) - for index in range(8) - ) - return UpdateBatch( - block_episode=5, - seed=seed, - visible_episode_indices=(1, 2, 3, 4, 5), - transitions=transitions, - ) - - -def test_ppo_native_update_uses_recorded_rollout_without_environment_steps() -> None: - env = gym.make("Pendulum-v1") - try: - algorithm = PPOAlgorithm( - env=env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" - ) - batch = _batch(algorithm) - model = algorithm.model - assert model is not None - model.set_logger(Logger(folder=None, output_formats=[])) - model.logger.record("train/stale_loss", 999.0) - before = model_fingerprint(algorithm) - log = run_adaptation_update(algorithm, PPOAdaptationAdapter(), batch) - assert log.loss_metrics - assert "train/stale_loss" not in log.loss_metrics - assert log.block_episode == 5 - assert log.transition_count == 15 - assert log.visible_episode_indices == (1, 2, 3, 4, 5) - assert log.update_seed == batch.seed - assert log.parameter_delta_l2 > 0.0 - assert log.fingerprint_before == before - assert log.fingerprint_after != before - assert model.num_timesteps == 0 - finally: - env.close() - - -def test_ppo_buffer_gae_matches_hand_calculation_for_terminal_and_truncation() -> None: - env = gym.make("Pendulum-v1") - try: - algorithm = PPOAlgorithm( - env=env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" - ) - model = algorithm.model - assert model is not None - values = (0.1, 0.2, 0.3, 0.4) - transitions = ( - Transition( - observation=np.zeros(3, dtype=np.float32), - action=np.zeros(1, dtype=np.float32), - reward=1.0, - next_observation=np.ones(3, dtype=np.float32), - terminated=False, - truncated=False, - behavior_log_prob=0.0, - behavior_value=values[0], - ), - Transition( - observation=np.ones(3, dtype=np.float32), - action=np.zeros(1, dtype=np.float32), - reward=2.0, - next_observation=np.full(3, 2.0, dtype=np.float32), - terminated=False, - truncated=True, - behavior_log_prob=0.0, - behavior_value=values[1], - behavior_next_value=0.5, - ), - Transition( - observation=np.full(3, 3.0, dtype=np.float32), - action=np.zeros(1, dtype=np.float32), - reward=3.0, - next_observation=np.full(3, 4.0, dtype=np.float32), - terminated=False, - truncated=False, - behavior_log_prob=0.0, - behavior_value=values[2], - ), - Transition( - observation=np.full(3, 4.0, dtype=np.float32), - action=np.zeros(1, dtype=np.float32), - reward=4.0, - next_observation=np.full(3, 5.0, dtype=np.float32), - terminated=True, - truncated=False, - behavior_log_prob=0.0, - behavior_value=values[3], - ), - ) - batch = UpdateBatch( - block_episode=5, - seed=derive_seed(31001, "update", 0), - visible_episode_indices=(1, 2, 3, 4, 5), - transitions=transitions, - ) - gamma = model.gamma - gae_lambda = model.gae_lambda - expected_last_truncated = 2.0 + gamma * 0.5 - values[1] - expected_first_truncated = ( - 1.0 + gamma * values[1] - values[0] - + gamma * gae_lambda * expected_last_truncated - ) - expected_last_terminated = 4.0 - values[3] - expected_first_terminated = ( - 3.0 + gamma * values[3] - values[2] - + gamma * gae_lambda * expected_last_terminated - ) - expected_advantages = np.asarray( - [ - expected_first_truncated, - expected_last_truncated, - expected_first_terminated, - expected_last_terminated, - ], - dtype=np.float32, - ) - captured: dict[str, np.ndarray] = {} - original_train = model.train - - def capture_buffer() -> None: - captured["advantages"] = model.rollout_buffer.advantages.copy().reshape(-1) - captured["returns"] = model.rollout_buffer.returns.copy().reshape(-1) - model.logger.record("train/loss", 1.0) - - model.train = capture_buffer - try: - PPOAdaptationAdapter().update(algorithm, batch) - finally: - model.train = original_train - np.testing.assert_allclose(captured["advantages"], expected_advantages, rtol=1e-6) - np.testing.assert_allclose( - captured["returns"], expected_advantages + np.asarray(values), rtol=1e-6 - ) - finally: - env.close() - - -def test_ppo_update_restores_caller_rng_and_repeats_from_same_update_seed() -> None: - envs = [gym.make("Pendulum-v1") for _ in range(3)] - try: - algorithms = [ - PPOAlgorithm( - env=env, - n_steps=8, - batch_size=4, - n_epochs=2, - seed=31001, - device="cpu", - ) - for env in envs - ] - assert len({model_fingerprint(algorithm) for algorithm in algorithms}) == 1 - python_state = random.getstate() - numpy_state = np.random.get_state() - torch_state = torch.random.get_rng_state().clone() - batch = _static_ppo_batch(derive_seed(31001, "update", 0)) - fingerprints = [] - for algorithm in algorithms[:2]: - run_adaptation_update(algorithm, PPOAdaptationAdapter(), batch) - fingerprints.append(model_fingerprint(algorithm)) - assert random.getstate() == python_state - after_numpy = np.random.get_state() - assert after_numpy[0] == numpy_state[0] - np.testing.assert_array_equal(after_numpy[1], numpy_state[1]) - assert after_numpy[2:] == numpy_state[2:] - assert torch.equal(torch.random.get_rng_state(), torch_state) - assert fingerprints[0] == fingerprints[1] - - other_batch = _static_ppo_batch(derive_seed(31001, "update", 1)) - run_adaptation_update(algorithms[2], PPOAdaptationAdapter(), other_batch) - assert model_fingerprint(algorithms[2]) != fingerprints[0] - finally: - for env in envs: - env.close() - - -@pytest.mark.parametrize( - ("failure", "error_match"), - [ - ("exception", "forced update failure"), - ("nonfinite_parameter", "non-finite model state"), - ("nonfinite_optimizer", "non-finite optimizer"), - ("invalid_shape", "shape or dtype"), - ], -) -def test_failed_ppo_update_restores_model_and_optimizer_state( - failure: str, error_match: str -) -> None: - env = gym.make("Pendulum-v1") - try: - algorithm = PPOAlgorithm( - env=env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" - ) - model = algorithm.model - assert model is not None - before_fingerprint = model_fingerprint(algorithm) - before_num_timesteps = model.num_timesteps - before_optimizer = model.policy.optimizer.state_dict() - - class FailingAdapter: - def update(self, target, batch) -> None: - del batch - target_model = target.model - parameter = next(target_model.policy.parameters()) - target_model.num_timesteps = 999 - if failure == "exception": - raise RuntimeError("forced update failure") - if failure == "nonfinite_parameter": - parameter.data.fill_(float("nan")) - elif failure == "nonfinite_optimizer": - target_model.policy.optimizer.state[parameter]["exp_avg"] = torch.full_like( - parameter, float("inf") - ) - else: - parameter.data = parameter.data.reshape(-1)[:-1] - - with pytest.raises((RuntimeError, FloatingPointError), match=error_match): - run_adaptation_update(algorithm, FailingAdapter(), _static_ppo_batch(1)) - restored = algorithm.model - assert restored is not None - assert model_fingerprint(algorithm) == before_fingerprint - assert restored.num_timesteps == before_num_timesteps - assert restored.policy.optimizer.state_dict() == before_optimizer - finally: - env.close() - - -def test_sac_native_update_uses_fresh_post_only_replay_buffer() -> None: - env = gym.make("Pendulum-v1") - try: - algorithm = SACAlgorithm( - env=env, - batch_size=4, - learning_starts=0, - gradient_steps=1, - buffer_size=100, - seed=31001, - device="cpu", - ) - batch = _batch(algorithm) - model = algorithm.model - assert model is not None - original_buffer = model.replay_buffer - before = model_fingerprint(algorithm) - native_train = model.train - replay_snapshot: dict[str, np.ndarray] = {} - - def inspect_replay(gradient_steps: int, batch_size: int) -> None: - replay = model.replay_buffer - assert replay is not None - size = replay.size() - replay_snapshot["observations"] = replay.observations[:size, 0].copy() - replay_snapshot["rewards"] = replay.rewards[:size, 0].copy() - replay_snapshot["dones"] = replay.dones[:size, 0].copy() - replay_snapshot["timeouts"] = replay.timeouts[:size, 0].copy() - native_train(gradient_steps=gradient_steps, batch_size=batch_size) - - model.train = inspect_replay - try: - log = run_adaptation_update(algorithm, SACAdaptationAdapter(), batch) - finally: - model.train = native_train - assert log.loss_metrics - assert log.block_episode == 5 - assert log.transition_count == 15 - assert log.visible_episode_indices == (1, 2, 3, 4, 5) - assert log.update_seed == batch.seed - assert log.parameter_delta_l2 > 0.0 - assert log.fingerprint_before == before - assert log.fingerprint_after != before - assert model.replay_buffer is original_buffer - assert original_buffer is not None and original_buffer.size() == 0 - assert model.num_timesteps == 0 - np.testing.assert_allclose( - replay_snapshot["observations"], - np.stack([transition.observation for transition in batch.transitions]), - ) - np.testing.assert_allclose( - replay_snapshot["rewards"], [transition.reward for transition in batch.transitions] - ) - np.testing.assert_array_equal( - replay_snapshot["dones"], - [float(transition.terminated or transition.truncated) for transition in batch.transitions], - ) - np.testing.assert_array_equal( - replay_snapshot["timeouts"], - [float(transition.truncated and not transition.terminated) for transition in batch.transitions], - ) - finally: - env.close() - - -def test_episode_collector_records_transition_and_keeps_policy_constant() -> None: - model_env = gym.make("Pendulum-v1") - try: - algorithm = PPOAlgorithm( - env=model_env, n_steps=8, batch_size=4, n_epochs=1, seed=31001, device="cpu" - ) - env = _OneStepEnv() - before = model_fingerprint(algorithm) - record = evaluate_episode( - algorithm=algorithm, - env=env, - training_seed=31001, - phase="post", - episode_index=1, - episode_seed=derive_seed(31001, "post", 1), - algorithm_name="ppo", - environment_name="fake_drone", - deterministic=True, - arm="shared", - ) - assert env.reset_seed == record.episode_seed - assert record.length == 1 - assert record.success is True - assert record.collision is False - assert len(record.transitions) == 1 - assert record.transitions[0].behavior_log_prob is not None - assert record.transitions[0].behavior_value is not None - assert record.transitions[0].behavior_next_value is not None - assert record.policy_fingerprint_start == record.policy_fingerprint_end == before - assert model_fingerprint(algorithm) == before - finally: - model_env.close() diff --git a/tests/test_protocol_adaptation_boundary.py b/tests/test_protocol_adaptation_boundary.py deleted file mode 100644 index afe0b00..0000000 --- a/tests/test_protocol_adaptation_boundary.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Leakage and ordering tests for the Issue #265 update boundary.""" - -from __future__ import annotations - -import numpy as np -import pytest - -from adaptive_rl.protocol.adaptation import ( - PostShiftEpisode, - Transition, - build_update_batch, - call_update_atomically, - validate_block_sequence, -) -from adaptive_rl.protocol.seeds import derive_seed - - -def _episode(training_seed: int, index: int) -> PostShiftEpisode: - transition = Transition( - observation=np.asarray([index], dtype=np.float32), - action=np.asarray([index], dtype=np.float32), - reward=float(index), - next_observation=np.asarray([index + 0.5], dtype=np.float32), - terminated=False, - truncated=True, - ) - return PostShiftEpisode( - index=index, - seed=derive_seed(training_seed, "post", index), - transitions=(transition,), - ) - - -@pytest.mark.parametrize("boundary", range(5, 15)) -def test_each_update_sees_exactly_its_completed_post_prefix(boundary: int) -> None: - training_seed = 31001 - history = tuple(_episode(training_seed, index) for index in range(1, boundary + 1)) - batch = build_update_batch(training_seed, history, block_episode=boundary) - - assert batch.visible_episode_indices == tuple(range(1, boundary + 1)) - assert [int(t.observation[0]) for t in batch.transitions] == list(range(1, boundary + 1)) - assert batch.seed == derive_seed(training_seed, "update", boundary - 5) - - -def test_update_boundary_rejects_future_missing_or_misordered_data() -> None: - history = tuple(_episode(31001, index) for index in range(1, 7)) - with pytest.raises(ValueError, match="exactly episodes 1..5"): - build_update_batch(31001, history, block_episode=5) - with pytest.raises(ValueError, match="ordered completed episodes"): - build_update_batch( - 31001, (history[0], history[2], history[1], *history[3:]), block_episode=6 - ) - with pytest.raises(ValueError, match="only after"): - build_update_batch(31001, history, block_episode=15) - - -def test_block_schedule_has_no_b15() -> None: - validate_block_sequence(tuple(range(5, 15))) - with pytest.raises(ValueError, match="exactly"): - validate_block_sequence(tuple(range(5, 16))) - - -def test_transition_arrays_are_detached_and_read_only() -> None: - observation = np.asarray([1.0], dtype=np.float32) - transition = Transition(observation, observation, 1.0, observation, False, True) - observation[0] = 99.0 - assert transition.observation[0] == 1.0 - with pytest.raises(ValueError): - transition.observation[0] = 2.0 - - -def test_failed_update_rolls_back_adapter_target_state() -> None: - class Target: - def __init__(self) -> None: - self.weight = [1.0] - - class FailingAdapter: - def update(self, algorithm, batch) -> None: - del batch - algorithm.weight[0] = 2.0 - raise RuntimeError("invalid update") - - target = Target() - with pytest.raises(RuntimeError, match="invalid update"): - call_update_atomically(target, FailingAdapter(), object()) # type: ignore[arg-type] - assert target.weight == [1.0] diff --git a/tests/test_protocol_fork.py b/tests/test_protocol_fork.py deleted file mode 100644 index c8f770a..0000000 --- a/tests/test_protocol_fork.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Regression coverage for paired policy fork isolation.""" - -from __future__ import annotations - -import pytest - -from adaptive_rl.protocol.fork import fork_adaptive_and_fixed, model_fingerprint - -torch = pytest.importorskip("torch") - - -class _Model(torch.nn.Module): - def __init__(self) -> None: - super().__init__() - self.layer = torch.nn.Linear(3, 2) - - -class _Algorithm: - def __init__(self) -> None: - self.model = _Model() - self.optimizer = torch.optim.Adam(self.model.parameters()) - - def predict(self, observation, deterministic=True): - del deterministic - with torch.no_grad(): - return self.model(torch.as_tensor(observation, dtype=torch.float32)), None - - def train(self, *args, **kwargs): - del args, kwargs - raise AssertionError("fixed facade must not expose training") - - -def test_fork_is_equal_at_start_and_independently_mutable() -> None: - source = _Algorithm() - expected = model_fingerprint(source) - adaptive, fixed, frozen_fingerprint = fork_adaptive_and_fixed(source) - - assert expected == frozen_fingerprint - assert model_fingerprint(adaptive) == fixed.fingerprint == expected - assert adaptive.model.layer.weight.data_ptr() != source.model.layer.weight.data_ptr() - assert ( - adaptive.model.layer.weight.data_ptr() - != fixed._FrozenPolicy__algorithm.model.layer.weight.data_ptr() - ) - assert adaptive.optimizer is not source.optimizer - assert fixed._FrozenPolicy__algorithm.optimizer is not source.optimizer - - with torch.no_grad(): - adaptive.model.layer.weight.add_(1.0) - assert model_fingerprint(adaptive) != expected - assert fixed.fingerprint == expected - - with pytest.raises(AttributeError): - fixed.train(10) - - -def test_model_fingerprint_is_deterministic_and_sensitive_to_buffers() -> None: - model = _Model() - initial = model_fingerprint(model) - assert model_fingerprint(model) == initial - with torch.no_grad(): - model.layer.bias[0].add_(0.25) - assert model_fingerprint(model) != initial