fix: correct continuous-action PPO scaling, seeding, and eval defaults; retrain pinned models

.gitignore

@@ -167,20 +167,29 @@ models/*_meta.json
 models/*_vecnorm.pkl
 
 # Pre-trained hybrid_cv01 models — explicitly tracked
+# VecNormalize stats are tracked too: play_ppo.py normalises observations from
+# <stem>_vecnorm.pkl at inference, so a model is not reproducible without it.
 !models/ppo_hcv01_*.zip
 !models/ppo_hcv01_*_vecnorm.pkl
 !models/ppo_hcv01_*_meta.json
 !models/hybrid_cv01_s42.zip
 !models/hybrid_cv01_s42_meta.json
+!models/hybrid_cv01_s42_vecnorm.pkl
 !models/hybrid_cv01_disc_s42.zip
 !models/hybrid_cv01_disc_s42_meta.json
+!models/hybrid_cv01_disc_s42_vecnorm.pkl
 !models/hybrid_cv01_s5775.zip
 !models/hybrid_cv01_s5775_meta.json
+!models/hybrid_cv01_s5775_vecnorm.pkl
 !models/hybrid_cv01_disc_s5775.zip
 !models/hybrid_cv01_disc_s5775_meta.json
+!models/hybrid_cv01_disc_s5775_vecnorm.pkl
 
 # Q-agent test artifacts
 tests/test_working_directory/*.json
 
+# Claude Code local session/config
+.claude/
+
 # Local dev docs (not for the repo)
 for_sig.md

models/hybrid_cv01_disc_s42_meta.json

@@ -18,7 +18,7 @@
     "n_envs": 32,
     "gamma": 0.99,
     "save_path": "models/hybrid_cv01_disc_s42.zip",
-    "seed": null,
+    "seed": 42,
     "ent_coef": 0.0,
     "learning_rate": 0.0003,
     "clip_range": 0.2,
@@ -28,7 +28,9 @@
     "start_speed": 1.0,
     "continuous_actions": false,
     "reward_limit": 50.0,
-    "max_episode_steps": 1000
+    "log_std_init": null,
+    "max_episode_steps": 1000,
+    "acc": 0.1
   },
   "config_source": "experiments/hybrid_cv01_disc_s42.yaml"
 }

models/hybrid_cv01_disc_s5775_meta.json

@@ -18,7 +18,7 @@
     "n_envs": 32,
     "gamma": 0.99,
     "save_path": "models/hybrid_cv01_disc_s5775.zip",
-    "seed": null,
+    "seed": 5775,
     "ent_coef": 0.0,
     "learning_rate": 0.0003,
     "clip_range": 0.2,
@@ -28,7 +28,9 @@
     "start_speed": 1.0,
     "continuous_actions": false,
     "reward_limit": 50.0,
-    "max_episode_steps": 1000
+    "log_std_init": null,
+    "max_episode_steps": 1000,
+    "acc": 0.1
   },
   "config_source": "experiments/hybrid_cv01_disc_s5775.yaml"
 }

models/hybrid_cv01_s42_meta.json

@@ -18,7 +18,7 @@
     "n_envs": 32,
     "gamma": 0.99,
     "save_path": "models/hybrid_cv01_s42.zip",
-    "seed": null,
+    "seed": 42,
     "ent_coef": 0.0,
     "learning_rate": 0.0003,
     "clip_range": 0.2,
@@ -28,7 +28,9 @@
     "start_speed": 1.0,
     "continuous_actions": true,
     "reward_limit": 50.0,
-    "max_episode_steps": 1000
+    "log_std_init": -3.2188758248682006,
+    "max_episode_steps": 1000,
+    "acc": 0.1
   },
   "config_source": "experiments/hybrid_cv01_s42.yaml"
 }

models/hybrid_cv01_s5775_meta.json

@@ -18,7 +18,7 @@
     "n_envs": 32,
     "gamma": 0.99,
     "save_path": "models/hybrid_cv01_s5775.zip",
-    "seed": null,
+    "seed": 5775,
     "ent_coef": 0.0,
     "learning_rate": 0.0003,
     "clip_range": 0.2,
@@ -28,7 +28,9 @@
     "start_speed": 1.0,
     "continuous_actions": true,
     "reward_limit": 50.0,
-    "max_episode_steps": 1000
+    "log_std_init": -3.2188758248682006,
+    "max_episode_steps": 1000,
+    "acc": 0.1
   },
   "config_source": "experiments/hybrid_cv01_s5775.yaml"
 }

scripts/play_ppo.py

@@ -135,8 +135,13 @@ def main() -> None:
     _ = parser.add_argument(
         "--randomize-start",
         action=argparse.BooleanOptionalAction,
-        default=config.training.randomize_start,
-        help="Randomise initial pendulum speed each episode (default from model sidecar).",
+        default=None,
+        help=(
+            "Randomise initial pendulum speed each episode. Default: the model "
+            "sidecar's value for single-speed playback, but forced OFF for "
+            "--speed-sweep so each row uses its exact requested speed. Pass "
+            "--randomize-start to re-enable it for a sweep."
+        ),
     )
     _ = parser.add_argument(
         "--start-speed",
@@ -165,8 +170,14 @@ def main() -> None:
         "--save-png",
         "--no-save-png",
         action=argparse.BooleanOptionalAction,
-        default=True,
-        help="Save 7-panel trajectory plot per episode alongside the model (default True).",
+        default=None,
+        help=(
+            "Save the per-episode trajectory plot alongside the model. Default: on "
+            "for single-speed playback, off for --speed-sweep (which would otherwise "
+            "write one PNG per swept speed). Recording traces needs render_mode "
+            "'plot'; --render-mode none is promoted to 'plot' (file only, no window) "
+            "when this is on, but --render-mode play-back cannot also save."
+        ),
     )
     _ = parser.add_argument(
         "--save-csv",
@@ -183,6 +194,31 @@ def main() -> None:
     )
     args = parser.parse_args()
 
+    if args.save_png is None:
+        args.save_png = not args.speed_sweep
+
+    if args.randomize_start is None:
+        args.randomize_start = False if args.speed_sweep else config.training.randomize_start
+        if args.speed_sweep and config.training.randomize_start:
+            LOGGER.info(
+                "--speed-sweep: randomize_start forced off (sidecar has it on) so each "
+                "swept speed is exact; pass --randomize-start to override."
+            )
+
+    # Trajectory traces are only recorded while render_mode == "plot" (see
+    # AntiPendulumEnv.step). Make --save-png actually take effect instead of
+    # silently doing nothing when the user asked not to render to a window.
+    render_mode = args.render_mode
+    if args.save_png:
+        if render_mode == "none":
+            render_mode = "plot"
+            LOGGER.info("--save-png: rendering to file (render_mode 'plot', no window).")
+        elif render_mode != "plot":
+            LOGGER.warning(
+                "--save-png has no effect with --render-mode %s; use 'plot' (or 'none') to save PNGs.",
+                render_mode,
+            )
+
     mep = args.max_episode_steps if args.max_episode_steps is not None else config.training.max_episode_steps
     speeds = SWEEP_SPEEDS if args.speed_sweep else [args.start_speed]
 
@@ -194,11 +230,12 @@ def main() -> None:
             "conf": AntiPendulumConfig(
                 start_speed=speeds[0],
                 randomize_start=args.randomize_start,
-                render_mode=args.render_mode,
+                render_mode=render_mode,
                 reward_fac=config.reward,
                 rail_limit=config.training.rail_limit,
                 reward_limit=config.training.reward_limit,
                 continuous_actions=args.continuous_actions,
+                acc=config.training.acc,
             ),
         },
         max_episode_steps=mep,

scripts/train_ppo.py

@@ -15,6 +15,7 @@
 
 import argparse
 import logging
+import math
 from pathlib import Path
 
 from crane_controller.crane_factory import build_crane
@@ -94,8 +95,9 @@ def main() -> None:  # noqa: PLR0915
     _ = parser.add_argument(
         "--seed",
         type=int,
-        default=None,
-        help="Random seed for PPO initialisation. Omit for non-deterministic training.",
+        default=config.training.seed,
+        help="Random seed for PPO initialisation (default from --config, or None for "
+        "non-deterministic training). Pass explicitly to override.",
     )
     _ = parser.add_argument(
         "--ent-coef",
@@ -153,8 +155,32 @@ def main() -> None:  # noqa: PLR0915
         default=config.training.max_episode_steps,
         help="TimeLimit cap per episode (default 1000).",
     )
+    _ = parser.add_argument(
+        "--acc",
+        type=float,
+        default=config.training.acc,
+        help="Crane acceleration magnitude in [-acc, +acc] (default 0.1). Continuous-action "
+        "training chatters and never settles at 0.1; try 0.01 for genuine rest "
+        "(see project_ppo_acc_squared_bug.md).",
+    )
+    _ = parser.add_argument(
+        "--log-std-init",
+        type=float,
+        default=config.training.log_std_init,
+        help="Initial log std for the continuous-action Gaussian policy. Default: "
+        "auto-derived as log(acc/2.5) so ~99%% of raw samples start inside the action "
+        "box (avoids clip-dominated early training, see project_ppo_acc_squared_bug.md). "
+        "Pass explicitly to override, e.g. 0.0 for SB3's own default (std=1.0). Only "
+        "affects fresh training, not --resume-from.",
+    )
     args = parser.parse_args()
 
+    # Auto-derive log_std_init from acc when not explicitly set, so it stays correctly
+    # sized for whatever acc this run uses (important once acc is swept/randomized across
+    # experiments, not just fixed at 0.1) — see project_ppo_acc_squared_bug.md.
+    if args.log_std_init is None and args.continuous_actions:
+        args.log_std_init = math.log(args.acc / 2.5)
+
     # Resolve final reward config: explicit --reward-fac beats loaded YAML/defaults.
     reward_config = RewardConfig(*args.reward_fac) if args.reward_fac is not None else config.reward
 
@@ -175,6 +201,8 @@ def main() -> None:  # noqa: PLR0915
             start_speed=args.start_speed,
             continuous_actions=args.continuous_actions,
             max_episode_steps=args.max_episode_steps,
+            acc=args.acc,
+            log_std_init=args.log_std_init,
         ),
         config_source=pre_args.config,
     )
@@ -219,6 +247,7 @@ def main() -> None:  # noqa: PLR0915
                 gamma=args.gamma,
                 save_path=args.save_path,
                 continuous_actions=args.continuous_actions,
+                acc=args.acc,
             ),
             config_source=pre_args.config,
         )
@@ -236,6 +265,7 @@ def main() -> None:  # noqa: PLR0915
                     rail_limit=args.rail_limit,
                     reward_limit=resume_config.training.reward_limit,
                     continuous_actions=args.continuous_actions,
+                    acc=resume_config.training.acc,
                 ),
             },
             save_path=args.save_path,
@@ -263,6 +293,7 @@ def main() -> None:  # noqa: PLR0915
                     rail_limit=experiment_config.training.rail_limit,
                     reward_limit=experiment_config.training.reward_limit,
                     continuous_actions=args.continuous_actions,
+                    acc=experiment_config.training.acc,
                 ),
             },
             save_path=args.save_path,
@@ -273,6 +304,7 @@ def main() -> None:  # noqa: PLR0915
             clip_range=args.clip_range,
             n_steps=args.n_steps,
             max_episode_steps=experiment_config.training.max_episode_steps,
+            log_std_init=experiment_config.training.log_std_init,
         )
         csv_path = str(Path(args.save_path).with_name(Path(args.save_path).stem + "_log.csv"))
         agent.do_training(args.steps, csv_path=csv_path)

src/crane_controller/envs/controlled_crane_pendulum.py

@@ -522,7 +522,7 @@ def reset(
                 self._reward_point = self._reward_plot_init("b.")
                 self.nresets = -1
 
-        else:  # reset between episodes. Data are available
+        elif self.rewards:  # reset between episodes, only if data is available
             self.reward_stats_calc(len(self.rewards))
             if self.conf.render_mode == "data":  # plot reward per step
                 x = self.reward_stats["steps"]
@@ -572,7 +572,9 @@ def step(
                 (observation, reward, terminated, truncated, info)
         """
         if self.conf.continuous_actions:
-            acc = float(np.asarray(action).flat[0]) * self.conf.acc
+            # action_space bounds are already ±self.conf.acc (physical units), so the
+            # sampled action is the acceleration directly — do not rescale by acc again.
+            acc = float(np.asarray(action).flat[0])
         else:
             action_idx = int(action)
             if action_idx not in self.action_to_acc:
@@ -610,7 +612,9 @@ def render(self, save_path: str | None = None) -> None:
         ----------
         save_path : str or None, optional
             If set and render_mode is ``"plot"``, save the figure to this path
-            instead of showing it interactively (default None).
+            instead of showing it interactively (default None). Trajectory
+            traces are only recorded while ``render_mode == "plot"``, so that
+            mode is required for a figure to be produced at all.
         """
         if self.conf.render_mode == "play-back":
             self.show_animation()

src/crane_controller/experiment_config.py

@@ -160,6 +160,21 @@ class TrainingConfig:
         100). Replaces the previous hardcoded value of 3000; shorter episodes
         let the discount factor propagate rail-penalty credit meaningfully
         (``0.99^100 ≈ 0.37`` vs ``0.99^3000 ≈ 10^-13``).
+    acc : float
+        Crane acceleration magnitude in ``[-acc, +acc]`` (default 0.1, matching
+        Sig's Q-learning agent). Continuous-action training at this value used to
+        converge to a persistent bang-bang chatter that never settled; the fix is
+        ``log_std_init`` (see below), not shrinking this value - ``0.1`` is fine
+        to leave at its default. See ``project_ppo_acc_squared_bug.md`` for the
+        full investigation, including a weaker-actuator alternative that also
+        works but sacrifices control authority.
+    log_std_init : float or None
+        Initial log standard deviation for the continuous-action Gaussian policy
+        (default None - auto-derived from ``acc`` as ``log(acc / 2.5)`` by
+        ``train_ppo.py`` so ~99% of raw samples start inside the action box; see
+        ``ProximalPolicyOptimizationAgent``'s docstring and
+        ``project_ppo_acc_squared_bug.md``). Pass explicitly to override, e.g.
+        ``0.0`` for SB3's own default (std=1.0).
     """
 
     steps: int = 100_000
@@ -176,7 +191,9 @@ class TrainingConfig:
     start_speed: float = 1.0
     continuous_actions: bool = True
     reward_limit: float = 50.0
+    log_std_init: float | None = None
     max_episode_steps: int = 1000
+    acc: float = 0.1
 
     @classmethod
     def from_dict(cls, d: Mapping[str, object]) -> TrainingConfig:
@@ -194,6 +211,7 @@ def from_dict(cls, d: Mapping[str, object]) -> TrainingConfig:
         """
         defaults = cls()
         seed_raw = d.get("seed", defaults.seed)
+        log_std_init_raw = d.get("log_std_init", defaults.log_std_init)
         return cls(
             steps=int(d.get("steps", defaults.steps)),  # type: ignore[arg-type,call-overload]
             n_envs=int(d.get("n_envs", defaults.n_envs)),  # type: ignore[arg-type,call-overload]
@@ -210,6 +228,8 @@ def from_dict(cls, d: Mapping[str, object]) -> TrainingConfig:
             continuous_actions=bool(d.get("continuous_actions", defaults.continuous_actions)),
             reward_limit=float(d.get("reward_limit", defaults.reward_limit)),  # type: ignore[arg-type]
             max_episode_steps=int(d.get("max_episode_steps", defaults.max_episode_steps)),  # type: ignore[arg-type,call-overload]
+            acc=float(d.get("acc", defaults.acc)),  # type: ignore[arg-type]
+            log_std_init=float(log_std_init_raw) if isinstance(log_std_init_raw, (int, float)) else None,
         )