From a09d608b3ae07d6d64f376d9af2641feb1dd10c6 Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Mon, 29 Jun 2026 13:37:35 +0200 Subject: [PATCH 1/7] Add log scaling support for reward coefficients and action types --- pufferlib/ocean/drive/drive.h | 133 +++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 59 deletions(-) diff --git a/pufferlib/ocean/drive/drive.h b/pufferlib/ocean/drive/drive.h index 940cf6a265..88e28ff5c0 100644 --- a/pufferlib/ocean/drive/drive.h +++ b/pufferlib/ocean/drive/drive.h @@ -164,7 +164,9 @@ static const int ROAD_OFFSETS[25][2] // Dynamics Models #define CLASSIC 0 #define JERK 1 - +// Action Type +#define DISCRETE 0 +#define CONTINUOUS 1 static const float ACCEL_LONG_LIMIT[2] = {-5.0f, 2.5f}; static const float ACCEL_LAT_LIMIT[2] = {-4.0f, 4.0f}; #define STEERING_LIMIT 0.667f @@ -465,26 +467,27 @@ struct Drive { typedef struct { float min_val; float max_val; + int log_scale; } RewardBound; static const RewardBound REWARD_BOUNDS[NUM_REWARD_COEFS] = { - {2.0f, 12.0f}, // REWARD_COEF_GOAL_RADIUS δ_goal ~ U(2, 12) - {0.0f, 20.0f}, // REWARD_COEF_GOAL_SPEED - {0.0f, 3.0f}, // REWARD_COEF_COLLISION α_collision ~ U(0, 3) - {0.0f, 3.0f}, // REWARD_COEF_OFFROAD α_boundary ~ U(0, 3) - {0.0f, 0.1f}, // REWARD_COEF_COMFORT α_comfort ~ U(0, 0.1) - {2.5e-4f, 2.5e-2f}, // REWARD_COEF_LANE_ALIGN α_l-align ~ U(2.5e-4, 2.5e-2) - {0.0f, 1.0f}, // REWARD_COEF_VEL_ALIGN α_vel-align ~ U(0, 1) - {2.5e-4f, 7.5e-3f}, // REWARD_COEF_LANE_CENTER α_l-center ~ U(2.5e-4, 7.5e-3) - {-0.5f, 0.5f}, // REWARD_COEF_CENTER_BIAS α_center-bias ~ U(-0.5, 0.5) - {0.0f, 5e-3f}, // REWARD_COEF_VELOCITY α_velocity = 2.5e-3 (fixed) - {2.5e-4f, 7.5e-3f}, // REWARD_COEF_REVERSE α_reverse ~ U(2.5e-4, 7.5e-3) - {0.0f, 1.0f}, // REWARD_COEF_STOP_LINE α_stop-line ~ U(0, 1) - {0.0f, 5e-5f}, // REWARD_COEF_TIMESTEP α_timestep = 2.5e-5 (fixed) - {0.0f, 1.0f}, // REWARD_COEF_OVERSPEED - {0.8f, 1.25f}, // REWARD_COEF_THROTTLE C_throttle - {0.8f, 1.25f}, // REWARD_COEF_STEER C_steer - {0.666f, 1.5f}, // REWARD_COEF_ACC C_acc + {2.0f, 12.0f, 0}, // REWARD_COEF_GOAL_RADIUS δ_goal ~ U(2, 12) + {0.0f, 20.0f, 0}, // REWARD_COEF_GOAL_SPEED δ_goal-speed ~ U(0, 20) + {0.0f, 3.0f, 0}, // REWARD_COEF_COLLISION α_collision ~ U(0, 3) + {0.0f, 3.0f, 0}, // REWARD_COEF_OFFROAD α_boundary ~ U(0, 3) + {0.0f, 0.1f, 1}, // REWARD_COEF_COMFORT α_comfort ~ logU(0, 0.1) + {2.5e-4f, 2.5e-2f, 1}, // REWARD_COEF_LANE_ALIGN α_l-align ~ logU(2.5e-4, 2.5e-2) + {0.0f, 1.0f, 0}, // REWARD_COEF_VEL_ALIGN α_vel-align ~ U(0, 1) + {2.5e-4f, 7.5e-3f, 1}, // REWARD_COEF_LANE_CENTER α_l-center ~ logU(2.5e-4, 7.5e-3) + {-0.5f, 0.5f, 0}, // REWARD_COEF_CENTER_BIAS α_center-bias ~ U(-0.5, 0.5) + {0.0f, 5e-3f, 0}, // REWARD_COEF_VELOCITY α_velocity = 2.5e-3 (fixed) + {2.5e-4f, 7.5e-3f, 1}, // REWARD_COEF_REVERSE α_reverse ~ logU(2.5e-4, 7.5e-3) + {0.0f, 1.0f, 0}, // REWARD_COEF_STOP_LINE α_stop-line ~ U(0, 1) + {0.0f, 5e-5f, 0}, // REWARD_COEF_TIMESTEP α_timestep = 2.5e-5 (fixed) + {0.0f, 1.0f, 0}, // REWARD_COEF_OVERSPEED α_overspeed ~ U(0, 1) + {0.8f, 1.25f, 0}, // REWARD_COEF_THROTTLE C_throttle + {0.8f, 1.25f, 0}, // REWARD_COEF_STEER C_steer + {0.666f, 1.5f, 0}, // REWARD_COEF_ACC C_acc }; // ======================================== @@ -534,16 +537,20 @@ static float compute_heading_diff(float heading1, float heading2) { return normalize_heading(heading1 - heading2); } -static float random_uniform(float min_val, float max_val) { +static float sample_uniform(float min_val, float max_val) { return min_val + ((float) rand() / (float) RAND_MAX) * (max_val - min_val); } -static float mixed_uniform(float a) { +static float sample_log_uniform(float min_val, float max_val) { + return expf(sample_uniform(logf(min_val), logf(max_val))); +} + +static float sample_mixed_uniform(float a) { // Mixed uniform distribution X(a) = 0.5*U(1/a, 1) + 0.5*U(1, a) if ((float) rand() / (float) RAND_MAX < 0.5f) { - return random_uniform(1.0f / a, 1.0f); + return sample_uniform(1.0f / a, 1.0f); } else { - return random_uniform(1.0f, a); + return sample_uniform(1.0f, a); } } @@ -2070,7 +2077,7 @@ static void compute_goals(Drive *env, int agent_idx) { for (int iter = 0; iter <= 4; iter++) { float total_spacing = 0.0f; for (int i = 0; i < num_target_waypoints; i++) { - goal_spacings[i] = random_uniform(env->min_waypoint_spacing, env->max_waypoint_spacing); + goal_spacings[i] = sample_uniform(env->min_waypoint_spacing, env->max_waypoint_spacing); total_spacing += goal_spacings[i]; } @@ -3024,9 +3031,9 @@ static int pop_completed_episode_summary(Drive *env, CompletedEpisodeSummary *ou static inline void sample_erratic_flags(Drive *env, Agent *agent) { agent->is_blind_partner - = (env->partner_blindness_prob > 0.0f && random_uniform(0.0f, 1.0f) < env->partner_blindness_prob) ? 1 : 0; + = (env->partner_blindness_prob > 0.0f && sample_uniform(0.0f, 1.0f) < env->partner_blindness_prob) ? 1 : 0; agent->is_phantom_braker - = (env->phantom_braking_prob > 0.0f && random_uniform(0.0f, 1.0f) < env->phantom_braking_prob) ? 1 : 0; + = (env->phantom_braking_prob > 0.0f && sample_uniform(0.0f, 1.0f) < env->phantom_braking_prob) ? 1 : 0; agent->phantom_braking_counter = 0; } @@ -3048,13 +3055,15 @@ static void generate_reward_coefs(Drive *env, Agent *agent) { }; for (int i = 0; i < (int) (sizeof(random_coefs) / sizeof(random_coefs[0])); i++) { int c = random_coefs[i]; - agent->reward_coefs[c] = random_uniform(REWARD_BOUNDS[c].min_val, REWARD_BOUNDS[c].max_val); + agent->reward_coefs[c] = REWARD_BOUNDS[c].log_scale + ? sample_log_uniform(REWARD_BOUNDS[c].min_val, REWARD_BOUNDS[c].max_val) + : sample_uniform(REWARD_BOUNDS[c].min_val, REWARD_BOUNDS[c].max_val); } agent->reward_coefs[REWARD_COEF_VELOCITY] = 2.5e-3f; agent->reward_coefs[REWARD_COEF_TIMESTEP] = 2.5e-5f; - agent->reward_coefs[REWARD_COEF_THROTTLE] = mixed_uniform(1.25f); - agent->reward_coefs[REWARD_COEF_STEER] = mixed_uniform(1.25f); - agent->reward_coefs[REWARD_COEF_ACC] = mixed_uniform(1.5f); + agent->reward_coefs[REWARD_COEF_THROTTLE] = sample_mixed_uniform(1.25f); + agent->reward_coefs[REWARD_COEF_STEER] = sample_mixed_uniform(1.25f); + agent->reward_coefs[REWARD_COEF_ACC] = sample_mixed_uniform(1.5f); } else { agent->reward_coefs[REWARD_COEF_GOAL_RADIUS] = env->goal_radius; agent->reward_coefs[REWARD_COEF_GOAL_SPEED] = env->goal_speed; @@ -3081,7 +3090,7 @@ static void generate_traffic_light_states(Drive *env) { float dt = env->dt; // 20% chance: disable ALL lights for this episode - int disable_all = (!env->eval_mode) && (random_uniform(0.0f, 1.0f) < TL_EPISODE_DISABLE_PROB); + int disable_all = (!env->eval_mode) && (sample_uniform(0.0f, 1.0f) < TL_EPISODE_DISABLE_PROB); for (int i = 0; i < env->num_traffic_elements; i++) { TrafficControlElement *tc = &env->traffic_elements[i]; @@ -3103,14 +3112,14 @@ static void generate_traffic_light_states(Drive *env) { if (!env->eval_mode) { // Individual removal - if (random_uniform(0.0f, 1.0f) < TL_INDIVIDUAL_REMOVE_PROB) { + if (sample_uniform(0.0f, 1.0f) < TL_INDIVIDUAL_REMOVE_PROB) { for (int t = 0; t < fill_steps; t++) { tc->states[t] = TRAFFIC_CONTROL_STATE_OFF; } continue; } // Always green - if (random_uniform(0.0f, 1.0f) < TL_ALWAYS_GREEN_PROB) { + if (sample_uniform(0.0f, 1.0f) < TL_ALWAYS_GREEN_PROB) { for (int t = 0; t < fill_steps; t++) { tc->states[t] = TRAFFIC_CONTROL_STATE_GREEN; } @@ -3125,9 +3134,9 @@ static void generate_traffic_light_states(Drive *env) { dur_yellow = TL_DEFAULT_YELLOW_DURATION; dur_red = TL_DEFAULT_RED_DURATION; } else { - dur_green = random_uniform(0.1 * TL_DEFAULT_GREEN_DURATION, TL_DEFAULT_GREEN_DURATION); - dur_yellow = random_uniform(0.5f * TL_DEFAULT_YELLOW_DURATION, 0.75f * TL_DEFAULT_YELLOW_DURATION); - dur_red = random_uniform(0.15f * TL_DEFAULT_RED_DURATION, 5.0f * TL_DEFAULT_RED_DURATION); + dur_green = sample_uniform(0.1 * TL_DEFAULT_GREEN_DURATION, TL_DEFAULT_GREEN_DURATION); + dur_yellow = sample_uniform(0.5f * TL_DEFAULT_YELLOW_DURATION, 0.75f * TL_DEFAULT_YELLOW_DURATION); + dur_red = sample_uniform(0.15f * TL_DEFAULT_RED_DURATION, 5.0f * TL_DEFAULT_RED_DURATION); } int steps_green = (int) (dur_green / dt); @@ -3248,12 +3257,12 @@ static bool spawn_agent(Drive *env, int agent_idx, int num_agents) { float spawn_length, spawn_width; if (env->eval_mode) { // Fixed size for eval mode - spawn_length = random_uniform(2.0f, 5.5f); - spawn_width = random_uniform(1.5f, 2.5f); + spawn_length = sample_uniform(2.0f, 5.5f); + spawn_width = sample_uniform(1.5f, 2.5f); } else { // Random size for training mode - spawn_length = random_uniform(0.8f, 7.0f); - spawn_width = random_uniform(0.8f, 2.7f); + spawn_length = sample_uniform(0.8f, 7.0f); + spawn_width = sample_uniform(0.8f, 2.7f); } if (spawn_width > spawn_length) { spawn_width = spawn_length; @@ -4624,8 +4633,17 @@ static int write_ego_obs(Drive *env, Agent *ego, float *obs, int obs_idx) { static int write_reward_target_obs(Drive *env, Agent *ego, float *obs, int obs_idx) { if (env->reward_conditioning) { for (int coef_idx = 0; coef_idx < NUM_REWARD_COEFS; coef_idx++) { - float normalized_coef = (ego->reward_coefs[coef_idx] - REWARD_BOUNDS[coef_idx].min_val) - / ((REWARD_BOUNDS[coef_idx].max_val - REWARD_BOUNDS[coef_idx].min_val) + 1e-8f); + float lo = REWARD_BOUNDS[coef_idx].min_val; + float hi = REWARD_BOUNDS[coef_idx].max_val; + float coef = ego->reward_coefs[coef_idx]; + float normalized_coef; + if (REWARD_BOUNDS[coef_idx].log_scale) { + // Match the log-uniform sampling so the conditioning signal stays even across [-1, 1]. + float clamped = fmaxf(lo, fminf(hi, coef)); + normalized_coef = (logf(clamped) - logf(lo)) / (logf(hi) - logf(lo)); + } else { + normalized_coef = (coef - lo) / ((hi - lo) + 1e-8f); + } float clamped_coef = fmaxf(0.0f, fminf(1.0f, normalized_coef)); obs[obs_idx++] = 2.0f * clamped_coef - 1.0f; } @@ -4682,7 +4700,7 @@ static int write_reward_target_obs(Drive *env, Agent *ego, float *obs, int obs_i } static int write_partner_obs(Drive *env, Agent *ego, int agent_idx, float *obs, int obs_idx, int *partner_count) { - if (ego->is_blind_partner && random_uniform(0.0f, 1.0f) < env->partner_blindness_trigger_prob) { + if (ego->is_blind_partner && sample_uniform(0.0f, 1.0f) < env->partner_blindness_trigger_prob) { int partner_obs_stride = env->obs_slots_partners_n * PARTNER_FEATURES; memset(&obs[obs_idx], 0, partner_obs_stride * sizeof(float)); *partner_count = 0; @@ -5010,7 +5028,7 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { phantom_braking_active = 1; } else if ( agent->is_phantom_braker && env->phantom_braking_trigger_prob > 0.0f - && random_uniform(0.0f, 1.0f) < env->phantom_braking_trigger_prob) { + && sample_uniform(0.0f, 1.0f) < env->phantom_braking_trigger_prob) { agent->phantom_braking_counter = env->phantom_braking_duration - 1; phantom_braking_active = 1; } @@ -5020,7 +5038,7 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { float acceleration = 0.0f; float steering = 0.0f; - if (env->action_type == 0) { // discrete + if (env->action_type == DISCRETE) { // Interpret action as a single integer: a = accel_idx * num_steer + steer_idx int *action_array = (int *) env->actions; int num_steer = sizeof(STEERING_VALUES) / sizeof(STEERING_VALUES[0]); @@ -5029,7 +5047,7 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { int steering_index = action_val % num_steer; acceleration = ACCELERATION_VALUES[acceleration_index]; steering = STEERING_VALUES[steering_index]; - } else if (env->action_type == 1) { // continuous + } else if (env->action_type == CONTINUOUS) { float (*action_array_f)[2] = (float (*)[2]) env->actions; acceleration = action_array_f[action_idx][0]; steering = action_array_f[action_idx][1]; @@ -5096,13 +5114,20 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { agent->jerk_lat = (new_a_lat - agent->accel_lat) / env->dt; agent->accel_long = new_a_long; agent->accel_lat = new_a_lat; - } else { - // JERK dynamics model + } else if (env->dynamics_model == JERK) { // Extract jerk action components float j_long, j_lat; - if (env->action_type == 1) { // continuous + if (env->action_type == DISCRETE) { + // Interpret action as a single integer: a = long_idx * num_lat + lat_idx + int *action_array = (int *) env->actions; + int num_lat = sizeof(JERK_LAT) / sizeof(JERK_LAT[0]); + int action_val = action_array[action_idx]; + int j_long_idx = action_val / num_lat; + int j_lat_idx = action_val % num_lat; + j_long = JERK_LONG[j_long_idx]; + j_lat = JERK_LAT[j_lat_idx]; + } else if (env->action_type == CONTINUOUS) { float (*action_array_f)[2] = (float (*)[2]) env->actions; - // Asymmetric scaling for longitudinal jerk to match discrete action space // Discrete: JERK_LONG = [-15, -4, 0, 4] (more braking than acceleration) float j_long_action = action_array_f[action_idx][0]; // [-1, 1] @@ -5111,18 +5136,8 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { } else { j_long = j_long_action * JERK_LONG[3]; // Positive: [0, 1] → [0, 4] (acceleration) } - // Symmetric scaling for lateral jerk j_lat = action_array_f[action_idx][1] * JERK_LAT[2]; - } else if (env->action_type == 0) { // discrete - // Interpret action as a single integer: a = long_idx * num_lat + lat_idx - int *action_array = (int *) env->actions; - int num_lat = sizeof(JERK_LAT) / sizeof(JERK_LAT[0]); - int action_val = action_array[action_idx]; - int j_long_idx = action_val / num_lat; - int j_lat_idx = action_val % num_lat; - j_long = JERK_LONG[j_long_idx]; - j_lat = JERK_LAT[j_lat_idx]; } if (phantom_braking_active) { From 74fdb884a382dd88fa6f9cd410fc658fc79039b8 Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Mon, 29 Jun 2026 13:52:48 +0200 Subject: [PATCH 2/7] Update REWARD_COEF_COMFORT bounds for log scaling adjustment --- pufferlib/ocean/drive/drive.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pufferlib/ocean/drive/drive.h b/pufferlib/ocean/drive/drive.h index 88e28ff5c0..d7589843df 100644 --- a/pufferlib/ocean/drive/drive.h +++ b/pufferlib/ocean/drive/drive.h @@ -475,7 +475,7 @@ static const RewardBound REWARD_BOUNDS[NUM_REWARD_COEFS] = { {0.0f, 20.0f, 0}, // REWARD_COEF_GOAL_SPEED δ_goal-speed ~ U(0, 20) {0.0f, 3.0f, 0}, // REWARD_COEF_COLLISION α_collision ~ U(0, 3) {0.0f, 3.0f, 0}, // REWARD_COEF_OFFROAD α_boundary ~ U(0, 3) - {0.0f, 0.1f, 1}, // REWARD_COEF_COMFORT α_comfort ~ logU(0, 0.1) + {1e-5f, 0.1f, 1}, // REWARD_COEF_COMFORT α_comfort ~ logU(1e-5f, 0.1) {2.5e-4f, 2.5e-2f, 1}, // REWARD_COEF_LANE_ALIGN α_l-align ~ logU(2.5e-4, 2.5e-2) {0.0f, 1.0f, 0}, // REWARD_COEF_VEL_ALIGN α_vel-align ~ U(0, 1) {2.5e-4f, 7.5e-3f, 1}, // REWARD_COEF_LANE_CENTER α_l-center ~ logU(2.5e-4, 7.5e-3) From 0ac730a39efe55f371b6265d1d9a52dc72fbc338 Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Thu, 30 Jul 2026 09:20:51 +0200 Subject: [PATCH 3/7] Refactor dynamics model handling for discrete action type in move_dynamics function --- pufferlib/ocean/drive/drive.h | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pufferlib/ocean/drive/drive.h b/pufferlib/ocean/drive/drive.h index dc368f48f3..60b3d49bfa 100644 --- a/pufferlib/ocean/drive/drive.h +++ b/pufferlib/ocean/drive/drive.h @@ -5075,11 +5075,19 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { agent->jerk_lat = (new_a_lat - agent->accel_lat) / env->dt; agent->accel_long = new_a_long; agent->accel_lat = new_a_lat; - } else { - // DYNAMICS_MODEL_JERK dynamics model + } else if (env->dynamics_model == DYNAMICS_MODEL_JERK) { // Extract jerk action components float j_long, j_lat; - if (env->action_type == ACTION_TYPE_CONTINUOUS) { + if (env->action_type == ACTION_TYPE_DISCRETE) { + // Interpret action as a single integer: a = long_idx * num_lat + lat_idx + int *action_array = (int *) env->actions; + int num_lat = sizeof(JERK_LAT) / sizeof(JERK_LAT[0]); + int action_val = action_array[action_idx]; + int j_long_idx = action_val / num_lat; + int j_lat_idx = action_val % num_lat; + j_long = JERK_LONG[j_long_idx]; + j_lat = JERK_LAT[j_lat_idx]; + } else if (env->action_type == ACTION_TYPE_CONTINUOUS) { float (*action_array_f)[2] = (float (*)[2]) env->actions; // Asymmetric scaling for longitudinal jerk to match discrete action space // Discrete: JERK_LONG = [-15, -4, 0, 4] (more braking than acceleration) @@ -5091,15 +5099,6 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { } // Symmetric scaling for lateral jerk j_lat = action_array_f[action_idx][1] * JERK_LAT[2]; - } else if (env->action_type == ACTION_TYPE_DISCRETE) { - // Interpret action as a single integer: a = long_idx * num_lat + lat_idx - int *action_array = (int *) env->actions; - int num_lat = sizeof(JERK_LAT) / sizeof(JERK_LAT[0]); - int action_val = action_array[action_idx]; - int j_long_idx = action_val / num_lat; - int j_lat_idx = action_val % num_lat; - j_long = JERK_LONG[j_long_idx]; - j_lat = JERK_LAT[j_lat_idx]; } if (phantom_braking_active) { From 361f725d55d1cb1802b0f26e430528b0bbf0a68a Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Thu, 30 Jul 2026 09:35:55 +0200 Subject: [PATCH 4/7] Replace random_uniform with sample_uniform for goal spacing calculations --- pufferlib/ocean/drive/drive.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pufferlib/ocean/drive/drive.h b/pufferlib/ocean/drive/drive.h index 60b3d49bfa..67efcd3c10 100644 --- a/pufferlib/ocean/drive/drive.h +++ b/pufferlib/ocean/drive/drive.h @@ -1861,7 +1861,7 @@ static bool generate_new_goals_from_route(Drive *env, Agent *agent) { // Sample a spacing per goal, then walk the route placing goals at those forward distances. float goal_spacings_meters[MAX_GOALS]; for (int goal_idx = 0; goal_idx < env->num_goals; goal_idx++) { - goal_spacings_meters[goal_idx] = random_uniform(env->min_goal_spacing, env->max_goal_spacing); + goal_spacings_meters[goal_idx] = sample_uniform(env->min_goal_spacing, env->max_goal_spacing); } float goal_x[MAX_GOALS], goal_y[MAX_GOALS], goal_z[MAX_GOALS]; @@ -1979,7 +1979,7 @@ static bool generate_new_goals_from_map(Drive *env, Agent *agent) { int requested_goal_count = 1 + rand() % env->num_goals; float goal_spacings_meters[MAX_GOALS]; for (int goal_idx = 0; goal_idx < requested_goal_count; goal_idx++) { - goal_spacings_meters[goal_idx] = random_uniform(env->min_goal_spacing, env->max_goal_spacing); + goal_spacings_meters[goal_idx] = sample_uniform(env->min_goal_spacing, env->max_goal_spacing); } float goal_x[MAX_GOALS], goal_y[MAX_GOALS], goal_z[MAX_GOALS]; int goal_lane[MAX_GOALS]; @@ -2047,7 +2047,7 @@ static int roll_goals(Drive *env, Agent *agent) { } // Walk one spacing forward to find the appended goal before touching the window. - float spacing_meters = random_uniform(env->min_goal_spacing, env->max_goal_spacing); + float spacing_meters = sample_uniform(env->min_goal_spacing, env->max_goal_spacing); float next_x, next_y, next_z, next_s_on_lane; int next_lane_idx, next_cursor_idx; // next_cursor_idx unused: single-step append, no chaining if (!route_point_at_distance( From 2f3f0947d9398c0a53380ccfe90029511a5c2e17 Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Thu, 30 Jul 2026 11:56:12 +0200 Subject: [PATCH 5/7] Update golden JSON files with revised environment metrics and reward components --- .../data/drive_rollout_golden.json | 10 +-- .../smoke_tests/data/drive_smoke_golden.json | 66 +++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/smoke_tests/data/drive_rollout_golden.json b/tests/smoke_tests/data/drive_rollout_golden.json index 9407d8022c..e9c428469d 100644 --- a/tests/smoke_tests/data/drive_rollout_golden.json +++ b/tests/smoke_tests/data/drive_rollout_golden.json @@ -6,7 +6,7 @@ "comfort_violation_count": 0.6431659708420435, "dnf_rate": 0.5284722222222222, "episode_length": 13.966666666666667, - "episode_return": -1.1519524534543355, + "episode_return": -0.7864702496263716, "lane_center_rate": 0.6887658986780378, "n": 16.0, "num_goals_reached": 0.0125, @@ -14,14 +14,14 @@ "red_light_violation_rate": 0.029166666666666667, "reward_components/ade": 0.0, "reward_components/collision": -0.049521034707625707, - "reward_components/comfort": -0.4500008859568172, + "reward_components/comfort": -0.10717053327502477, "reward_components/goal": 0.0006944444444444445, - "reward_components/lane_align": -0.026875105024211938, - "reward_components/lane_center": -0.00494278745027259, + "reward_components/lane_align": -0.011314233301931785, + "reward_components/lane_center": -0.00269687048987382, "reward_components/offroad": -0.5939088342918291, "reward_components/overspeed": 0.0, "reward_components/red_light": -0.016583363693724904, - "reward_components/reverse": -0.010919584835776024, + "reward_components/reverse": -0.006074528648362805, "reward_components/timestep": -8.832291556852094e-05, "reward_components/velocity": 0.00019301686790842924, "score": 0.0, diff --git a/tests/smoke_tests/data/drive_smoke_golden.json b/tests/smoke_tests/data/drive_smoke_golden.json index 0681f113c0..99137c4e72 100644 --- a/tests/smoke_tests/data/drive_smoke_golden.json +++ b/tests/smoke_tests/data/drive_smoke_golden.json @@ -1,48 +1,48 @@ { "env": { - "avg_distance_per_infraction": 12.509860291200525, - "avg_speed_per_agent": 1.3230969975976383, - "collision_rate": 0.029411764705882353, - "comfort_violation_count": 0.6215264201164246, - "dnf_rate": 0.5551470588235294, - "episode_length": 13.941176470588236, - "episode_return": -1.0949732065200806, - "lane_center_rate": 0.6929002439274508, + "avg_distance_per_infraction": 12.7494588692983, + "avg_speed_per_agent": 1.3614558180173237, + "collision_rate": 0.03125, + "comfort_violation_count": 0.6339364565081067, + "dnf_rate": 0.5555555555555556, + "episode_length": 13.833333333333334, + "episode_return": -0.7382989443010755, + "lane_center_rate": 0.6906335850556692, "n": 16.0, "num_goals_reached": 0.0, "obs/max": 49.0, - "obs/mean": 0.24835189001169056, - "obs/min": -1.737496261484921, + "obs/mean": 0.2485432226676494, + "obs/min": -1.8799336636438966, "offroad_rate": 0.375, - "red_light_violation_rate": 0.04044117647058824, + "red_light_violation_rate": 0.03819444444444445, "reward_components/ade": 0.0, - "reward_components/collision": -0.04708075435722575, - "reward_components/comfort": -0.434675477883395, + "reward_components/collision": -0.0461330728398429, + "reward_components/comfort": -0.08267618824417393, "reward_components/goal": 0.0, - "reward_components/lane_align": -0.02614135141758358, - "reward_components/lane_center": -0.004757273558746366, - "reward_components/offroad": -0.5461728187168345, + "reward_components/lane_align": -0.011884897611000471, + "reward_components/lane_center": -0.002693182950477219, + "reward_components/offroad": -0.5681077059772279, "reward_components/overspeed": 0.0, - "reward_components/red_light": -0.02611259094384663, - "reward_components/reverse": -0.010194735135883093, - "reward_components/timestep": -8.70496330796467e-05, - "reward_components/velocity": 0.00024882864411612153, + "reward_components/red_light": -0.021054884910376534, + "reward_components/reverse": -0.005850944318808615, + "reward_components/timestep": -8.739583184554552e-05, + "reward_components/velocity": 0.00018931993609941046, "score": 0.0, - "velocity_progress_sum": 0.02285407153506051 + "velocity_progress_sum": 0.01941619658221801 }, "losses": { - "approx_kl": 0.005990775767713785, - "clipfrac": 0.07198660714285714, - "ema_max": 2.792989134788513, - "entropy": 2.4691788809640065, - "explained_variance": -0.019130587577819824, - "filter_threshold": 0.027929891347885132, - "filtered_fraction": 0.10776361529548084, - "kept_fraction": 0.8922363847045192, - "masked_fraction": 0.1572265625, - "old_approx_kl": -0.010950545647314616, - "policy_loss": 0.000791141896375588, - "value_loss": 0.1787161481167589 + "approx_kl": 0.002686771214939654, + "clipfrac": 0.01957417589922746, + "ema_max": 2.9573644399642944, + "entropy": 2.4768027464548745, + "explained_variance": -0.005404114723205566, + "filter_threshold": 0.029573644399642943, + "filtered_fraction": 0.22146507666098803, + "kept_fraction": 0.778534923339012, + "masked_fraction": 0.14013671875, + "old_approx_kl": 0.004910027297834556, + "policy_loss": -0.0017952998168766499, + "value_loss": 0.36018707354863483 }, "meta": { "bptt_horizon": 64, From d975ae08e39d19bad5712a940439534e00b46b0e Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Tue, 4 Aug 2026 10:23:03 +0200 Subject: [PATCH 6/7] Refactor sampling functions to use consistent RNG state handling --- pufferlib/ocean/drive/drive.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pufferlib/ocean/drive/drive.h b/pufferlib/ocean/drive/drive.h index 99404e486d..af66c913bb 100644 --- a/pufferlib/ocean/drive/drive.h +++ b/pufferlib/ocean/drive/drive.h @@ -386,8 +386,8 @@ static float sample_uniform(unsigned int *rng_state, float min_val, float max_va return min_val + ((float) rand_r(rng_state) / (float) RAND_MAX) * (max_val - min_val); } -static float sample_log_uniform(float min_val, float max_val) { - return expf(sample_uniform(logf(min_val), logf(max_val))); +static float sample_log_uniform(unsigned int *rng_state, float min_val, float max_val) { + return expf(sample_uniform(rng_state, logf(min_val), logf(max_val))); } static float sample_mixed_uniform(unsigned int *rng_state, float a) { @@ -395,14 +395,14 @@ static float sample_mixed_uniform(unsigned int *rng_state, float a) { if ((float) rand_r(rng_state) / (float) RAND_MAX < 0.5f) { return sample_uniform(rng_state, 1.0f / a, 1.0f); } - return random_uniform(rng_state, 1.0f, a); + return sample_uniform(rng_state, 1.0f, a); } static void begin_episode_rng(Drive *env) { if (env->use_exact_episode_seed) { env->episode_seed = env->seed_stream_state; } else { - return sample_uniform(1.0f, a); + env->episode_seed = (unsigned int) rand_r(&env->seed_stream_state); } env->rng_state = env->episode_seed; } @@ -2060,7 +2060,7 @@ static void generate_reward_coefs(Drive *env, Agent *agent) { for (int i = 0; i < (int) (sizeof(random_coefs) / sizeof(random_coefs[0])); i++) { int c = random_coefs[i]; agent->reward_coefs[c] = REWARD_BOUNDS[c].log_scale - ? sample_log_uniform(REWARD_BOUNDS[c].min_val, REWARD_BOUNDS[c].max_val) + ? sample_log_uniform(&env->rng_state, REWARD_BOUNDS[c].min_val, REWARD_BOUNDS[c].max_val) : sample_uniform(&env->rng_state, REWARD_BOUNDS[c].min_val, REWARD_BOUNDS[c].max_val); } agent->reward_coefs[REWARD_COEF_VELOCITY] = 2.5e-3f; From dcd1b319f870ee739164062b3c230fe8309dfcb0 Mon Sep 17 00:00:00 2001 From: Valentin Charraut Date: Tue, 4 Aug 2026 12:10:36 +0200 Subject: [PATCH 7/7] Update golden JSON files with revised environment metrics and reward components --- .../data/drive_rollout_golden.json | 36 +++++----- .../smoke_tests/data/drive_smoke_golden.json | 70 +++++++++---------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/tests/smoke_tests/data/drive_rollout_golden.json b/tests/smoke_tests/data/drive_rollout_golden.json index 9f03badf18..8e96952f46 100644 --- a/tests/smoke_tests/data/drive_rollout_golden.json +++ b/tests/smoke_tests/data/drive_rollout_golden.json @@ -1,29 +1,29 @@ { "env": { - "avg_distance_per_infraction": 12.350080193413628, - "avg_speed_per_agent": 1.3601282782024808, - "collision_rate": 0.029861111111111113, - "comfort_violation_count": 0.6431659708420435, - "dnf_rate": 0.5284722222222222, - "episode_length": 13.966666666666667, - "episode_return": -0.7864702496263716, - "lane_center_rate": 0.6887658986780378, + "avg_distance_per_infraction": 12.06708054339632, + "avg_speed_per_agent": 1.360816847770772, + "collision_rate": 0.023936170212765957, + "comfort_violation_count": 0.6436155717423622, + "dnf_rate": 0.535904255319149, + "episode_length": 13.48936170212766, + "episode_return": -0.7790729422518547, + "lane_center_rate": 0.6959362321711601, "n": 16.0, "num_goals_reached": 0.00997340425531915, "offroad_rate": 0.4115691489361702, "red_light_violation_rate": 0.025930851063829786, "reward_components/ade": 0.0, - "reward_components/collision": -0.049521034707625707, - "reward_components/comfort": -0.10717053327502477, - "reward_components/goal": 0.0006944444444444445, - "reward_components/lane_align": -0.011314233301931785, - "reward_components/lane_center": -0.00269687048987382, - "reward_components/offroad": -0.5939088342918291, + "reward_components/collision": -0.042797685620632575, + "reward_components/comfort": -0.09395423511716913, + "reward_components/goal": 0.003324468085106383, + "reward_components/lane_align": -0.01126101781227725, + "reward_components/lane_center": -0.0026710387282172575, + "reward_components/offroad": -0.6115261237037942, "reward_components/overspeed": 0.0, - "reward_components/red_light": -0.016583363693724904, - "reward_components/reverse": -0.006074528648362805, - "reward_components/timestep": -8.832291556852094e-05, - "reward_components/velocity": 0.00019301686790842924, + "reward_components/red_light": -0.014523565179688181, + "reward_components/reverse": -0.005763038997835619, + "reward_components/timestep": -8.58909574007873e-05, + "reward_components/velocity": 0.00018516867541930356, "score": 0.0, "velocity_progress_sum": 0.018232046272308428 }, diff --git a/tests/smoke_tests/data/drive_smoke_golden.json b/tests/smoke_tests/data/drive_smoke_golden.json index 419e784884..9dd4f2b4bf 100644 --- a/tests/smoke_tests/data/drive_smoke_golden.json +++ b/tests/smoke_tests/data/drive_smoke_golden.json @@ -1,48 +1,48 @@ { "env": { - "avg_distance_per_infraction": 12.7494588692983, - "avg_speed_per_agent": 1.3614558180173237, - "collision_rate": 0.03125, - "comfort_violation_count": 0.6339364565081067, - "dnf_rate": 0.5555555555555556, - "episode_length": 13.833333333333334, - "episode_return": -0.7382989443010755, - "lane_center_rate": 0.6906335850556692, + "avg_distance_per_infraction": 12.15254513422648, + "avg_speed_per_agent": 1.369708173804813, + "collision_rate": 0.020833333333333332, + "comfort_violation_count": 0.6313128405147128, + "dnf_rate": 0.53125, + "episode_length": 13.38888888888889, + "episode_return": -0.747443911102083, + "lane_center_rate": 0.6955425375037723, "n": 16.0, - "num_goals_reached": 0.019736842105263157, + "num_goals_reached": 0.020833333333333332, "obs/max": 49.0, - "obs/mean": 0.2485432226676494, - "obs/min": -1.8799336636438966, - "offroad_rate": 0.375, - "red_light_violation_rate": 0.03819444444444445, + "obs/mean": 0.24867265624925494, + "obs/min": -1.6757474672049284, + "offroad_rate": 0.4131944444444444, + "red_light_violation_rate": 0.024305555555555556, "reward_components/ade": 0.0, - "reward_components/collision": -0.0461330728398429, - "reward_components/comfort": -0.08267618824417393, + "reward_components/collision": -0.034726881318622164, + "reward_components/comfort": -0.09129529053138362, "reward_components/goal": 0.0, - "reward_components/lane_align": -0.011884897611000471, - "reward_components/lane_center": -0.002693182950477219, - "reward_components/offroad": -0.5681077059772279, + "reward_components/lane_align": -0.010500837311459085, + "reward_components/lane_center": -0.002869318226455814, + "reward_components/offroad": -0.593594147099389, "reward_components/overspeed": 0.0, - "reward_components/red_light": -0.021054884910376534, - "reward_components/reverse": -0.005850944318808615, - "reward_components/timestep": -8.739583184554552e-05, - "reward_components/velocity": 0.00018931993609941046, + "reward_components/red_light": -0.009579533297154639, + "reward_components/reverse": -0.0050971122044656016, + "reward_components/timestep": -8.596354139929947e-05, + "reward_components/velocity": 0.0003051760419718145, "score": 0.0, - "velocity_progress_sum": 0.01941619658221801 + "velocity_progress_sum": 0.028703066365172464 }, "losses": { - "approx_kl": 0.002686771214939654, - "clipfrac": 0.01957417589922746, - "ema_max": 2.9573644399642944, - "entropy": 2.4768027464548745, - "explained_variance": -0.005404114723205566, - "filter_threshold": 0.029573644399642943, - "filtered_fraction": 0.22146507666098803, - "kept_fraction": 0.778534923339012, - "masked_fraction": 0.14013671875, - "old_approx_kl": 0.004910027297834556, - "policy_loss": -0.0017952998168766499, - "value_loss": 0.36018707354863483 + "approx_kl": 0.005358540724652509, + "clipfrac": 0.05141958842674891, + "ema_max": 2.412422329187393, + "entropy": 2.458675662676493, + "explained_variance": 0.009576380252838135, + "filter_threshold": 0.02412422329187393, + "filtered_fraction": 0.15625, + "kept_fraction": 0.84375, + "masked_fraction": 0.140625, + "old_approx_kl": 0.004483325717349847, + "policy_loss": -0.0015975118149071932, + "value_loss": 0.18829950441916785 }, "meta": { "bptt_horizon": 64,