diff --git a/benchmarks/piper-control-profile-v01/README.md b/benchmarks/piper-control-profile-v01/README.md new file mode 100644 index 0000000..7d2d390 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/README.md @@ -0,0 +1,81 @@ +# PiPER contact-path profiling v0.1 + +This development experiment reduces allocations inside `PiperTaskEnv._contacts`: +one private scratch force array per scan, cleared before each native query, and +membership sets computed once per scan. Every 1 ms physics, force, safety and +outcome check remains active. No contact result is cached between calls. + +Four uninstrumented ABBA runs on the same previously selected synthetic command +tape measured **15.70% lower pooled mean environment-step wall compute** and +**12.60% lower process CPU compute**. Both clocks improved in both matched pairs. +This is two pairs on one shared host, not a general speed estimate. Timing +excludes policy inference, setup, serialization, sensors and physical actuators. + +| Run | Source | Mean wall, ms | Mean CPU, ms | Wall p95, ms | +| --- | --- | ---: | ---: | ---: | +| A1 | Baseline | 1.468496 | 1.412751 | 2.542359 | +| B1 | Candidate | 1.232893 | 1.220514 | 1.886604 | +| B2 | Candidate | 1.305327 | 1.293750 | 1.986173 | +| A2 | Baseline | 1.542341 | 1.463971 | 2.606366 | + +One baseline step exceeded its nominal 20 ms budget. These are unpaced simulator +compute windows, not hardware deadlines. The diagnostic profiler identified +contact scans as the largest recorded self-time hotspot; inclusive profiler +totals overlap, and profiler timings are not the performance comparison. + +## Equivalence and retained failure + +Ten separate diagnostic processes cover the original tape, open-jaw, hold, +feedback telemetry without a fault, and telemetry with a scheduled jaw-opening +pulse. All 14 scheduled processes completed. Each diagnostic retained an initial +snapshot and 20,000 native ticks: 200,010 snapshots total. Independent streaming +readback exactly matched all five baseline/candidate pairs, including 19 numeric +array fields, raw and active contacts, force thresholds, cumulative contact +counts, boundary observations and independently recomputed outcome state. + +The original comparison reader rejected three dense pairs because their expanded +JSON was about 74.8 MB, over its 64 MiB limit. Its `all_passed: false` receipt is +preserved. A separate bounded streaming audit checked the **same recorded files**; +there were no simulation or timing retries. Its SHA-256 is +`1a4f4a1093961b59c03a3a244d123afed35f5a5f77c98d1467a393a20c2d6fd4`. + +Original and telemetry cases passed the unchanged scorer; open-jaw and hold +failed. The pulse did not cause a task failure, so this is not recovery evidence. +Recorded equivalence does not establish real-world accuracy. The prior held-out +learned result remains **5/12**, versus **12/12** for the authored teacher. + +## Inspect and reproduce + +The [evidence dataset](https://huggingface.co/datasets/Dvidia/piper-control-profile-v01) +contains source snapshots, protocols, every recorded attempt, original and +supplemental reports, and a guarded archive verifier/reconstructor. The archive +uses content-addressed blobs to avoid storing identical evidence twice. No robot +meshes, credentials, footage or learned weights are included. + +Use Python 3.12 and the repository's `arm` optional dependencies. Provision pinned +PiPER assets using the existing robot simulator instructions. Baseline source is +`7d9faaf4962f0afd36425399c190457cce86c2b6`; the candidate is this branch. After +verifying and reconstructing the archive, use the actual evidence paths in: + +```sh +python benchmarks/piper-control-profile-v01/compare.py \ + --baseline-root ../baseline --candidate-root . \ + --python .venv/bin/python --episode evidence/comparison/preserved-episode.json.gz \ + --assets robot-assets --output runs/new-control-comparison +``` + +`compare.py` is the frozen experimental harness, including its original 64 MiB +reader limit. A repeat may therefore retain a reader refusal after all native +attempts finish. Run `readback_audit.py --help` for the separate streaming audit; +use a new audit output path and preserve both receipts. Neither tool overwrites +prior evidence. The archive also retains the exact verifier used in this study. + +For the diagnostic cProfile run, use `profile.py --help` with `PYTHONPATH` pointing +to the baseline source. It requires the exact published prior model and selected +episode digests. Raw `.pstats` files stay local because they contain machine +paths; portable profiles and complete JSON function tables are included. + +This is an experimental source patch, not a replacement wheel or production +rollout. The published dev3 wheel remains unchanged. Do not reuse an installed +skill's old runtime receipt after changing its source. Broader paired scenarios +and a fresh package qualification remain necessary before promoting the patch. diff --git a/benchmarks/piper-control-profile-v01/compare.py b/benchmarks/piper-control-profile-v01/compare.py new file mode 100644 index 0000000..daf1753 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/compare.py @@ -0,0 +1,587 @@ +"""Controlled, offline PiPER source comparison; never a hardware latency claim. + +The fixed ABBA timing runs use the uninstrumented environment and a preserved +1,000-command tape. Separate diagnostic processes record every post-scoring +1 ms tick. Their substantial copying/contact/FK overhead is excluded from the +timing comparison. Every scheduled attempt is retained, including failures. +""" +from __future__ import annotations + +import argparse +from dataclasses import asdict +import gzip +from hashlib import sha256 +import io +import json +import math +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +import traceback + +SCHEMA = 'dvidia.piper-control-comparison/v1' +EPISODE_SHA = '7954c32ae4bc689f8947d8bc7786e028bf18dc1a02a100a7a38bdc779bf8193d' +ALLOWED_SOURCE_DIFFERENCES = ('piper_task.py',) +SOURCE_FILES = ('__init__.py', 'piper_task.py', 'reference_task.py', 'reference_sim.py', + 'robot_profiles.py', 'robot_profiles.lock.json', 'reference_assets.lock.json', + 'env.py', 'piper_feedback_env.py', 'piper_imitation_score.py') +CASES = ('original', 'open-jaw', 'hold', 'feedback-no-pulse', 'feedback-pulse') +PULSE = {'start_s': 8.0, 'end_s': 8.12, 'minimum_opening_m': .035} +NATIVE_ARRAYS = ('qpos', 'qvel', 'qacc', 'qacc_warmstart', 'ctrl', 'act', + 'actuator_force', 'qfrc_applied', 'xfrc_applied', 'qfrc_bias', + 'qfrc_constraint', 'xpos', 'xquat', 'site_xpos', 'site_xmat') +SCORER_FIELDS = ('elapsed', 'steps', 'dwell', 'lift_dwell', 'max_lift', 'max_pad_force', + 'critical_contact_ticks', 'lift_seen', 'grasp_seen', 'carry_complete', + 'retention_lost', 'carry_contact_loss', 'max_carry_contact_loss', + 'carry_contact_loss_events', 'first_success_time', 'reason', 'done') +EPISODE_FIELDS = ('config', 'seed', 'valid', 'success', 'reason', 'error', + 'simulated_seconds', 'initial_object_position_m', 'compiled_model_sha256', + 'trajectory', 'actions', 'metrics') +SCOPE = ('One previously checked synthetic PiPER scenario, preserved commands and privileged ' + 'native simulator state. Allocation-only source comparison, not new skill training, ' + 'held-out success, hardware latency, calibrated contact or a real-time guarantee.') +ADOPTION_RULE = {'required_exact_equivalence': True, + 'required_lower_candidate_mean_each_pair': ['step_wall', 'step_cpu'], + 'minimum_pooled_mean_reduction_percent_each_clock': 5., + 'on_failure': 'Do not promote; retain as negative or inconclusive. No post-result tuning.'} + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def identity(raw): + return {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} + + +def write(path, value): + with Path(path).open('xb') as stream: + stream.write(json.dumps(value, sort_keys=True, indent=2, allow_nan=False).encode()+b'\n') + + +def write_gzip(path, value): + with Path(path).open('xb') as raw: + with gzip.GzipFile(filename='', fileobj=raw, mode='wb', mtime=0) as stream: + stream.write(canonical(value)) + + +def decode(raw): + def pairs(rows): + result = {} + for key, value in rows: + if key in result: + raise ValueError('Duplicate JSON field') + result[key] = value + return result + def nonfinite(token): + raise ValueError('Nonfinite JSON token: '+token) + result = json.loads(raw, object_pairs_hook=pairs, parse_constant=nonfinite) + canonical(result) + return result + + +def read_gzip(path, maximum=64*1024*1024): + raw = Path(path).read_bytes() + if len(raw) > maximum: + raise ValueError('Compressed document exceeds bound') + with gzip.GzipFile(fileobj=io.BytesIO(raw)) as stream: + expanded = stream.read(maximum+1) + if len(expanded) > maximum: + raise ValueError('Expanded document exceeds bound') + return decode(expanded) + + +def inventory(root): + folder = Path(root)/'src'/'dvidia_training' + rows = {} + for name in SOURCE_FILES: + path = folder/name + if path.is_symlink() or not path.is_file(): + raise ValueError('Source must be a regular file: '+name) + rows[name] = identity(path.read_bytes()) + return rows + + +def check_inventory(root, expected): + if inventory(root) != expected: + raise ValueError('Frozen source changed during the comparison') + + +def source_differences(left, right): + differences = sorted(name for name in set(left) | set(right) if left.get(name) != right.get(name)) + if set(differences) - set(ALLOWED_SOURCE_DIFFERENCES): + raise ValueError('Source differs outside the declared allocation-only file: '+', '.join(differences)) + if not differences: + raise ValueError('Baseline and candidate sources are identical') + return differences + + +def schedule(pairs=2): + if type(pairs) is not int or not 2 <= pairs <= 8 or pairs % 2: + raise ValueError('Use an even number of paired runs in [2,8]') + return [variant for pair in range(pairs) + for variant in (('baseline', 'candidate') if pair % 2 == 0 else ('candidate', 'baseline'))] + + +def summary(values): + values = sorted(values) + if not values: + return {'count': 0} + def percentile(q): + rank = (len(values)-1)*q + low, high = math.floor(rank), math.ceil(rank) + return values[low]+(values[high]-values[low])*(rank-low) + return {'count': len(values), 'sum_s': sum(values), 'mean_s': sum(values)/len(values), + 'p50_s': percentile(.5), 'p95_s': percentile(.95), 'p99_s': percentile(.99), + 'max_s': values[-1], 'over_nominal_20ms': sum(item > .02 for item in values)} + + +def load_tape(path): + raw = Path(path).read_bytes() + if len(raw) > 8*1024*1024 or sha256(raw).hexdigest() != EPISODE_SHA: + raise ValueError('Only the exact preserved installed-smoke episode is accepted') + episode = read_gzip(path, 32*1024*1024) + if (episode.get('seed') != 261517 or len(episode.get('actions', [])) != 1000 + or len(episode.get('trajectory', [])) != 1001 + or episode.get('config', {}).get('timestep') != .001 + or episode['config'].get('control_dt') != .02 + or episode['config'].get('timeout_s') != 20.): + raise ValueError('Preserved tape dimensions, seed or cadence differ') + commands = [] + for index, action in enumerate(episode['actions']): + if (set(action) != {'t_seconds', 'joint_targets_rad', 'gripper_position_m'} + or type(action['joint_targets_rad']) is not list + or len(action['joint_targets_rad']) != 6 + or any(type(value) not in (int, float) or not math.isfinite(value) + for value in action['joint_targets_rad']) + or type(action['gripper_position_m']) not in (int, float) + or not 0 <= action['gripper_position_m'] <= .035 + or not math.isclose(action['t_seconds'], index*.02, rel_tol=0, abs_tol=1e-8)): + raise ValueError('Malformed preserved command tape') + commands.append({key: action[key] for key in ('joint_targets_rad', 'gripper_position_m')}) + return episode, commands, identity(raw) + + +def exact_document(left, right, fields=None): + keys = sorted(set(left) | set(right)) if fields is None else fields + different = [key for key in keys if key not in left or key not in right + or canonical(left[key]) != canonical(right[key])] + return {'exact_match': not different, 'different_fields': different, 'fields_compared': list(keys)} + + +def compare_arrays(left, right): + """Bitwise numeric equality, including signed zero; never allow pickle.""" + import numpy as np + differences = [] + with np.load(left, allow_pickle=False) as a, np.load(right, allow_pickle=False) as b: + for key in sorted(set(a.files) | set(b.files)): + if key not in a.files or key not in b.files: + differences.append({'field': key, 'reason': 'missing_array'}) + continue + x, y = a[key], b[key] + if x.dtype.kind not in 'iufb' or y.dtype.kind not in 'iufb': + raise ValueError('Dense evidence must contain numeric arrays only') + if x.shape != y.shape or x.dtype != y.dtype: + differences.append({'field': key, 'reason': 'shape_or_dtype'}) + elif x.tobytes() != y.tobytes(): + # Byte comparison catches signed-zero differences too. + byte_diff = np.flatnonzero(x.view(np.uint8).reshape(-1) != y.view(np.uint8).reshape(-1)) + row_bytes = (x[0].nbytes if x.ndim and len(x) else x.dtype.itemsize) + differences.append({'field': key, 'reason': 'bitwise_mismatch', + 'first_row': int(byte_diff[0]//max(row_bytes, 1))}) + return {'exact_match': not differences, 'differences': differences} + + +def _diagnostic_class(base, np, mj): + class Dense(base): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._dense_arrays = {} + self._dense_rows = [] + + def _outcome_tick(self, **kwargs): + super()._outcome_tick(**kwargs) + self.capture(kwargs) + + def capture(self, scorer_inputs=None): + arrays = {name: np.asarray(getattr(self.data, name)).copy() for name in NATIVE_ARRAYS} + integration = np.empty(mj.mj_stateSize(self.model, mj.mjtState.mjSTATE_INTEGRATION)) + mj.mj_getState(self.model, self.data, integration, mj.mjtState.mjSTATE_INTEGRATION) + arrays.update(integration_state=integration, + native_time=np.asarray(self.data.time, dtype=np.float64), + command_targets=np.asarray(self.command_targets).copy(), + command_gripper=np.asarray(self.command_gripper, dtype=np.float64)) + for name, value in arrays.items(): + self._dense_arrays.setdefault(name, []).append(value) + counts, forces, supported, critical, pairs = self._contacts() + raw_contacts = [] + for index, contact in enumerate(self.data.contact[:self.data.ncon]): + force = np.zeros(6) + mj.mj_contactForce(self.model, self.data, index, force) + raw_contacts.append({'geom_ids': [int(contact.geom1), int(contact.geom2)], + 'dim': int(contact.dim), 'efc_address': int(contact.efc_address), + 'distance_m': float(contact.dist), 'position_m': contact.pos.tolist(), + 'frame': contact.frame.tolist(), 'friction': contact.friction.tolist(), + 'force_native': force.tolist()}) + self._dense_rows.append({ + 'native_tick': len(self._dense_rows), + 'contacts': {'counts': counts, 'forces': forces, 'table': supported, + 'critical': critical, 'active_pairs': pairs, 'raw': raw_contacts}, + 'scorer_inputs': ({name: value.item() if isinstance(value, np.generic) else value + for name, value in scorer_inputs.items()} + if scorer_inputs is not None else None), + 'scorer': {name: getattr(self, name) for name in SCORER_FIELDS}, + 'contact_pair_native_ticks': dict(self.contact_pair_native_ticks), + 'warnings': list(self.warnings), + 'native_warnings': [{'number': int(row.number), 'lastinfo': int(row.lastinfo)} + for row in self.data.warning]}) + + def save_dense(self, folder): + # Each row was copied while current; no views or pickle cross ticks. + arrays = {name: np.asarray(rows) for name, rows in self._dense_arrays.items()} + with (folder/'native-arrays.npz').open('xb') as stream: + np.savez_compressed(stream, **arrays) + write_gzip(folder/'native-contacts-scorer.json.gz', self._dense_rows) + return {'rows': len(self._dense_rows), + 'array_fields': list(arrays), + 'sample_timing': 'Initial post-reset state, then each post-physics/post-scoring native 1 ms tick; control-step bookkeeping is also compared at 20 ms boundaries.', + 'overhead': 'Diagnostic callbacks copy state and re-read contacts; all diagnostic runs are excluded from the performance comparison.'} + return Dense + + +def worker(args): + folder = args.output + protocol_raw = args.protocol.read_bytes() + protocol = decode(protocol_raw) + if sha256(protocol_raw).hexdigest() != args.protocol_sha: + raise ValueError('Protocol changed before worker launch') + root = args.source_root.resolve() + frozen = protocol['sources'][args.variant] + check_inventory(root, frozen) + # -s/-P and PYTHONPATH isolate the requested source. Fail closed if an + # editable install or external environment redirected package resolution. + from dvidia_training import piper_task, reference_sim as sim, robot_profiles + from dvidia_training.piper_imitation_score import score_episode + import numpy as np + import mujoco as mj + if Path(piper_task.__file__).resolve() != root/'src/dvidia_training/piper_task.py': + raise ValueError('Worker imported a different source checkout') + expected, commands, tape_id = load_tape(args.episode) + if tape_id != protocol['episode']: + raise ValueError('Tape changed after protocol freeze') + if identity(Path(sys.executable).read_bytes()) != protocol['interpreter']: + raise ValueError('Worker interpreter differs from protocol') + base = piper_task.PiperTaskEnv + kwargs = {} + if args.case.startswith('feedback-'): + from dvidia_training.piper_feedback_env import FeedbackPiperTaskEnv, JawPulse + base = FeedbackPiperTaskEnv + kwargs['pulses'] = (JawPulse(**PULSE),) if args.case == 'feedback-pulse' else () + if args.mode == 'diagnostic': + base = _diagnostic_class(base, np, mj) + env = base(args.assets, piper_task.PiperTaskConfig(**expected['config']), **kwargs) + episode = {'config': asdict(env.config), 'seed': expected['seed'], 'trajectory': [], + 'actions': [], 'error': None} + timing = {'step_wall_samples_s': [], 'step_cpu_samples_s': []} + result = {'schema': SCHEMA, 'variant': args.variant, 'mode': args.mode, 'case': args.case, + 'protocol_sha256': args.protocol_sha, 'status': 'started', 'scope': SCOPE, + 'runtime': {'python': platform.python_version(), 'machine': platform.machine(), + 'system': platform.system(), 'mujoco': mj.__version__, 'numpy': np.__version__}} + start_wall, start_cpu = time.perf_counter(), time.process_time() + try: + setup = time.perf_counter() + observation, info = env.reset(expected['seed']) + timing['setup_reset_wall_s'] = time.perf_counter()-setup + episode.update(initial_object_position_m=env.initial_object_position.tolist(), + compiled_model_sha256=sim.model_digest(env.model)) + result['assets'] = robot_profiles.verify_assets('piper', args.assets) + result['model'] = {'sha256': episode['compiled_model_sha256'], + 'dimensions': {name: int(getattr(env.model, name)) + for name in ('nq', 'nv', 'nu', 'neq', 'nbody', 'ngeom')}, + 'timestep_s': float(env.model.opt.timestep), + 'native_gains': env.model.actuator_gainprm.tolist(), + 'native_bias': env.model.actuator_biasprm.tolist(), + 'native_force_limits': env.model.actuator_forcerange.tolist(), + 'native_control_limits': env.model.actuator_ctrlrange.tolist()} + if episode['compiled_model_sha256'] != expected['compiled_model_sha256']: + raise ValueError('Compiled model differs from preserved tape') + episode['trajectory'].append(observation) + if args.mode == 'diagnostic': + env.capture() + hold_targets = observation['joint_position_rad'][:] + for saved in commands: + action = {'joint_targets_rad': saved['joint_targets_rad'][:], + 'gripper_position_m': saved['gripper_position_m']} + if args.case == 'open-jaw': + action['gripper_position_m'] = .035 + elif args.case == 'hold': + action = {'joint_targets_rad': hold_targets[:], 'gripper_position_m': .035} + record = {'t_seconds': observation['t_seconds'], **action} + episode['actions'].append(record) + observation.update(issued_joint_targets_rad=action['joint_targets_rad'][:], + issued_gripper_position_m=action['gripper_position_m'], command=record) + wall, cpu = time.perf_counter(), time.process_time() + observation, info = env.step(action) + timing['step_wall_samples_s'].append(time.perf_counter()-wall) + timing['step_cpu_samples_s'].append(time.process_time()-cpu) + episode['trajectory'].append(observation) + if env.done: + break + episode.update(valid=info['valid'], success=info['success'], reason=info['reason'], + simulated_seconds=info['simulation_time'], + metrics={**info, 'contact_pair_native_ticks': dict(env.contact_pair_native_ticks)}) + result['score'] = score_episode(episode, expected['config']['target_position_m'][:2]) + result['boundary_comparison_to_preserved'] = exact_document(expected, episode, EPISODE_FIELDS) + if len(episode['actions']) != 1000 or len(episode['trajectory']) != 1001: + raise ValueError('Attempt did not complete its fixed command budget') + if not info['valid'] or info['warnings']: + raise ValueError('Native attempt invalid or emitted warnings') + if args.case in ('original', 'feedback-no-pulse') and not result['boundary_comparison_to_preserved']['exact_match']: + raise ValueError('Original/no-pulse mechanics differs from preserved episode') + if args.case in ('open-jaw', 'hold') and result['score']['success']: + raise ValueError('Negative control unexpectedly passed the independent scorer') + check_inventory(root, frozen) + result['status'] = 'complete' + except Exception as exc: + episode['error'] = type(exc).__name__+': '+str(exc) + result.update(status='failed', error=episode['error']) + (folder/'worker-traceback.txt').write_text(traceback.format_exc()) + finally: + timing.update(worker_wall_s=time.perf_counter()-start_wall, + worker_process_cpu_s=time.process_time()-start_cpu) + result['timing'] = {**timing, + 'environment_step_wall': summary(timing['step_wall_samples_s']), + 'environment_step_cpu': summary(timing['step_cpu_samples_s']), + 'scope': 'Unpaced shared-host CPU compute. Step includes validation, 20 native 1 ms integrations, safety/contact/scoring and returned observation/info; no policy inference. Reset includes asset checks, model compile and settling. Worker total also includes provenance/record bookkeeping. Diagnostic samples are never used as speed evidence.'} + if args.mode == 'diagnostic': + result['dense'] = env.save_dense(folder) + write_gzip(folder/'episode.json.gz', episode) + try: + check_inventory(root, frozen) + result['source_unchanged'] = True + except Exception as exc: + result.update(status='failed', source_unchanged=False, source_error=str(exc)) + write(folder/'result.json', result) + return 0 if result['status'] == 'complete' else 1 + + +def compare_attempts(left, right, diagnostic=False): + a, b = decode((left/'result.json').read_bytes()), decode((right/'result.json').read_bytes()) + checks = {'both_complete': a['status'] == b['status'] == 'complete', + 'same_model': exact_document(a.get('model', {}), b.get('model', {})), + 'same_runtime': exact_document(a.get('runtime', {}), b.get('runtime', {})), + 'same_assets': exact_document(a.get('assets', {}), b.get('assets', {})), + 'same_score': exact_document(a.get('score', {}), b.get('score', {})), + 'same_boundaries': exact_document(read_gzip(left/'episode.json.gz'), + read_gzip(right/'episode.json.gz'), EPISODE_FIELDS)} + if diagnostic: + checks['same_native_arrays'] = compare_arrays(left/'native-arrays.npz', right/'native-arrays.npz') + x, y = read_gzip(left/'native-contacts-scorer.json.gz'), read_gzip(right/'native-contacts-scorer.json.gz') + first = next((index for index, (p, q) in enumerate(zip(x, y)) if canonical(p) != canonical(q)), None) + checks['same_native_contacts_scorer'] = {'exact_match': len(x) == len(y) and first is None, + 'baseline_rows': len(x), 'candidate_rows': len(y), + 'first_different_row': first} + checks['complete_native_cadence'] = len(x) == len(y) == 20001 + checks['exact_match'] = all(value if type(value) is bool else value['exact_match'] + for value in checks.values()) + return checks + + +def check_inputs(script, protocol_path, protocol_sha, python, protocol, episode_path): + if identity(Path(script).read_bytes()) != protocol['harness']: + raise ValueError('Harness changed during the comparison') + if sha256(Path(protocol_path).read_bytes()).hexdigest() != protocol_sha: + raise ValueError('Protocol changed during the comparison') + if identity(Path(python).read_bytes()) != protocol['interpreter']: + raise ValueError('Interpreter changed during the comparison') + if identity(Path(episode_path).read_bytes()) != protocol['episode']: + raise ValueError('Preserved episode changed during the comparison') + + +def adoption_result(timing_runs, exact_equivalence): + complete = len(timing_runs) >= 4 and len(timing_runs) % 2 == 0 and all( + row['status'] == 'complete' for row in timing_runs) + pair_checks, pooled = [], {} + if complete: + for index in range(0, len(timing_runs), 2): + pair = {row['variant']: row for row in timing_runs[index:index+2]} + if set(pair) != {'baseline', 'candidate'}: + raise ValueError('Each timing pair must contain baseline and candidate') + pair_checks.append({'pair': index//2, **{ + clock: pair['candidate'][clock]['mean_s'] < pair['baseline'][clock]['mean_s'] + for clock in ('step_wall', 'step_cpu')}}) + for clock in ('step_wall', 'step_cpu'): + means = {} + for variant in ('baseline', 'candidate'): + rows = [row[clock] for row in timing_runs if row['variant'] == variant] + means[variant] = sum(row['sum_s'] for row in rows)/sum(row['count'] for row in rows) + pooled[clock] = {'mean_s': means, + 'candidate_mean_reduction_percent': 100*(1-means['candidate']/means['baseline'])} + eligible = (exact_equivalence and complete + and all(row[clock] for row in pair_checks for clock in ('step_wall', 'step_cpu')) + and all(row['candidate_mean_reduction_percent'] >= 5. for row in pooled.values())) + return {'rule': ADOPTION_RULE, 'eligible_under_predeclared_rule': bool(eligible), + 'paired_lower_mean': pair_checks, 'pooled': pooled, + 'decision': 'Eligible for parent review' if eligible else 'Do not promote runtime change'} + + +def run(args): + output = args.output.absolute() + if output.exists() or output.is_symlink(): + raise ValueError('Output must be new; no evidence is overwritten') + order = schedule(args.pairs) + roots = {'baseline': args.baseline_root.resolve(), 'candidate': args.candidate_root.resolve()} + if roots['baseline'] == roots['candidate']: + raise ValueError('Baseline and candidate require independent source checkouts') + sources = {name: inventory(root) for name, root in roots.items()} + differences = source_differences(sources['baseline'], sources['candidate']) + expected, commands, tape_id = load_tape(args.episode) + output.mkdir(parents=True) + script = Path(__file__).resolve() + # Preserve a venv executable's spelling: resolving its symlink would launch + # the global interpreter and silently lose that environment's dependencies. + python = args.python.absolute() + protocol = {'schema': SCHEMA, 'scope': SCOPE, 'sources': sources, + 'source_differences': differences, 'allowed_source_differences': list(ALLOWED_SOURCE_DIFFERENCES), + 'harness': identity(script.read_bytes()), 'interpreter': identity(python.read_bytes()), + 'episode': tape_id, 'commands_sha256': sha256(canonical(commands)).hexdigest(), + 'seed': expected['seed'], 'config': expected['config'], + 'timing_order': order, 'timing_pairs': args.pairs, + 'diagnostic_cases': list(CASES), 'diagnostic_pulse': PULSE, + 'adoption_rule': ADOPTION_RULE, + 'dense_native_state_fields': list(NATIVE_ARRAYS)+['integration_state', 'native_time', 'command_targets', 'command_gripper'], + 'dense_scorer_fields': list(SCORER_FIELDS), + 'comparison': 'Exact boundary JSON, bitwise numeric native arrays, exact ordered raw/active contacts and cumulative scorer. No timing or source hashes are present in mechanics documents; their identities are compared separately.', + 'timing_design': 'Two or more fixed matched pairs in alternating AB/BA order; no retry, tuning, warm-up selection, dropped attempts or best-run selection. Diagnostic callbacks run only after the independent performance schedule.'} + write(output/'protocol.json', protocol) + protocol_sha = sha256((output/'protocol.json').read_bytes()).hexdigest() + (output/'preserved-episode.json.gz').write_bytes(args.episode.read_bytes()) + for variant, root in roots.items(): + destination = output/'sources'/variant + destination.mkdir(parents=True) + for name in SOURCE_FILES: + (destination/name).write_bytes((root/'src/dvidia_training'/name).read_bytes()) + (output/'compare.py').write_bytes(script.read_bytes()) + planned = [{'id': f'timing-{index:02d}-{variant}', 'variant': variant, 'mode': 'timing', 'case': 'original'} + for index, variant in enumerate(order)] + planned += [{'id': f'dense-{case}-{variant}', 'variant': variant, 'mode': 'diagnostic', 'case': case} + for case in CASES for variant in ('baseline', 'candidate')] + write(output/'schedule.json', planned) + attempts = [] + for row in planned: + destination = output/row['id'] + destination.mkdir() + write(destination/'started.json', {**row, 'protocol_sha256': protocol_sha}) + status = {**row, 'returncode': None} + try: + for variant, root in roots.items(): + check_inventory(root, sources[variant]) + check_inputs(script, output/'protocol.json', protocol_sha, python, protocol, + output/'preserved-episode.json.gz') + command = [str(python), '-s', '-P', str(script), '--worker', + '--source-root', str(roots[row['variant']]), '--variant', row['variant'], + '--protocol', str(output/'protocol.json'), '--protocol-sha', protocol_sha, + '--episode', str(output/'preserved-episode.json.gz'), '--assets', str(args.assets.resolve()), + '--output', str(destination), '--mode', row['mode'], '--case', row['case']] + environment = {key: value for key, value in os.environ.items() + if key not in ('PYTHONPATH', 'PYTHONHOME', 'PYTHONSTARTUP')} + environment.update(PYTHONPATH=str(roots[row['variant']]/'src'), PYTHONNOUSERSITE='1', PYTHONDONTWRITEBYTECODE='1', + OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1', VECLIB_MAXIMUM_THREADS='1') + wall = time.perf_counter() + with (destination/'stdout.txt').open('xb') as stdout, (destination/'stderr.txt').open('xb') as stderr: + completed = subprocess.run(command, cwd=destination, env=environment, stdout=stdout, + stderr=stderr, timeout=args.worker_timeout, check=False) + status.update(returncode=completed.returncode, subprocess_wall_s=time.perf_counter()-wall) + for variant, root in roots.items(): + check_inventory(root, sources[variant]) + check_inputs(script, output/'protocol.json', protocol_sha, python, protocol, + output/'preserved-episode.json.gz') + except Exception as exc: + status['error'] = type(exc).__name__+': '+str(exc) + write(destination/'process.json', status) + attempts.append(status) + print(json.dumps({'attempt': row['id'], 'returncode': status['returncode'], + 'error': status.get('error')}), flush=True) + comparisons = [] + for index in range(0, len(order), 2): + ids = planned[index:index+2] + by_variant = {row['variant']: output/row['id'] for row in ids} + try: + result = compare_attempts(by_variant['baseline'], by_variant['candidate']) + except Exception as exc: + result = {'exact_match': False, 'error': type(exc).__name__+': '+str(exc)} + comparisons.append({'kind': 'timing-pair', 'pair': index//2, **result}) + for case in CASES: + try: + result = compare_attempts(output/f'dense-{case}-baseline', output/f'dense-{case}-candidate', True) + except Exception as exc: + result = {'exact_match': False, 'error': type(exc).__name__+': '+str(exc)} + comparisons.append({'kind': 'dense-equivalence', 'case': case, **result}) + timing_runs = [] + for row in planned[:len(order)]: + try: + result = decode((output/row['id']/'result.json').read_bytes()) + timing_runs.append({'id': row['id'], 'variant': row['variant'], 'status': result['status'], + 'step_wall': result['timing']['environment_step_wall'], + 'step_cpu': result['timing']['environment_step_cpu'], + 'setup_reset_wall_s': result['timing'].get('setup_reset_wall_s')}) + except Exception: + timing_runs.append({'id': row['id'], 'variant': row['variant'], 'status': 'missing-result'}) + paired = [] + for index in range(0, len(timing_runs), 2): + pair = {row['variant']: row for row in timing_runs[index:index+2]} + if all(row['status'] == 'complete' for row in pair.values()): + a, b = pair['baseline']['step_wall'], pair['candidate']['step_wall'] + paired.append({'pair': index//2, 'baseline_sum_s': a['sum_s'], 'candidate_sum_s': b['sum_s'], + 'candidate_total_step_reduction_percent': 100*(1-b['sum_s']/a['sum_s']), + 'candidate_p95_reduction_percent': 100*(1-b['p95_s']/a['p95_s'])}) + unchanged = all(inventory(roots[name]) == frozen for name, frozen in sources.items()) + try: + check_inputs(script, output/'protocol.json', protocol_sha, python, protocol, + output/'preserved-episode.json.gz') + except Exception: + unchanged = False + report = {'schema': SCHEMA, 'protocol_sha256': protocol_sha, 'scope': SCOPE, + 'attempts': attempts, 'planned_attempts': len(planned), 'source_unchanged': unchanged, + 'comparisons': comparisons, 'timing_runs': timing_runs, 'paired_timing': paired, + 'all_passed': unchanged and all(row.get('returncode') == 0 and 'error' not in row for row in attempts) + and all(row['exact_match'] for row in comparisons), + 'interpretation': 'These unpaced shared-host CPU samples compare one unchanged synthetic command tape. No inference time is included. Do not infer new task accuracy, hardware response, minimum hardware or real-time safety. Report every paired result; two pairs cannot establish robust population speedup.'} + report['adoption'] = adoption_result(timing_runs, report['all_passed']) + write(output/'comparison.json', report) + return report + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--worker', action='store_true', help=argparse.SUPPRESS) + parser.add_argument('--baseline-root', type=Path) + parser.add_argument('--candidate-root', type=Path) + parser.add_argument('--python', type=Path, default=Path(sys.executable)) + parser.add_argument('--episode', type=Path, required=True) + parser.add_argument('--assets', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--pairs', type=int, default=2) + parser.add_argument('--worker-timeout', type=float, default=300.) + parser.add_argument('--source-root', type=Path) + parser.add_argument('--variant', choices=('baseline', 'candidate')) + parser.add_argument('--protocol', type=Path) + parser.add_argument('--protocol-sha') + parser.add_argument('--mode', choices=('timing', 'diagnostic')) + parser.add_argument('--case', choices=CASES) + args = parser.parse_args(argv) + if args.worker: + return worker(args) + if args.baseline_root is None or args.candidate_root is None: + parser.error('Both independent source roots are required') + report = run(args) + print(json.dumps({'all_passed': report['all_passed'], 'attempts': len(report['attempts']), + 'paired_timing': report['paired_timing']}), flush=True) + return 0 if report['all_passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/piper-control-profile-v01/comparison.json b/benchmarks/piper-control-profile-v01/comparison.json new file mode 100644 index 0000000..1645c31 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/comparison.json @@ -0,0 +1,652 @@ +{ + "adoption": { + "decision": "Do not promote runtime change", + "eligible_under_predeclared_rule": false, + "paired_lower_mean": [ + { + "pair": 0, + "step_cpu": true, + "step_wall": true + }, + { + "pair": 1, + "step_cpu": true, + "step_wall": true + } + ], + "pooled": { + "step_cpu": { + "candidate_mean_reduction_percent": 12.599688117239072, + "mean_s": { + "baseline": 0.0014383610000000013, + "candidate": 0.001257132 + } + }, + "step_wall": { + "candidate_mean_reduction_percent": 15.69721381493122, + "mean_s": { + "baseline": 0.0015054188838694244, + "candidate": 0.0012691100628580898 + } + } + }, + "rule": { + "minimum_pooled_mean_reduction_percent_each_clock": 5.0, + "on_failure": "Do not promote; retain as negative or inconclusive. No post-result tuning.", + "required_exact_equivalence": true, + "required_lower_candidate_mean_each_pair": [ + "step_wall", + "step_cpu" + ] + } + }, + "all_passed": false, + "attempts": [ + { + "case": "original", + "id": "timing-00-baseline", + "mode": "timing", + "returncode": 0, + "subprocess_wall_s": 2.420006916858256, + "variant": "baseline" + }, + { + "case": "original", + "id": "timing-01-candidate", + "mode": "timing", + "returncode": 0, + "subprocess_wall_s": 2.0803113747388124, + "variant": "candidate" + }, + { + "case": "original", + "id": "timing-02-candidate", + "mode": "timing", + "returncode": 0, + "subprocess_wall_s": 2.0139063335955143, + "variant": "candidate" + }, + { + "case": "original", + "id": "timing-03-baseline", + "mode": "timing", + "returncode": 0, + "subprocess_wall_s": 2.4118966250680387, + "variant": "baseline" + }, + { + "case": "original", + "id": "dense-original-baseline", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 6.503215416800231, + "variant": "baseline" + }, + { + "case": "original", + "id": "dense-original-candidate", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 6.568463541567326, + "variant": "candidate" + }, + { + "case": "open-jaw", + "id": "dense-open-jaw-baseline", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 5.4805178749375045, + "variant": "baseline" + }, + { + "case": "open-jaw", + "id": "dense-open-jaw-candidate", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 6.379173707682639, + "variant": "candidate" + }, + { + "case": "hold", + "id": "dense-hold-baseline", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 5.533948041964322, + "variant": "baseline" + }, + { + "case": "hold", + "id": "dense-hold-candidate", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 5.328347083181143, + "variant": "candidate" + }, + { + "case": "feedback-no-pulse", + "id": "dense-feedback-no-pulse-baseline", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 7.0851437500678, + "variant": "baseline" + }, + { + "case": "feedback-no-pulse", + "id": "dense-feedback-no-pulse-candidate", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 7.633159082848579, + "variant": "candidate" + }, + { + "case": "feedback-pulse", + "id": "dense-feedback-pulse-baseline", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 7.811261875089258, + "variant": "baseline" + }, + { + "case": "feedback-pulse", + "id": "dense-feedback-pulse-candidate", + "mode": "diagnostic", + "returncode": 0, + "subprocess_wall_s": 8.18520720815286, + "variant": "candidate" + } + ], + "comparisons": [ + { + "both_complete": true, + "exact_match": true, + "kind": "timing-pair", + "pair": 0, + "same_assets": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "bytes_verified", + "commit", + "files_verified", + "lock_sha256", + "profile", + "repository", + "simulation_asset_license" + ] + }, + "same_boundaries": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "config", + "seed", + "valid", + "success", + "reason", + "error", + "simulated_seconds", + "initial_object_position_m", + "compiled_model_sha256", + "trajectory", + "actions", + "metrics" + ] + }, + "same_model": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "dimensions", + "native_bias", + "native_control_limits", + "native_force_limits", + "native_gains", + "sha256", + "timestep_s" + ] + }, + "same_runtime": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "machine", + "mujoco", + "numpy", + "python", + "system" + ] + }, + "same_score": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "native_metrics_source", + "native_outcome_success", + "native_outcome_valid", + "reason", + "reference_xy_m", + "required_dwell_s", + "required_lift_dwell_s", + "sample_interval_s", + "success", + "terminal_dwell_s", + "terminal_reference_success", + "terminal_xy_error_m", + "trace_lift_dwell_s", + "trajectory_complete" + ] + } + }, + { + "both_complete": true, + "exact_match": true, + "kind": "timing-pair", + "pair": 1, + "same_assets": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "bytes_verified", + "commit", + "files_verified", + "lock_sha256", + "profile", + "repository", + "simulation_asset_license" + ] + }, + "same_boundaries": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "config", + "seed", + "valid", + "success", + "reason", + "error", + "simulated_seconds", + "initial_object_position_m", + "compiled_model_sha256", + "trajectory", + "actions", + "metrics" + ] + }, + "same_model": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "dimensions", + "native_bias", + "native_control_limits", + "native_force_limits", + "native_gains", + "sha256", + "timestep_s" + ] + }, + "same_runtime": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "machine", + "mujoco", + "numpy", + "python", + "system" + ] + }, + "same_score": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "native_metrics_source", + "native_outcome_success", + "native_outcome_valid", + "reason", + "reference_xy_m", + "required_dwell_s", + "required_lift_dwell_s", + "sample_interval_s", + "success", + "terminal_dwell_s", + "terminal_reference_success", + "terminal_xy_error_m", + "trace_lift_dwell_s", + "trajectory_complete" + ] + } + }, + { + "case": "original", + "error": "ValueError: Expanded document exceeds bound", + "exact_match": false, + "kind": "dense-equivalence" + }, + { + "both_complete": true, + "case": "open-jaw", + "complete_native_cadence": true, + "exact_match": true, + "kind": "dense-equivalence", + "same_assets": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "bytes_verified", + "commit", + "files_verified", + "lock_sha256", + "profile", + "repository", + "simulation_asset_license" + ] + }, + "same_boundaries": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "config", + "seed", + "valid", + "success", + "reason", + "error", + "simulated_seconds", + "initial_object_position_m", + "compiled_model_sha256", + "trajectory", + "actions", + "metrics" + ] + }, + "same_model": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "dimensions", + "native_bias", + "native_control_limits", + "native_force_limits", + "native_gains", + "sha256", + "timestep_s" + ] + }, + "same_native_arrays": { + "differences": [], + "exact_match": true + }, + "same_native_contacts_scorer": { + "baseline_rows": 20001, + "candidate_rows": 20001, + "exact_match": true, + "first_different_row": null + }, + "same_runtime": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "machine", + "mujoco", + "numpy", + "python", + "system" + ] + }, + "same_score": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "native_metrics_source", + "native_outcome_success", + "native_outcome_valid", + "reason", + "reference_xy_m", + "required_dwell_s", + "required_lift_dwell_s", + "sample_interval_s", + "success", + "terminal_dwell_s", + "terminal_reference_success", + "terminal_xy_error_m", + "trace_lift_dwell_s", + "trajectory_complete" + ] + } + }, + { + "both_complete": true, + "case": "hold", + "complete_native_cadence": true, + "exact_match": true, + "kind": "dense-equivalence", + "same_assets": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "bytes_verified", + "commit", + "files_verified", + "lock_sha256", + "profile", + "repository", + "simulation_asset_license" + ] + }, + "same_boundaries": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "config", + "seed", + "valid", + "success", + "reason", + "error", + "simulated_seconds", + "initial_object_position_m", + "compiled_model_sha256", + "trajectory", + "actions", + "metrics" + ] + }, + "same_model": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "dimensions", + "native_bias", + "native_control_limits", + "native_force_limits", + "native_gains", + "sha256", + "timestep_s" + ] + }, + "same_native_arrays": { + "differences": [], + "exact_match": true + }, + "same_native_contacts_scorer": { + "baseline_rows": 20001, + "candidate_rows": 20001, + "exact_match": true, + "first_different_row": null + }, + "same_runtime": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "machine", + "mujoco", + "numpy", + "python", + "system" + ] + }, + "same_score": { + "different_fields": [], + "exact_match": true, + "fields_compared": [ + "native_metrics_source", + "native_outcome_success", + "native_outcome_valid", + "reason", + "reference_xy_m", + "required_dwell_s", + "required_lift_dwell_s", + "sample_interval_s", + "success", + "terminal_dwell_s", + "terminal_reference_success", + "terminal_xy_error_m", + "trace_lift_dwell_s", + "trajectory_complete" + ] + } + }, + { + "case": "feedback-no-pulse", + "error": "ValueError: Expanded document exceeds bound", + "exact_match": false, + "kind": "dense-equivalence" + }, + { + "case": "feedback-pulse", + "error": "ValueError: Expanded document exceeds bound", + "exact_match": false, + "kind": "dense-equivalence" + } + ], + "interpretation": "These unpaced shared-host CPU samples compare one unchanged synthetic command tape. No inference time is included. Do not infer new task accuracy, hardware response, minimum hardware or real-time safety. Report every paired result; two pairs cannot establish robust population speedup.", + "paired_timing": [ + { + "baseline_sum_s": 1.4684963417239487, + "candidate_p95_reduction_percent": 25.79315547361659, + "candidate_sum_s": 1.2328931200318038, + "candidate_total_step_reduction_percent": 16.043841240731815, + "pair": 0 + }, + { + "baseline_sum_s": 1.5423414260149002, + "candidate_p95_reduction_percent": 23.795316993807734, + "candidate_sum_s": 1.3053270056843758, + "candidate_total_step_reduction_percent": 15.367182410636659, + "pair": 1 + } + ], + "planned_attempts": 14, + "protocol_sha256": "386c34e8201a86dbe81709805b7c0d47ec4c257120023514153cc4df652f5184", + "schema": "dvidia.piper-control-comparison/v1", + "scope": "One previously checked synthetic PiPER scenario, preserved commands and privileged native simulator state. Allocation-only source comparison, not new skill training, held-out success, hardware latency, calibrated contact or a real-time guarantee.", + "source_unchanged": true, + "timing_runs": [ + { + "id": "timing-00-baseline", + "setup_reset_wall_s": 0.367916792165488, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.012059000000000042, + "mean_s": 0.0014127509999999999, + "over_nominal_20ms": 0, + "p50_s": 0.0011879999999999669, + "p95_s": 0.002274649999999988, + "p99_s": 0.002799599999999997, + "sum_s": 1.4127509999999999 + }, + "step_wall": { + "count": 1000, + "max_s": 0.012310583610087633, + "mean_s": 0.0014684963417239487, + "over_nominal_20ms": 0, + "p50_s": 0.0012011665385216475, + "p95_s": 0.002542358636856079, + "p99_s": 0.003286311766132712, + "sum_s": 1.4684963417239487 + }, + "variant": "baseline" + }, + { + "id": "timing-01-candidate", + "setup_reset_wall_s": 0.28948104102164507, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.008727999999999847, + "mean_s": 0.0012205139999999996, + "over_nominal_20ms": 0, + "p50_s": 0.0010654999999999415, + "p95_s": 0.0018321999999998393, + "p99_s": 0.0024473500000000855, + "sum_s": 1.2205139999999997 + }, + "step_wall": { + "count": 1000, + "max_s": 0.010767832864075899, + "mean_s": 0.0012328931200318038, + "over_nominal_20ms": 0, + "p50_s": 0.0010669578332453966, + "p95_s": 0.001886604120954871, + "p99_s": 0.0025030286889523266, + "sum_s": 1.2328931200318038 + }, + "variant": "candidate" + }, + { + "id": "timing-02-candidate", + "setup_reset_wall_s": 0.25960616674274206, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.00799099999999986, + "mean_s": 0.0012937500000000002, + "over_nominal_20ms": 0, + "p50_s": 0.001134999999999775, + "p95_s": 0.001920649999999912, + "p99_s": 0.0027079999999999327, + "sum_s": 1.2937500000000002 + }, + "step_wall": { + "count": 1000, + "max_s": 0.008000124711543322, + "mean_s": 0.0013053270056843757, + "over_nominal_20ms": 0, + "p50_s": 0.0011370624415576458, + "p95_s": 0.0019861731911078096, + "p99_s": 0.00277163659222424, + "sum_s": 1.3053270056843758 + }, + "variant": "candidate" + }, + { + "id": "timing-03-baseline", + "setup_reset_wall_s": 0.3268076251260936, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.018218000000000067, + "mean_s": 0.001463971000000003, + "over_nominal_20ms": 0, + "p50_s": 0.0012210000000000276, + "p95_s": 0.002330100000000157, + "p99_s": 0.0029761600000003073, + "sum_s": 1.463971000000003 + }, + "step_wall": { + "count": 1000, + "max_s": 0.02005712501704693, + "mean_s": 0.0015423414260149002, + "over_nominal_20ms": 1, + "p50_s": 0.001237792195752263, + "p95_s": 0.0026063663186505436, + "p99_s": 0.003594016665592789, + "sum_s": 1.5423414260149002 + }, + "variant": "baseline" + } + ] +} diff --git a/benchmarks/piper-control-profile-v01/package_evidence.py b/benchmarks/piper-control-profile-v01/package_evidence.py new file mode 100644 index 0000000..615cd7e --- /dev/null +++ b/benchmarks/piper-control-profile-v01/package_evidence.py @@ -0,0 +1,411 @@ +"""Deterministic, offline packaging of reviewed synthetic control-profile evidence. + +Logical files retain their original bytes, represented once per SHA256 blob. +This does not rerun physics, change frozen evidence, publish, or create weights. +""" +from __future__ import annotations + +import argparse +import gzip +from hashlib import sha256 +import json +from pathlib import Path, PurePosixPath +import re +import stat +import zipfile + +SCHEMA = 'dvidia.evidence-blobs/v1' +AUDIT_SHA = '1a4f4a1093961b59c03a3a244d123afed35f5a5f77c98d1467a393a20c2d6fd4' +MAX_BLOB = 128*1024*1024 +MAX_EXPANDED = 512*1024*1024 +MAX_ARCHIVE = 2*1024*1024*1024 +# Split test prefixes so this packaged audit source is not itself a path leak. +PRIVATE = (b'/'+b'Users/', b'/private/'+b'tmp/', b'/opt/'+b'homebrew/', b'file:///'+b'Users/') +SECRETS = tuple(re.compile(value) for value in ( + rb'hf_[A-Za-z0-9]{20,}', rb'gh[pousr]_[A-Za-z0-9]{20,}', + rb'apikey_[A-Za-z0-9_]{20,}', rb'AKIA[A-Z0-9]{16}')) + +VERIFY_SOURCE = r'''"""Verify or reconstruct a DVIDIA content-addressed evidence archive, offline. + +No archive code is imported, no pickle is read, and extractall is never used. +""" +from hashlib import sha256 +import argparse +import json +from pathlib import Path, PurePosixPath +import re +import stat +import zipfile + +MAX_BLOB = 128*1024*1024 +MAX_TOTAL = 2*1024*1024*1024 +MAX_FILES = 4096 +HEADER_FILES = {'README.md', 'LICENSE', 'verify_extract.py', 'manifest.json'} + +def safe_path(value): + if (type(value) is not str or len(value) > 400 or not value + or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]*', value) + or any(part in ('', '.', '..') for part in value.split('/'))): + raise ValueError('Unsafe logical path') + return value + +def unique(rows): + value = {} + for key, item in rows: + if key in value: raise ValueError('Duplicate JSON field') + value[key] = item + return value + +def decode(raw): + def bad(token): raise ValueError('Nonfinite JSON: '+token) + return json.loads(raw, object_pairs_hook=unique, parse_constant=bad) + +def verify(archive, output=None): + with zipfile.ZipFile(archive) as bundle: + entries = bundle.infolist() + names = [entry.filename for entry in entries] + if len(names) != len(set(names)) or len(names) > MAX_FILES: + raise ValueError('Duplicate or excessive archive members') + if sum(entry.file_size for entry in entries) > MAX_TOTAL: + raise ValueError('Archive expands beyond the total budget') + for entry in entries: + if (entry.filename not in HEADER_FILES + and not re.fullmatch(r'blobs/[0-9a-f]{64}', entry.filename)): + raise ValueError('Unexpected archive member') + if (entry.file_size > MAX_BLOB or entry.flag_bits & 1 + or stat.S_IFMT(entry.external_attr >> 16) == stat.S_IFLNK + or entry.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED)): + raise ValueError('Unsafe archive member or expansion budget') + if not HEADER_FILES.issubset(names) or bundle.getinfo('manifest.json').file_size > 8*1024*1024: + raise ValueError('Missing or oversized archive manifest') + manifest = decode(bundle.read('manifest.json')) + if (manifest.get('schema') != 'dvidia.evidence-blobs/v1' + or type(manifest.get('files')) is not list + or not 1 <= len(manifest['files']) <= MAX_FILES): + raise ValueError('Unknown evidence manifest') + paths, blobs, records = set(), {}, {} + total_logical = 0 + for row in manifest['files']: + if type(row) is not dict or set(row) != {'path', 'bytes', 'sha256', 'blob'}: + raise ValueError('Invalid logical file record') + path = safe_path(row['path']) + digest = row['sha256'] + if (path in paths or type(digest) is not str + or not re.fullmatch(r'[0-9a-f]{64}', digest) + or type(row['bytes']) is not int or not 0 <= row['bytes'] <= MAX_BLOB + or row['blob'] != 'blobs/'+digest): + raise ValueError('Duplicate path or invalid blob identity') + paths.add(path) + records[path] = row + total_logical += row['bytes'] + if row['blob'] in blobs and blobs[row['blob']] != row['bytes']: + raise ValueError('Conflicting blob lengths') + blobs[row['blob']] = row['bytes'] + if total_logical > MAX_TOTAL: + raise ValueError('Logical reconstruction exceeds the total budget') + if any(str(parent) in paths for path in paths for parent in PurePosixPath(path).parents + if str(parent) != '.'): + raise ValueError('File and directory paths collide') + if set(names) != HEADER_FILES | set(blobs): + raise ValueError('Missing or unreferenced blob') + for name, expected_size in blobs.items(): + if bundle.getinfo(name).file_size != expected_size: + raise ValueError('Blob size mismatch') + digest, size = sha256(), 0 + with bundle.open(name) as stream: + while chunk := stream.read(65536): + size += len(chunk) + if size > expected_size: raise ValueError('Blob exceeds declared size') + digest.update(chunk) + if size != expected_size or digest.hexdigest() != name[6:]: + raise ValueError('Blob digest mismatch') + for physical, logical in (('README.md', 'README.md'), ('LICENSE', 'LICENSE'), + ('verify_extract.py', 'tools/verify_extract.py')): + row = records.get(logical) + content = bundle.read(physical) + if row is None or len(content) != row['bytes'] or sha256(content).hexdigest() != row['sha256']: + raise ValueError('Header differs from its logical inventory') + if output is not None: + destination = Path(output) + if destination.exists() or destination.is_symlink(): + raise ValueError('Reconstruction destination must be new') + destination.mkdir(parents=True, exist_ok=False) + root = destination.resolve() + for path in sorted(paths): + target = destination.joinpath(*PurePosixPath(path).parts) + target.parent.mkdir(parents=True, exist_ok=True) + if not target.parent.resolve().is_relative_to(root): + raise ValueError('Reconstruction escaped its destination') + with bundle.open(records[path]['blob']) as source, target.open('xb') as sink: + while chunk := source.read(65536): sink.write(chunk) + return {'verified': True, 'logical_files': len(paths), 'unique_blobs': len(blobs), + 'logical_bytes': total_logical, 'unique_blob_bytes': sum(blobs.values()), + 'reconstructed': output is not None} + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('archive', type=Path) + parser.add_argument('--output', type=Path) + args = parser.parse_args() + print(json.dumps(verify(args.archive, args.output), sort_keys=True)) + +if __name__ == '__main__': main() +''' + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def pretty(value): + return json.dumps(value, sort_keys=True, indent=2, allow_nan=False).encode()+b'\n' + + +def file_identity(path, maximum=MAX_BLOB): + digest, size = sha256(), 0 + with Path(path).open('rb') as stream: + while chunk := stream.read(1024*1024): + digest.update(chunk) + size += len(chunk) + if size > maximum: raise ValueError('File exceeds the packaging budget') + return {'bytes': size, 'sha256': digest.hexdigest()} + + +def scan_stream(stream, maximum=MAX_EXPANDED): + tail, size = b'', 0 + while chunk := stream.read(65536): + size += len(chunk) + if size > maximum: raise ValueError('Audit scan exceeded its expansion budget') + data = tail+chunk + if any(pattern in data for pattern in PRIVATE) or any(pattern.search(data) for pattern in SECRETS): + raise ValueError('Private path or credential-shaped content in public payload') + tail = data[-256:] + return size + + +def scan_file(path): + with Path(path).open('rb') as stream: scan_stream(stream, MAX_BLOB) + if path.suffix == '.gz': + with gzip.open(path, 'rb') as stream: scan_stream(stream) + elif path.suffix == '.npz': + with zipfile.ZipFile(path) as archive: + if sum(row.file_size for row in archive.infolist()) > MAX_EXPANDED: + raise ValueError('Numeric archive exceeds scan budget') + for row in archive.infolist(): + with archive.open(row) as stream: scan_stream(stream) + + +def safe_path(path): + if (type(path) is not str or not path or len(path) > 400 + or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]*', path) + or any(part in ('', '.', '..') for part in path.split('/'))): + raise ValueError('Unsafe logical evidence path') + return path + + +def check_frozen_evidence(root, audit): + """Require the audit's exact scientific roster and bytes, not fresh claims.""" + expected = audit.get('frozen_evidence') + if type(expected) is not dict or not expected: + raise ValueError('Missing audited frozen evidence inventory') + if root.is_symlink() or not root.is_dir(): + raise ValueError('Frozen evidence root must be a regular directory') + for name, record in expected.items(): + safe_path(name) + if (type(record) is not dict or set(record) != {'bytes', 'sha256'} + or type(record['bytes']) is not int or not 0 <= record['bytes'] <= MAX_BLOB + or type(record['sha256']) is not str + or not re.fullmatch(r'[0-9a-f]{64}', record['sha256'])): + raise ValueError('Malformed audited frozen file identity') + actual = set() + for source in root.rglob('*'): + if source.is_symlink(): raise ValueError('Frozen evidence contains a symlink') + if source.is_dir(): continue + if not source.is_file(): raise ValueError('Frozen evidence contains a nonregular file') + actual.add(source.relative_to(root).as_posix()) + if actual != set(expected) | {'readback-audit.json'}: + raise ValueError('Frozen evidence roster differs from the pinned audit') + for name, record in expected.items(): + if file_identity(root/name) != record: + raise ValueError('Frozen evidence differs from the pinned audit: '+name) + return {name: dict(record) for name, record in expected.items()} + + +def zip_entry(name, content=None, source=None): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + return info + + +def build_archive(output, files, generated, excluded, expected_identities=None): + """Build a deterministic archive; originals are rehashed while copying.""" + logical, blobs, scanned = [], {}, set() + for name, source in sorted(files.items()): + safe_path(name) + if source.is_symlink() or not source.is_file(): raise ValueError('Only regular evidence files are accepted') + identity = file_identity(source) + if name in (expected_identities or {}) and identity != expected_identities[name]: + raise ValueError('Frozen evidence changed before archive indexing: '+name) + if identity['sha256'] not in scanned: + scan_file(source) + scanned.add(identity['sha256']) + logical.append({'path': name, **identity, 'blob': 'blobs/'+identity['sha256']}) + blobs.setdefault(identity['sha256'], source) + for name, content in sorted(generated.items()): + safe_path(name) + if name in files: raise ValueError('Generated and original logical paths collide') + import io + scan_stream(io.BytesIO(content), MAX_BLOB) + digest = sha256(content).hexdigest() + logical.append({'path': name, 'bytes': len(content), 'sha256': digest, 'blob': 'blobs/'+digest}) + blobs.setdefault(digest, content) + logical.sort(key=lambda row: row['path']) + manifest = {'schema': SCHEMA, 'files': logical, 'excluded_originals': excluded, + 'layout': 'Original logical bytes reconstructed from SHA256 blobs; identical payloads share one blob.', + 'metadata_scope': 'manifest.json describes every logical file; it does not hash itself. Direct README/LICENSE/verifier copies match their logical inventory.', + 'license': 'Existing project LICENSE retained; upstream notices included. No robot asset meshes, simulator binaries or learned model weights bundled.'} + manifest_bytes = pretty(manifest) + with zipfile.ZipFile(output, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=9, allowZip64=True) as archive: + for name, content in [('README.md', generated['README.md']), ('LICENSE', generated['LICENSE']), + ('verify_extract.py', generated['tools/verify_extract.py']), ('manifest.json', manifest_bytes)]: + archive.writestr(zip_entry(name), content, compresslevel=9) + for digest, source in sorted(blobs.items()): + info = zip_entry('blobs/'+digest) + # Gzip/NPZ data are already compressed. Fixed STORE avoids needless + # second compression while preserving exact original bytes. + if isinstance(source, Path) and source.suffix in ('.gz', '.npz'): + info.compress_type = zipfile.ZIP_STORED + with archive.open(info, 'w', force_zip64=True) as sink: + checksum = sha256() + if isinstance(source, bytes): + sink.write(source); checksum.update(source) + else: + with source.open('rb') as stream: + while chunk := stream.read(1024*1024): sink.write(chunk); checksum.update(chunk) + if checksum.hexdigest() != digest: raise ValueError('Evidence changed while packaging') + return manifest + + +def prepare(repo, comparison, profile, output): + if output.exists() or output.is_symlink(): raise ValueError('Release output must be new') + audit_raw = (comparison/'readback-audit.json').read_bytes() + if sha256(audit_raw).hexdigest() != AUDIT_SHA: raise ValueError('Supplemental audit identity changed') + audit = json.loads(audit_raw) + original_raw = (comparison/'comparison.json').read_bytes() + original = json.loads(original_raw) + if (not audit['passed'] or not audit['evidence_unchanged'] + or not audit['eligible_under_predeclared_rule_after_complete_readback'] + or original['all_passed'] is not False + or file_identity(comparison/'comparison.json') != audit['original_report']): + raise ValueError('Original fail-closed report or supplemental audit differs') + frozen = check_frozen_evidence(comparison, audit) + expected_identities = {'comparison/'+name: record for name, record in frozen.items()} + expected_identities['comparison/readback-audit.json'] = {'bytes': len(audit_raw), 'sha256': AUDIT_SHA} + files, excluded = {}, [] + for prefix, root in (('comparison', comparison), ('profiling', profile)): + for source in sorted(root.rglob('*')): + if source.is_dir(): continue + name = prefix+'/'+source.relative_to(root).as_posix() + if source.is_symlink(): raise ValueError('Evidence may not contain symlinks') + if source.name == 'raw.pstats': + excluded.append({'path': name, **file_identity(source), + 'reason': 'Original raw profiler format contains local source paths. Portable stripped-directory stats and complete JSON tables are included.'}) + continue + if source.suffix == '.pyc' or '__pycache__' in source.parts: + excluded.append({'path': name, **file_identity(source), 'reason': 'Interpreter cache is not research evidence'}) + continue + try: + if source.suffix == '.txt' or 'traceback' in source.name: + scan_file(source) + except ValueError: + if source.suffix == '.txt' or 'traceback' in source.name: + excluded.append({'path': name, **file_identity(source), 'reason': 'Local log contains private paths or credential-shaped content'}) + continue + raise + files[name] = source + extras = { + 'tools/profile.py': repo/'benchmarks/piper-control-profile-v01/profile.py', + 'tools/compare.py': repo/'benchmarks/piper-control-profile-v01/compare.py', + 'tools/package_evidence.py': Path(__file__), + 'tools/README.md': repo/'benchmarks/piper-control-profile-v01/README.md', + 'tools/readback_audit.py': repo/'benchmarks/piper-control-profile-v01/readback_audit.py', + 'tests/test_profile.py': repo/'benchmarks/piper-control-profile-v01/test_profile.py', + 'tests/test_package_evidence.py': repo/'benchmarks/piper-control-profile-v01/test_package_evidence.py', + 'tests/test_piper_control_compare.py': repo/'tests/test_piper_control_compare.py', + 'tests/test_piper_contact_scratch.py': repo/'tests/test_piper_contact_scratch.py', + 'licenses/agilex_piper-LICENSE.txt': repo/'third_party/robot-models/agilex_piper-LICENSE.txt', + 'licenses/mujoco-LICENSE.txt': repo/'third_party/mujoco/LICENSE.txt', + 'licenses/mujoco-THIRD_PARTY_NOTICES.txt': repo/'third_party/mujoco/THIRD_PARTY_NOTICES.txt', + } + files.update(extras) + pooled = original['adoption']['pooled'] + summary = {'schema': 'dvidia.piper-control-profile-public-summary/v1', 'source_branch': 'feat/piper-control-profile-v01', + 'scope': original['scope'], 'timing_pairs': 2, 'planned_attempts': original['planned_attempts'], + 'timing': pooled, 'paired_timing': original['paired_timing'], + 'original_report': {'all_passed': False, **file_identity(comparison/'comparison.json'), + 'reason': 'Original dense reader rejected three case pairs exceeding its 64 MiB expansion bound.'}, + 'supplemental_readback': {'passed': True, **file_identity(comparison/'readback-audit.json'), + 'expanded_bound_bytes': audit['expanded_bound_bytes'], + 'total_dense_snapshots_compared': audit['total_dense_snapshots_compared'], + 'evidence_unchanged': True, + 'eligible_under_predeclared_rule_after_complete_readback': True}, + 'historical_learned_heldout_result': {'successes': 5, 'attempts': 12, 'unchanged': True}, + 'production_promotion': False, 'new_wheel': False, 'physical_robot_ready': False, + 'limitations': ['Two matched shared-host pairs on one fixed synthetic command tape.', + 'No learned inference in timed environment-step measurements.', + 'Dense diagnostics are separate from timing runs.', + 'No new held-out accuracy, physical calibration, recovery-robustness or real-time claim.', + 'Scheduled jaw pulse also succeeds; it is perturbation-equivalence coverage.']} + readme = b'''# DVIDIA | PiPER control profiling v0.1\n\nThis archive preserves one bounded allocation-only optimization experiment, its\noriginal fail-closed report, and a separate complete readback of the same evidence.\nIt contains synthetic simulator traces and development tools, not model weights.\n\nTwo matched AB/BA pairs observed 15.7% lower pooled mean environment-step wall\ncompute and 12.6% lower process CPU. These are shared-host observations on one\npreserved command tape, with no policy inference in the timed steps. They do not\nestablish general speedup, real-time operation or physical robot performance.\nThe historical learned held-out result remains 5/12. No new wheel or production\npromotion is part of this evidence release.\n\nAll 14 scheduled attempts are retained: four timing attempts plus five dense\nconditions for both source variants. The original comparison.json remains false\nbecause three dense JSON pairs exceeded its 64 MiB reader bound. The supplemental\nreadback-audit.json uses a 512 MiB bound and confirms exact numeric state, contacts,\ncadence and scorer equivalence across 200,010 saved snapshots, without rerunning\nphysics or timing. The fixed pulse also succeeds; this is not recovery validation.\n\nArchive layout is content-addressed: manifest.json maps every logical path to\nits exact byte count and SHA256 blob. Identical payloads share one blob. Original\nraw.pstats files are excluded because they contain local paths; portable stripped\nstats and complete JSON profiling tables remain. Any excluded logs are listed.\n\nCopy the top-level verify_extract.py from this archive after checking the release\narchive checksum. It uses only Python's standard library:\n\n```sh\npython verify_extract.py piper-control-profile-v01-evidence.zip\npython verify_extract.py piper-control-profile-v01-evidence.zip --output evidence\n```\n\nThe helper rejects traversal, symlinks, duplicate members, unknown blobs, oversized\ncontent and digest mismatches. It never imports archive code or reads pickle.\nReconstruction restores the original logical paths and bytes. Raw native NPZ\narrays and canonical gzip contact/scorer traces remain unchanged.\n\nSource branch: feat/piper-control-profile-v01. Frozen source snapshots, profiling\nand comparison harnesses, tests, the independent readback verifier and upstream\nnotices are included. The project's existing MIT LICENSE is retained. Robot asset\nmeshes, simulator binaries, credentials and learned model weights are omitted.\n\nDVIDIA research: https://research.dvidia.org | hello@dvidia.org\n''' + generated = {'README.md': readme, 'LICENSE': (repo/'LICENSE').read_bytes(), + 'tools/verify_extract.py': VERIFY_SOURCE.encode(), 'summary.json': pretty(summary)} + output.mkdir(parents=True, exist_ok=False) + stage = output/'huggingface' + stage.mkdir() + archive = stage/'piper-control-profile-v01-evidence.zip' + manifest = build_archive(archive, files, generated, excluded, expected_identities) + namespace = {'__name__': 'dvidia_evidence_verifier'} + exec(compile(VERIFY_SOURCE, 'verify_extract.py', 'exec'), namespace) + verified = namespace['verify'](archive) + check_frozen_evidence(comparison, audit) + if file_identity(comparison/'readback-audit.json') != expected_identities['comparison/readback-audit.json']: + raise ValueError('Pinned readback receipt changed while packaging') + summary['archive'] = {'filename': archive.name, **file_identity(archive, MAX_ARCHIVE), + 'logical_files': verified['logical_files'], 'unique_blobs': verified['unique_blobs'], + 'logical_bytes': verified['logical_bytes'], 'unique_blob_bytes': verified['unique_blob_bytes']} + card = ('''---\nlicense: mit\npretty_name: DVIDIA PiPER control profiling v0.1\ntask_categories:\n- robotics\ntags:\n- robotics\n- simulation\n- mujoco\n- piper\n- cpu\n- reproducibility\n---\n\n''').encode()+readme + payload = {'README.md': card, 'LICENSE': generated['LICENSE'], 'summary.json': pretty(summary), + 'comparison.json': original_raw, 'readback-audit.json': audit_raw} + for name, content in payload.items(): (stage/name).write_bytes(content) + if set(path.name for path in stage.iterdir()) != set(payload) | {archive.name}: + raise ValueError('Hugging Face staging must contain exactly six files') + for path in stage.iterdir(): + if path.suffix != '.zip': scan_file(path) + receipt = {'schema': 'dvidia.control-profile-evidence-package/v1', + 'target_dataset': 'Dvidia/piper-control-profile-v01', 'remote_operations': False, + 'archive': summary['archive'], 'verified': verified, + 'frozen_comparison_verified': {'files': len(frozen), 'roster_exact': True, + 'audit_sha256': AUDIT_SHA, + 'checked_before_indexing_during_copy_and_after_archive': True}, + 'scan': {'passed': True, 'scope': 'Raw blobs, decoded gzip JSON, numeric ZIP members and staged text; private local-path and credential-prefix checks.', + 'excluded_originals': excluded}, + 'staged_files': [{'path': path.name, **file_identity(path, MAX_ARCHIVE)} for path in sorted(stage.iterdir())]} + (output/'manifest.json').write_bytes(pretty(manifest)) + (output/'package-receipt.json').write_bytes(pretty(receipt)) + return receipt + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repo-root', type=Path, default=Path('.')) + parser.add_argument('--comparison', type=Path, required=True) + parser.add_argument('--profile', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args(argv) + receipt = prepare(args.repo_root, args.comparison, args.profile, args.output) + print(json.dumps(receipt['archive'], sort_keys=True)) + + +if __name__ == '__main__': main() diff --git a/benchmarks/piper-control-profile-v01/profile.py b/benchmarks/piper-control-profile-v01/profile.py new file mode 100644 index 0000000..d86843e --- /dev/null +++ b/benchmarks/piper-control-profile-v01/profile.py @@ -0,0 +1,293 @@ +"""Development-only profiling of one immutable, selected synthetic PiPER tape. + +Run from a checkout with PYTHONPATH=src. No runtime modules or global simulator +functions are patched. Baseline cProfile runs are diagnostic and have profiling +overhead; their timings must not be compared with an unprofiled speed headline. +The separate replay mode checks the immutable saved action tape. It is a fixed +scenario mechanics diagnostic, not a new skill-success benchmark or speed comparison. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from dataclasses import asdict +import gzip +from hashlib import sha256 +import io +import json +import math +from pathlib import Path +import platform +import pstats +import time + +SCHEMA = 'dvidia.piper-control-profile/v1' +MODEL_SHA = '2976a605575d9edefb5db366e72c5e006c0ce85a5531ad3d61512436864c8ded' +EPISODE_SHA = '7954c32ae4bc689f8947d8bc7786e028bf18dc1a02a100a7a38bdc779bf8193d' +COMPARISON_FIELDS = ('config', 'seed', 'valid', 'success', 'reason', 'error', 'simulated_seconds', + 'initial_object_position_m', 'compiled_model_sha256', 'model_sha256', + 'trajectory', 'actions', 'metrics') +SCOPE = ('One selected synthetic PiPER development scenario, fixed model/action tape and unchanged ' + '1 ms mechanics/contact/outcome checks. Privileged native state, no physical robot. ' + 'Not a held-out evaluation, real-time guarantee or a replacement for the frozen 5/12 benchmark.') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def identity(raw): + return {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} + + +def write(path, value): + with Path(path).open('xb') as stream: + stream.write(json.dumps(value, sort_keys=True, indent=2, allow_nan=False).encode()+b'\n') + + +def decode(raw): + def pairs(rows): + result = {} + for key, value in rows: + if key in result: + raise ValueError('Duplicate JSON field') + result[key] = value + return result + def nonfinite(token): + raise ValueError('Nonfinite JSON number: '+token) + document = json.loads(raw, object_pairs_hook=pairs, parse_constant=nonfinite) + canonical(document) + return document + + +def compare_episode(expected, actual): + """Exact finite JSON comparison; time measurements are deliberately excluded.""" + differences = [] + for key in COMPARISON_FIELDS: + if key not in expected or key not in actual or canonical(expected[key]) != canonical(actual[key]): + differences.append(key) + return {'exact_match': not differences, 'different_fields': differences, + 'fields_compared': list(COMPARISON_FIELDS), + 'comparison_scope': 'Every recorded 20 ms observation including contacts/forces/commands; exact outcomes and cumulative native contact ticks. Not a saved full 1 ms state tape.'} + + +def summary(values): + ordered = sorted(values) + if not ordered: + return {'count': 0} + def percentile(q): + rank = (len(ordered)-1)*q + low, high = math.floor(rank), math.ceil(rank) + return ordered[low]+(ordered[high]-ordered[low])*(rank-low) + return {'count': len(ordered), 'sum_s': sum(ordered), 'mean_s': sum(ordered)/len(ordered), + 'p50_s': percentile(.5), 'p95_s': percentile(.95), 'p99_s': percentile(.99), + 'max_s': ordered[-1], 'over_nominal_20ms': sum(value > .02 for value in ordered)} + + +def load_inputs(episode_path, model_path): + from dvidia_training.piper_imitation_benchmark import config, runtime_hashes + from dvidia_training.piper_imitation_score import score_episode + episode_raw, model_raw = Path(episode_path).read_bytes(), Path(model_path).read_bytes() + if len(episode_raw) > 8*1024*1024 or sha256(episode_raw).hexdigest() != EPISODE_SHA: + raise ValueError('Only the exact preserved selected episode is in this diagnostic protocol') + if len(model_raw) > 8*1024*1024 or sha256(model_raw).hexdigest() != MODEL_SHA: + raise ValueError('Only the exact pinned prior model is in this diagnostic protocol') + with gzip.GzipFile(fileobj=io.BytesIO(episode_raw)) as stream: + expanded = stream.read(32*1024*1024+1) + if len(expanded) > 32*1024*1024: + raise ValueError('Episode expands beyond the diagnostic budget') + episode = decode(expanded) + case = {'id': 'installed-known-goal1', 'start_xy': [.309, -.04], + 'goal_xy': [.40, -.08], 'seed': 261517} + if (episode.get('implementation_sha256') != runtime_hashes() + or episode.get('model_sha256') != MODEL_SHA + or episode.get('seed') != case['seed'] + or canonical(episode.get('config')) != canonical(asdict(config(case))) + or len(episode.get('trajectory', [])) != 1001 or len(episode.get('actions', [])) != 1000 + or not score_episode(episode, case['goal_xy'])['success']): + raise ValueError('Preserved episode, runtime, config or fixed score differs from the protocol') + return episode, case, {'episode': identity(episode_raw), 'model': identity(model_raw)} + + +def source_inventory(destination): + from dvidia_training import piper_imitation_benchmark as module + names = list(module.RUNTIME_FILES)+['env.py', 'piper_learned_skill.py'] + folder = destination/'sources' + folder.mkdir() + rows = [] + for name, path in [(name, Path(module.__file__).with_name(name)) for name in names]+[('profile.py', Path(__file__))]: + raw = path.read_bytes() + (folder/name).write_bytes(raw) + rows.append({'name': name, **identity(raw)}) + return rows + + +def check_sources(inventory): + from dvidia_training import piper_imitation_benchmark as module + for row in inventory: + path = Path(__file__) if row['name'] == 'profile.py' else Path(module.__file__).with_name(row['name']) + if sha256(path.read_bytes()).hexdigest() != row['sha256']: + raise ValueError('Profiling source changed during the diagnostic') + + +def profile_rows(profiler): + stats = pstats.Stats(profiler) + rows = [] + categories = defaultdict(list) + for (filename, line, name), (primitive, total, own, cumulative, _) in stats.stats.items(): + row = {'file': Path(filename).name, 'line': line, 'function': name, + 'primitive_calls': primitive, 'total_calls': total, + 'self_seconds': own, 'inclusive_seconds': cumulative} + rows.append(row) + category = None + if 'mj_step' in name: category = 'native_mj_step' + elif 'mj_forward' in name: category = 'native_mj_forward' + elif 'mj_contactForce' in name: category = 'native_contact_force' + elif 'mj_objectVelocity' in name: category = 'native_object_velocity' + elif name == '_contacts' and Path(filename).name == 'piper_task.py': category = 'contact_extraction_inclusive' + elif name == '_finite' and Path(filename).name in ('piper_task.py', 'reference_task.py'): category = 'finite_checks_inclusive' + elif name in ('observation', 'info') and Path(filename).name in ('piper_task.py', 'reference_task.py'): category = 'observation_and_info_inclusive' + elif name == 'predict' and Path(filename).name == 'piper_imitation.py': category = 'policy_inclusive' + elif name == 'step' and Path(filename).name == 'piper_task.py': category = 'environment_step_inclusive' + elif name == 'reset' and Path(filename).name == 'piper_task.py': category = 'reset_inclusive' + elif name == 'rollout' and Path(filename).name == 'piper_imitation_benchmark.py': category = 'rollout_orchestration_inclusive' + if category: categories[category].append(row) + return {'functions_by_self_time': sorted(rows, key=lambda row: row['self_seconds'], reverse=True), + 'functions_by_inclusive_time': sorted(rows, key=lambda row: row['inclusive_seconds'], reverse=True), + 'categories': dict(categories), + 'scope': 'Inclusive categories overlap and must not be added. cProfile self time is exclusive; rollout self includes setup/loop/recording bookkeeping, not recording alone.'} + + +def baseline(assets, model, case, expected, folder): + # This script is called profile.py. Hide its directory briefly so cProfile + # imports the standard-library profile module rather than this file again. + import sys + prior = list(sys.path) + try: + directory = Path(__file__).resolve().parent + sys.path[:] = [entry for entry in sys.path if Path(entry or '.').resolve() != directory] + import cProfile + finally: + sys.path[:] = prior + from dvidia_training.piper_imitation_benchmark import rollout, save_episode + from dvidia_training.piper_imitation_score import score_episode + results = [] + for timer_name, timer in (('wall', time.perf_counter_ns), ('cpu', time.process_time_ns)): + target = folder/('cprofile-'+timer_name) + target.mkdir() + write(target/'started.json', {'timer': timer_name, 'scope': SCOPE}) + profiler = cProfile.Profile(timer=timer, timeunit=1e-9) + wall, cpu = time.perf_counter(), time.process_time() + episode = profiler.runcall(rollout, assets, case, model) + wall, cpu = time.perf_counter()-wall, time.process_time()-cpu + profiler.dump_stats(str(target/'raw.pstats')) + # Keep original raw stats locally; this second form strips local paths. + pstats.Stats(profiler).strip_dirs().dump_stats(str(target/'portable.pstats')) + save_episode(target/'episode.json.gz', episode) + comparison = compare_episode(expected, episode) + result = {'timer': timer_name, 'wall_seconds': wall, 'process_cpu_seconds': cpu, + 'profiling_overhead': 'cProfile enabled throughout original rollout, including reset/model/asset verification. Not a performance headline.', + 'comparison': comparison, 'score': score_episode(episode, case['goal_xy']), + 'functions': profile_rows(profiler), 'runtime_reported_timing': { + key: value for key, value in episode['timing'].items() if not key.endswith('_samples_s')}} + write(target/'receipt.json', result) + results.append(result) + print(json.dumps({'phase': 'cprofile-'+timer_name, 'wall_seconds': wall, + 'process_cpu_seconds': cpu, 'exact_match': comparison['exact_match'], + 'score_success': result['score']['success']}), flush=True) + if not comparison['exact_match']: + raise ValueError('Profiled original rollout differs from the preserved tape; evidence retained') + return results + + +def replay(assets, expected): + """Unmodified environment, original tape, exact 20 ms records and outcomes.""" + from dvidia_training.piper_task import PiperTaskEnv, PiperTaskConfig + from dvidia_training import reference_sim as sim + env = PiperTaskEnv(assets, PiperTaskConfig(**expected['config'])) + started_wall, started_cpu = time.perf_counter(), time.process_time() + obs, info = env.reset(expected['seed']) + episode = {key: expected[key] for key in ('config', 'seed', 'model_sha256')} + episode.update(trajectory=[obs], actions=[], initial_object_position_m=env.initial_object_position.tolist(), + compiled_model_sha256=sim.model_digest(env.model), error=None) + step_wall, step_cpu = [], [] + for saved in expected['actions']: + action = {key: saved[key] for key in ('joint_targets_rad', 'gripper_position_m')} + record = {'t_seconds': obs['t_seconds'], **action} + episode['actions'].append(record) + obs.update(issued_joint_targets_rad=list(action['joint_targets_rad']), + issued_gripper_position_m=action['gripper_position_m'], command=record) + wall, cpu = time.perf_counter(), time.process_time() + obs, info = env.step(action) + step_wall.append(time.perf_counter()-wall) + step_cpu.append(time.process_time()-cpu) + episode['trajectory'].append(obs) + if env.done: break + episode.update(valid=info['valid'], success=info['success'], reason=info['reason'], + simulated_seconds=info['simulation_time'], + metrics={**info, 'contact_pair_native_ticks': env.contact_pair_native_ticks}) + return episode, {'wall_seconds': time.perf_counter()-started_wall, + 'process_cpu_seconds': time.process_time()-started_cpu, + 'environment_step_wall': summary(step_wall), 'environment_step_cpu': summary(step_cpu)} + + +def tape_replay(assets, model, case, expected, folder): + from dvidia_training.piper_imitation_benchmark import save_episode + from dvidia_training.piper_imitation_score import score_episode + write(folder/'started.json', {'variant': 'action_tape', 'scope': SCOPE}) + episode, timing = replay(assets, expected) + save_episode(folder/'episode.json.gz', episode) + result = {'variant': 'action_tape', 'timing': timing, + 'comparison': compare_episode(expected, episode), 'score': score_episode(episode, case['goal_xy']), + 'speed_scope': 'Single exact-tape replay, not a controlled speed comparison. No policy inference; setup/provenance work differs from original rollout. Do not compare against historical or cProfile timings.'} + write(folder/'replay-receipt.json', result) + print(json.dumps({'phase': 'action_tape', 'wall_seconds': timing['wall_seconds'], + 'exact_match': result['comparison']['exact_match'], 'score_success': result['score']['success']}), flush=True) + if not result['comparison']['exact_match']: + raise ValueError('Saved-action replay differs from the preserved tape; evidence retained') + return [result] + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--episode', type=Path, required=True) + parser.add_argument('--model', type=Path, required=True) + parser.add_argument('--assets', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--mode', choices=('baseline', 'replay'), default='baseline') + args = parser.parse_args(argv) + if args.output.exists() or args.output.is_symlink(): + parser.error('Output must be a new directory; all prior evidence is preserved') + expected, case, inputs = load_inputs(args.episode, args.model) + from dvidia_training import robot_profiles, reference_sim as sim + provenance = robot_profiles.verify_assets('piper', args.assets) + mj, np = sim._native_imports() + args.output.mkdir(parents=True, exist_ok=False) + inventory = source_inventory(args.output) + protocol = {'schema': SCHEMA, 'scope': SCOPE, 'mode': args.mode, 'inputs': inputs, 'case': case, + 'source_inventory': inventory, 'asset_provenance': provenance, + 'runtime': {'python': platform.python_version(), 'machine': platform.machine(), + 'system': platform.system(), 'mujoco': mj.__version__, 'numpy': np.__version__}, + 'roster': ['cprofile-wall', 'cprofile-cpu'] if args.mode == 'baseline' else ['action_tape'], + 'accuracy_freeze': 'Exact saved observation/action/outcome equality and unchanged scorer; timing fields excluded. No policy/physics changes or result-based retries.', + 'raw_profile_path_scope': 'raw.pstats may contain local source paths; portable.pstats and JSON function labels strip directories.'} + write(args.output/'protocol.json', protocol) + failure = None + started_wall, started_cpu = time.perf_counter(), time.process_time() + try: + results = (baseline if args.mode == 'baseline' else tape_replay)(args.assets, args.model, case, expected, args.output) + check_sources(inventory) + except Exception as exc: + failure = {'type': type(exc).__name__, 'message': str(exc)} + results = None + receipt = {'schema': SCHEMA, 'scope': SCOPE, 'mode': args.mode, 'error': failure, + 'complete': failure is None, 'wall_seconds': time.perf_counter()-started_wall, + 'process_cpu_seconds': time.process_time()-started_cpu, + 'results': results, 'physical_robot_ready': False} + write(args.output/'receipt.json', receipt) + if failure: print(json.dumps({'error': failure}), flush=True) + return 0 if failure is None else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/piper-control-profile-v01/readback-audit.json b/benchmarks/piper-control-profile-v01/readback-audit.json new file mode 100644 index 0000000..7c317b0 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/readback-audit.json @@ -0,0 +1,1570 @@ +{ + "cases": [ + { + "arrays": { + "all_finite": true, + "exact_bitwise_match": true, + "external_forces_zero": true, + "fields": [ + { + "dtype": "float64", + "name": "qpos", + "shape": [ + 20001, + 15 + ] + }, + { + "dtype": "float64", + "name": "qvel", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc_warmstart", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "ctrl", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "act", + "shape": [ + 20001, + 0 + ] + }, + { + "dtype": "float64", + "name": "actuator_force", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "qfrc_applied", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xfrc_applied", + "shape": [ + 20001, + 12, + 6 + ] + }, + { + "dtype": "float64", + "name": "qfrc_bias", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qfrc_constraint", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xpos", + "shape": [ + 20001, + 12, + 3 + ] + }, + { + "dtype": "float64", + "name": "xquat", + "shape": [ + 20001, + 12, + 4 + ] + }, + { + "dtype": "float64", + "name": "site_xpos", + "shape": [ + 20001, + 1, + 3 + ] + }, + { + "dtype": "float64", + "name": "site_xmat", + "shape": [ + 20001, + 1, + 9 + ] + }, + { + "dtype": "float64", + "name": "integration_state", + "shape": [ + 20001, + 138 + ] + }, + { + "dtype": "float64", + "name": "native_time", + "shape": [ + 20001 + ] + }, + { + "dtype": "float64", + "name": "command_targets", + "shape": [ + 20001, + 6 + ] + }, + { + "dtype": "float64", + "name": "command_gripper", + "shape": [ + 20001 + ] + } + ], + "native_clock_cadence_s": 0.001, + "rows": 20001 + }, + "canonical_contact_scorer_payload_identity": { + "bytes": 74862375, + "sha256": "a64ece5c4c88f88a0ff60d520f4c2957e99ec337183a319865db9d76cdf6ca4c" + }, + "case": "original", + "exact_boundaries_model_runtime_assets_score": true, + "score_success": true, + "scorer": { + "active_pairs_recomputed_from_raw_forces": true, + "contact_pair_native_histogram_exact": true, + "native_cadence_s": 0.001, + "rows": 20001, + "scorer_recomputed_exact": true, + "warnings": 0 + } + }, + { + "arrays": { + "all_finite": true, + "exact_bitwise_match": true, + "external_forces_zero": true, + "fields": [ + { + "dtype": "float64", + "name": "qpos", + "shape": [ + 20001, + 15 + ] + }, + { + "dtype": "float64", + "name": "qvel", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc_warmstart", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "ctrl", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "act", + "shape": [ + 20001, + 0 + ] + }, + { + "dtype": "float64", + "name": "actuator_force", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "qfrc_applied", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xfrc_applied", + "shape": [ + 20001, + 12, + 6 + ] + }, + { + "dtype": "float64", + "name": "qfrc_bias", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qfrc_constraint", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xpos", + "shape": [ + 20001, + 12, + 3 + ] + }, + { + "dtype": "float64", + "name": "xquat", + "shape": [ + 20001, + 12, + 4 + ] + }, + { + "dtype": "float64", + "name": "site_xpos", + "shape": [ + 20001, + 1, + 3 + ] + }, + { + "dtype": "float64", + "name": "site_xmat", + "shape": [ + 20001, + 1, + 9 + ] + }, + { + "dtype": "float64", + "name": "integration_state", + "shape": [ + 20001, + 138 + ] + }, + { + "dtype": "float64", + "name": "native_time", + "shape": [ + 20001 + ] + }, + { + "dtype": "float64", + "name": "command_targets", + "shape": [ + 20001, + 6 + ] + }, + { + "dtype": "float64", + "name": "command_gripper", + "shape": [ + 20001 + ] + } + ], + "native_clock_cadence_s": 0.001, + "rows": 20001 + }, + "canonical_contact_scorer_payload_identity": { + "bytes": 55051977, + "sha256": "1fd88f3b036782b9c0f15727eecb9e9eeb85077886b226d355eddc4648dc3462" + }, + "case": "open-jaw", + "exact_boundaries_model_runtime_assets_score": true, + "score_success": false, + "scorer": { + "active_pairs_recomputed_from_raw_forces": true, + "contact_pair_native_histogram_exact": true, + "native_cadence_s": 0.001, + "rows": 20001, + "scorer_recomputed_exact": true, + "warnings": 0 + } + }, + { + "arrays": { + "all_finite": true, + "exact_bitwise_match": true, + "external_forces_zero": true, + "fields": [ + { + "dtype": "float64", + "name": "qpos", + "shape": [ + 20001, + 15 + ] + }, + { + "dtype": "float64", + "name": "qvel", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc_warmstart", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "ctrl", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "act", + "shape": [ + 20001, + 0 + ] + }, + { + "dtype": "float64", + "name": "actuator_force", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "qfrc_applied", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xfrc_applied", + "shape": [ + 20001, + 12, + 6 + ] + }, + { + "dtype": "float64", + "name": "qfrc_bias", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qfrc_constraint", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xpos", + "shape": [ + 20001, + 12, + 3 + ] + }, + { + "dtype": "float64", + "name": "xquat", + "shape": [ + 20001, + 12, + 4 + ] + }, + { + "dtype": "float64", + "name": "site_xpos", + "shape": [ + 20001, + 1, + 3 + ] + }, + { + "dtype": "float64", + "name": "site_xmat", + "shape": [ + 20001, + 1, + 9 + ] + }, + { + "dtype": "float64", + "name": "integration_state", + "shape": [ + 20001, + 138 + ] + }, + { + "dtype": "float64", + "name": "native_time", + "shape": [ + 20001 + ] + }, + { + "dtype": "float64", + "name": "command_targets", + "shape": [ + 20001, + 6 + ] + }, + { + "dtype": "float64", + "name": "command_gripper", + "shape": [ + 20001 + ] + } + ], + "native_clock_cadence_s": 0.001, + "rows": 20001 + }, + "canonical_contact_scorer_payload_identity": { + "bytes": 55043502, + "sha256": "4123e3e2a0af753d9edaf33db082d86b000ae99c59810f67ca076e303761ef47" + }, + "case": "hold", + "exact_boundaries_model_runtime_assets_score": true, + "score_success": false, + "scorer": { + "active_pairs_recomputed_from_raw_forces": true, + "contact_pair_native_histogram_exact": true, + "native_cadence_s": 0.001, + "rows": 20001, + "scorer_recomputed_exact": true, + "warnings": 0 + } + }, + { + "arrays": { + "all_finite": true, + "exact_bitwise_match": true, + "external_forces_zero": true, + "fields": [ + { + "dtype": "float64", + "name": "qpos", + "shape": [ + 20001, + 15 + ] + }, + { + "dtype": "float64", + "name": "qvel", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc_warmstart", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "ctrl", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "act", + "shape": [ + 20001, + 0 + ] + }, + { + "dtype": "float64", + "name": "actuator_force", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "qfrc_applied", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xfrc_applied", + "shape": [ + 20001, + 12, + 6 + ] + }, + { + "dtype": "float64", + "name": "qfrc_bias", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qfrc_constraint", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xpos", + "shape": [ + 20001, + 12, + 3 + ] + }, + { + "dtype": "float64", + "name": "xquat", + "shape": [ + 20001, + 12, + 4 + ] + }, + { + "dtype": "float64", + "name": "site_xpos", + "shape": [ + 20001, + 1, + 3 + ] + }, + { + "dtype": "float64", + "name": "site_xmat", + "shape": [ + 20001, + 1, + 9 + ] + }, + { + "dtype": "float64", + "name": "integration_state", + "shape": [ + 20001, + 138 + ] + }, + { + "dtype": "float64", + "name": "native_time", + "shape": [ + 20001 + ] + }, + { + "dtype": "float64", + "name": "command_targets", + "shape": [ + 20001, + 6 + ] + }, + { + "dtype": "float64", + "name": "command_gripper", + "shape": [ + 20001 + ] + } + ], + "native_clock_cadence_s": 0.001, + "rows": 20001 + }, + "canonical_contact_scorer_payload_identity": { + "bytes": 74862375, + "sha256": "a64ece5c4c88f88a0ff60d520f4c2957e99ec337183a319865db9d76cdf6ca4c" + }, + "case": "feedback-no-pulse", + "exact_boundaries_model_runtime_assets_score": true, + "score_success": true, + "scorer": { + "active_pairs_recomputed_from_raw_forces": true, + "contact_pair_native_histogram_exact": true, + "native_cadence_s": 0.001, + "rows": 20001, + "scorer_recomputed_exact": true, + "warnings": 0 + } + }, + { + "arrays": { + "all_finite": true, + "exact_bitwise_match": true, + "external_forces_zero": true, + "fields": [ + { + "dtype": "float64", + "name": "qpos", + "shape": [ + 20001, + 15 + ] + }, + { + "dtype": "float64", + "name": "qvel", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qacc_warmstart", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "ctrl", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "act", + "shape": [ + 20001, + 0 + ] + }, + { + "dtype": "float64", + "name": "actuator_force", + "shape": [ + 20001, + 7 + ] + }, + { + "dtype": "float64", + "name": "qfrc_applied", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xfrc_applied", + "shape": [ + 20001, + 12, + 6 + ] + }, + { + "dtype": "float64", + "name": "qfrc_bias", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "qfrc_constraint", + "shape": [ + 20001, + 14 + ] + }, + { + "dtype": "float64", + "name": "xpos", + "shape": [ + 20001, + 12, + 3 + ] + }, + { + "dtype": "float64", + "name": "xquat", + "shape": [ + 20001, + 12, + 4 + ] + }, + { + "dtype": "float64", + "name": "site_xpos", + "shape": [ + 20001, + 1, + 3 + ] + }, + { + "dtype": "float64", + "name": "site_xmat", + "shape": [ + 20001, + 1, + 9 + ] + }, + { + "dtype": "float64", + "name": "integration_state", + "shape": [ + 20001, + 138 + ] + }, + { + "dtype": "float64", + "name": "native_time", + "shape": [ + 20001 + ] + }, + { + "dtype": "float64", + "name": "command_targets", + "shape": [ + 20001, + 6 + ] + }, + { + "dtype": "float64", + "name": "command_gripper", + "shape": [ + 20001 + ] + } + ], + "native_clock_cadence_s": 0.001, + "rows": 20001 + }, + "canonical_contact_scorer_payload_identity": { + "bytes": 74847125, + "sha256": "bb084f8431a2ebcec7a7926b2f640953f5faba71e3c0e92addcb3a951239ff0f" + }, + "case": "feedback-pulse", + "exact_boundaries_model_runtime_assets_score": true, + "score_success": true, + "scorer": { + "active_pairs_recomputed_from_raw_forces": true, + "contact_pair_native_histogram_exact": true, + "native_cadence_s": 0.001, + "rows": 20001, + "scorer_recomputed_exact": true, + "warnings": 0 + } + } + ], + "eligible_under_predeclared_rule_after_complete_readback": true, + "evidence_unchanged": true, + "expanded_bound_bytes": 536870912, + "frozen_evidence": { + "compare.py": { + "bytes": 34513, + "sha256": "f458adeaa84963e492a7e659bd91cce810ec3e1582eebbbe43e52219f926a05a" + }, + "comparison.json": { + "bytes": 17508, + "sha256": "11351eebac7d9d615f9c34db3a8fa72133ddbc1d8b7186e501ee9dfe0d3ff934" + }, + "dense-feedback-no-pulse-baseline/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "dense-feedback-no-pulse-baseline/native-arrays.npz": { + "bytes": 27851918, + "sha256": "bc7773ea416b91ae3c011fa50282a414269d902fb8959c29c4c3f1fd6a0d7e26" + }, + "dense-feedback-no-pulse-baseline/native-contacts-scorer.json.gz": { + "bytes": 8970868, + "sha256": "161986418c1a602647fad6af1ec7d911a147fbf2a39f3a61eca2a5b17eb4de09" + }, + "dense-feedback-no-pulse-baseline/process.json": { + "bytes": 186, + "sha256": "a740d8b3177c209e349353d2f3a47915b8a7cb5c4188228e18dcdadcf2a11456" + }, + "dense-feedback-no-pulse-baseline/result.json": { + "bytes": 63943, + "sha256": "2154c0ea1c342181b30ba1503d4b1d714f9d256d062d5e049a20230ba6e99f17" + }, + "dense-feedback-no-pulse-baseline/started.json": { + "bytes": 216, + "sha256": "fd62c80043036b256f6cc5aa1e52a07bbcee913709364fbf6de4875bd7c9c2f0" + }, + "dense-feedback-no-pulse-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-no-pulse-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-no-pulse-candidate/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "dense-feedback-no-pulse-candidate/native-arrays.npz": { + "bytes": 27851918, + "sha256": "bc7773ea416b91ae3c011fa50282a414269d902fb8959c29c4c3f1fd6a0d7e26" + }, + "dense-feedback-no-pulse-candidate/native-contacts-scorer.json.gz": { + "bytes": 8970868, + "sha256": "161986418c1a602647fad6af1ec7d911a147fbf2a39f3a61eca2a5b17eb4de09" + }, + "dense-feedback-no-pulse-candidate/process.json": { + "bytes": 190, + "sha256": "9f60bbf36c2519d0d31b8fd6818e2b4393bb8dc4afb336c520a2125d396a7e57" + }, + "dense-feedback-no-pulse-candidate/result.json": { + "bytes": 63852, + "sha256": "70d021a0e655fa6b0bb37fef617785a8561a5776b831d325fbbff87f1a80a4e7" + }, + "dense-feedback-no-pulse-candidate/started.json": { + "bytes": 218, + "sha256": "305d77617c7f4c4df96f2ccde1d9e3afe3a558eb241d149df9268d89853e950d" + }, + "dense-feedback-no-pulse-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-no-pulse-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-pulse-baseline/episode.json.gz": { + "bytes": 670321, + "sha256": "2bf8704bb4dd563187ac8f1b82a11675b458c2b4ce98ef7f5d03a0455dbc7bfb" + }, + "dense-feedback-pulse-baseline/native-arrays.npz": { + "bytes": 27862679, + "sha256": "860bb16fbccf0bc364e6ef8d527215fab3921629d197449c3a3d538ded39417c" + }, + "dense-feedback-pulse-baseline/native-contacts-scorer.json.gz": { + "bytes": 8976230, + "sha256": "e7c26f56e61ca777c76d6b1a53f9a57eb350eb52abbcd8d9481978a0861e4f2f" + }, + "dense-feedback-pulse-baseline/process.json": { + "bytes": 182, + "sha256": "69c843f1ce7a3591ebedc1c014448080884b59e516421b6dff2f57cfb85c54bf" + }, + "dense-feedback-pulse-baseline/result.json": { + "bytes": 63896, + "sha256": "8a7133becac77902a4963e80f4aad974f5370765b314dabb8feea4eb1b686c2b" + }, + "dense-feedback-pulse-baseline/started.json": { + "bytes": 210, + "sha256": "8ff74e3deb8f3211618fb1caa90e3891bb16fe7e249d9ccc924a0635af8798aa" + }, + "dense-feedback-pulse-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-pulse-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-pulse-candidate/episode.json.gz": { + "bytes": 670321, + "sha256": "2bf8704bb4dd563187ac8f1b82a11675b458c2b4ce98ef7f5d03a0455dbc7bfb" + }, + "dense-feedback-pulse-candidate/native-arrays.npz": { + "bytes": 27862679, + "sha256": "860bb16fbccf0bc364e6ef8d527215fab3921629d197449c3a3d538ded39417c" + }, + "dense-feedback-pulse-candidate/native-contacts-scorer.json.gz": { + "bytes": 8976230, + "sha256": "e7c26f56e61ca777c76d6b1a53f9a57eb350eb52abbcd8d9481978a0861e4f2f" + }, + "dense-feedback-pulse-candidate/process.json": { + "bytes": 183, + "sha256": "8d976c3877db798e95502f2c596f4760d2fe0d295d1942101b599735c29604a0" + }, + "dense-feedback-pulse-candidate/result.json": { + "bytes": 63793, + "sha256": "e3e9267d565110c20632a3399b146216cf984e08127218a2a59ff6237370048c" + }, + "dense-feedback-pulse-candidate/started.json": { + "bytes": 212, + "sha256": "28e921b5f30938bc5a9c6d66b3adb82cb01bdb7c4c8802fd960b17734ad24d7a" + }, + "dense-feedback-pulse-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-feedback-pulse-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-hold-baseline/episode.json.gz": { + "bytes": 334438, + "sha256": "33c461230e02b07c8295a4993abab9c4173fb13481349c0b9cff275416ab25fd" + }, + "dense-hold-baseline/native-arrays.npz": { + "bytes": 12242175, + "sha256": "50637dc812283c5f55f91677f271142f70433b6c09e3104a6ae9a6ba62e81164" + }, + "dense-hold-baseline/native-contacts-scorer.json.gz": { + "bytes": 4481056, + "sha256": "e57a5c1f82593a5e538219cb91d2b44b5b9c1d4d76389b675ef527be51175953" + }, + "dense-hold-baseline/process.json": { + "bytes": 162, + "sha256": "c2b667b5b48937ed52cd833b44bcafd547f1c056e18d552d96185cfbcf6c2c28" + }, + "dense-hold-baseline/result.json": { + "bytes": 64112, + "sha256": "057c2f83cb9079b9cc1aa2fed49a5a16b6dec76eddd3a96035a4915a21db6438" + }, + "dense-hold-baseline/started.json": { + "bytes": 190, + "sha256": "0d4d0f3da55035128ecfc681f120a7f9aada99f825b2c6e73a255a2131e58da4" + }, + "dense-hold-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-hold-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-hold-candidate/episode.json.gz": { + "bytes": 334438, + "sha256": "33c461230e02b07c8295a4993abab9c4173fb13481349c0b9cff275416ab25fd" + }, + "dense-hold-candidate/native-arrays.npz": { + "bytes": 12242175, + "sha256": "50637dc812283c5f55f91677f271142f70433b6c09e3104a6ae9a6ba62e81164" + }, + "dense-hold-candidate/native-contacts-scorer.json.gz": { + "bytes": 4481056, + "sha256": "e57a5c1f82593a5e538219cb91d2b44b5b9c1d4d76389b675ef527be51175953" + }, + "dense-hold-candidate/process.json": { + "bytes": 164, + "sha256": "e905a713178fa9b9ea405e8689a61cde255ebd6f471aa7151ac2dc70672dbb68" + }, + "dense-hold-candidate/result.json": { + "bytes": 64208, + "sha256": "0700d3660da53aa2a4486f0f4ec25bdfc2e3730f16bc2606a1674b865134b20b" + }, + "dense-hold-candidate/started.json": { + "bytes": 192, + "sha256": "f25b2812ac4edc8df25f24b20a34afb5b1b4ba6ef615444ff4f1f0a034b47694" + }, + "dense-hold-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-hold-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-open-jaw-baseline/episode.json.gz": { + "bytes": 619195, + "sha256": "fc2c870d282d4e3c81343fe698e005519bdbac80d893249aa8ee249dddc54a76" + }, + "dense-open-jaw-baseline/native-arrays.npz": { + "bytes": 27358310, + "sha256": "25bf28eff07dae99d3447a39c30271e852482cd9c70bbdea945535cb915c13c6" + }, + "dense-open-jaw-baseline/native-contacts-scorer.json.gz": { + "bytes": 4492364, + "sha256": "aa13633390394459c4ba5d3d0fdbd2c1b2c8061c4a80fc41f2a671c4d8d57eea" + }, + "dense-open-jaw-baseline/process.json": { + "bytes": 171, + "sha256": "51f218da115bc5b6e66662677868d1b43e770063dbf0793fbe47fce0ded4a487" + }, + "dense-open-jaw-baseline/result.json": { + "bytes": 64173, + "sha256": "737ad6b282214fd1281d23a7cb6de2fc65dde73c6007d85a2da125a66fae052b" + }, + "dense-open-jaw-baseline/started.json": { + "bytes": 198, + "sha256": "41a39a0457d71b4829a51b240da0723ffd17252e9ceaff3a3347351ed9b5bb15" + }, + "dense-open-jaw-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-open-jaw-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-open-jaw-candidate/episode.json.gz": { + "bytes": 619195, + "sha256": "fc2c870d282d4e3c81343fe698e005519bdbac80d893249aa8ee249dddc54a76" + }, + "dense-open-jaw-candidate/native-arrays.npz": { + "bytes": 27358310, + "sha256": "25bf28eff07dae99d3447a39c30271e852482cd9c70bbdea945535cb915c13c6" + }, + "dense-open-jaw-candidate/native-contacts-scorer.json.gz": { + "bytes": 4492364, + "sha256": "aa13633390394459c4ba5d3d0fdbd2c1b2c8061c4a80fc41f2a671c4d8d57eea" + }, + "dense-open-jaw-candidate/process.json": { + "bytes": 172, + "sha256": "878641d6240e69b31e533bfbfafa080e91b4fbaca379aed1a4eaaecbc5bc73f8" + }, + "dense-open-jaw-candidate/result.json": { + "bytes": 64088, + "sha256": "72b046b99947b652389ce74b677aa08761a6a31eae95d172f9f333039d03e153" + }, + "dense-open-jaw-candidate/started.json": { + "bytes": 200, + "sha256": "175a8d9ce0c01accb74af89e0a1424f5b1742c28e17355a41ca16c1b365923f3" + }, + "dense-open-jaw-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-open-jaw-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-original-baseline/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "dense-original-baseline/native-arrays.npz": { + "bytes": 27851918, + "sha256": "bc7773ea416b91ae3c011fa50282a414269d902fb8959c29c4c3f1fd6a0d7e26" + }, + "dense-original-baseline/native-contacts-scorer.json.gz": { + "bytes": 8970868, + "sha256": "161986418c1a602647fad6af1ec7d911a147fbf2a39f3a61eca2a5b17eb4de09" + }, + "dense-original-baseline/process.json": { + "bytes": 170, + "sha256": "e30dc1226edf7600a935c693d1a21240977013841b145814f20976f45b87b708" + }, + "dense-original-baseline/result.json": { + "bytes": 63992, + "sha256": "ca72cd30c47939891cc0e59cacbf84bf6672dfbbb80974cbedc9d2129f6d8046" + }, + "dense-original-baseline/started.json": { + "bytes": 198, + "sha256": "439e7d9f39b7a378d6c72bfd742d7bca22969ee1d43e729a4433dc85e2678db0" + }, + "dense-original-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-original-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-original-candidate/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "dense-original-candidate/native-arrays.npz": { + "bytes": 27851918, + "sha256": "bc7773ea416b91ae3c011fa50282a414269d902fb8959c29c4c3f1fd6a0d7e26" + }, + "dense-original-candidate/native-contacts-scorer.json.gz": { + "bytes": 8970868, + "sha256": "161986418c1a602647fad6af1ec7d911a147fbf2a39f3a61eca2a5b17eb4de09" + }, + "dense-original-candidate/process.json": { + "bytes": 172, + "sha256": "ee1be93655bafed9803e9df45806dcbdcb175da3d5f98c9711b766b02fcdd7d2" + }, + "dense-original-candidate/result.json": { + "bytes": 64039, + "sha256": "49f9a665d77d43dc529d42c2db42a8b48966e398b813dd637d2b56c79861f4df" + }, + "dense-original-candidate/started.json": { + "bytes": 200, + "sha256": "719e4c9e69fb17f9d2b02212ffedbfe3f465332b4fc2f52ac9139f0fc530281a" + }, + "dense-original-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dense-original-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "preserved-episode.json.gz": { + "bytes": 695222, + "sha256": "7954c32ae4bc689f8947d8bc7786e028bf18dc1a02a100a7a38bdc779bf8193d" + }, + "protocol.json": { + "bytes": 6716, + "sha256": "386c34e8201a86dbe81709805b7c0d47ec4c257120023514153cc4df652f5184" + }, + "readback-audit.py": { + "bytes": 14619, + "sha256": "874bf579d302219bdda7f30112e7cccc57f9aa2e7589b37b77067a3eb729ffe4" + }, + "schedule.json": { + "bytes": 1733, + "sha256": "47fe764d9edda5aec607c4bfddff09e3f3bd466ad24ca0ac5abc558097ade88d" + }, + "sources/baseline/__init__.py": { + "bytes": 106, + "sha256": "4c0db7cec4588f99f061f7783bbfba210e246205654a1dcb6c8c03c252254dcb" + }, + "sources/baseline/env.py": { + "bytes": 26684, + "sha256": "929c1d8c3c4cafe0054c633ee5f3e51c551dd6b26ae014ab1ea9219b58eca8e4" + }, + "sources/baseline/piper_feedback_env.py": { + "bytes": 11247, + "sha256": "0dbae423231968a81bdeed52226d8dffaf8df622013ed3a360d7217148f1bcac" + }, + "sources/baseline/piper_imitation_score.py": { + "bytes": 11462, + "sha256": "0cd4663abfdd7f14f2f33d93e9a2bd45122b998d6160d45b8afcc880512cba2b" + }, + "sources/baseline/piper_task.py": { + "bytes": 34355, + "sha256": "e66a44a8df4e7a8ee42e4f3866f889f8797cdfb8e2688a146e09b75d2c76664f" + }, + "sources/baseline/reference_assets.lock.json": { + "bytes": 10029, + "sha256": "941eb6f9fd474635a590b82c66b59a9d5124d8e0d9ddc203006f5b921d8d2861" + }, + "sources/baseline/reference_sim.py": { + "bytes": 24336, + "sha256": "c1201d22eb13dd07a236ee63cc4ba68e129f6067e8db61ed41af2d820eb5e3b8" + }, + "sources/baseline/reference_task.py": { + "bytes": 34166, + "sha256": "bbc768ccc202b8dd41cbd235b206e6478806ce47c1f30ae15dc662c0c1edcc3e" + }, + "sources/baseline/robot_profiles.lock.json": { + "bytes": 35002, + "sha256": "4b2ee2ca675aceb2f826983cb3082a3e08e9aa0ff419b23078abcff08ebcc6fe" + }, + "sources/baseline/robot_profiles.py": { + "bytes": 26493, + "sha256": "fb90782a6a9979e282182118ae0e3e29d00f101255cc57378dd3f692672bb13d" + }, + "sources/candidate/__init__.py": { + "bytes": 106, + "sha256": "4c0db7cec4588f99f061f7783bbfba210e246205654a1dcb6c8c03c252254dcb" + }, + "sources/candidate/env.py": { + "bytes": 26684, + "sha256": "929c1d8c3c4cafe0054c633ee5f3e51c551dd6b26ae014ab1ea9219b58eca8e4" + }, + "sources/candidate/piper_feedback_env.py": { + "bytes": 11247, + "sha256": "0dbae423231968a81bdeed52226d8dffaf8df622013ed3a360d7217148f1bcac" + }, + "sources/candidate/piper_imitation_score.py": { + "bytes": 11462, + "sha256": "0cd4663abfdd7f14f2f33d93e9a2bd45122b998d6160d45b8afcc880512cba2b" + }, + "sources/candidate/piper_task.py": { + "bytes": 34612, + "sha256": "59dcb645197fb5d7f9e1c200e10cfe937ae26153648c1a2e57b5ff89de6874ed" + }, + "sources/candidate/reference_assets.lock.json": { + "bytes": 10029, + "sha256": "941eb6f9fd474635a590b82c66b59a9d5124d8e0d9ddc203006f5b921d8d2861" + }, + "sources/candidate/reference_sim.py": { + "bytes": 24336, + "sha256": "c1201d22eb13dd07a236ee63cc4ba68e129f6067e8db61ed41af2d820eb5e3b8" + }, + "sources/candidate/reference_task.py": { + "bytes": 34166, + "sha256": "bbc768ccc202b8dd41cbd235b206e6478806ce47c1f30ae15dc662c0c1edcc3e" + }, + "sources/candidate/robot_profiles.lock.json": { + "bytes": 35002, + "sha256": "4b2ee2ca675aceb2f826983cb3082a3e08e9aa0ff419b23078abcff08ebcc6fe" + }, + "sources/candidate/robot_profiles.py": { + "bytes": 26493, + "sha256": "fb90782a6a9979e282182118ae0e3e29d00f101255cc57378dd3f692672bb13d" + }, + "timing-00-baseline/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "timing-00-baseline/process.json": { + "bytes": 161, + "sha256": "3dfedf4804d1b9a63463b43e000a413ea9db1ea5b7bf2f366400e6fba4649e06" + }, + "timing-00-baseline/result.json": { + "bytes": 63782, + "sha256": "7308e30925c2ebfdef25dd1c6c088301eb70604c07eb387083e939c725f539fa" + }, + "timing-00-baseline/started.json": { + "bytes": 189, + "sha256": "621d0aa629bcaf12000c769e7fd59db2862407771682b3834c9f3c5cb5845208" + }, + "timing-00-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-00-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-01-candidate/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "timing-01-candidate/process.json": { + "bytes": 164, + "sha256": "0fa27234568a4e66bea65e08fd061a0f08e1a7b9ae19cd833fe28b97652b3c62" + }, + "timing-01-candidate/result.json": { + "bytes": 63814, + "sha256": "43284756f69fd8041b189e5dc366c22dc266a37445f1b1d3d4e3928e0dba3120" + }, + "timing-01-candidate/started.json": { + "bytes": 191, + "sha256": "94631c9e0bc2b1fa4dd8adb9f7d2b73a10aa588b466f5fb9f5d8f097e5d6907a" + }, + "timing-01-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-01-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-02-candidate/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "timing-02-candidate/process.json": { + "bytes": 164, + "sha256": "07c1a4e7993b63076a701dc0e2918905c57c902b7cff967a3ab4582d0ab94bc9" + }, + "timing-02-candidate/result.json": { + "bytes": 63817, + "sha256": "bbbce2591cf2cff0bd41275fec7057ee16f052eea54bab26094316189eb947d2" + }, + "timing-02-candidate/started.json": { + "bytes": 191, + "sha256": "9c17b1516d051bcea618ad6ae5a2273c8b608f4891c78e7ec040a9abfba48f3e" + }, + "timing-02-candidate/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-02-candidate/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-03-baseline/episode.json.gz": { + "bytes": 670482, + "sha256": "a062d71e034c07f7a6341d0f842c97a40c4d8282cd9aa8aa6d61ba3afe32ab06" + }, + "timing-03-baseline/process.json": { + "bytes": 162, + "sha256": "c33e39175e6da5fcf51c9961bd3f3a06c9a9a927528612256c00f58afdffaa8f" + }, + "timing-03-baseline/result.json": { + "bytes": 63728, + "sha256": "cd87d6b47b232ed1e1a2185493e7d201b9a00f39ecf8167ee52c95ff9e26833d" + }, + "timing-03-baseline/started.json": { + "bytes": 189, + "sha256": "90aa35bef7d92d4c4afcff7d276ff30b05f732947033cc239c5967d60dc5df01" + }, + "timing-03-baseline/stderr.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "timing-03-baseline/stdout.txt": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "interpretation": "Observed two-pair shared-host CPU improvement with exact saved mechanics. Not a general latency guarantee, accuracy gain, hardware test, or recovery-robustness evaluation; the fixed pulse also succeeds.", + "original_report": { + "bytes": 17508, + "sha256": "11351eebac7d9d615f9c34db3a8fa72133ddbc1d8b7186e501ee9dfe0d3ff934" + }, + "passed": true, + "protocol": { + "bytes": 6716, + "sha256": "386c34e8201a86dbe81709805b7c0d47ec4c257120023514153cc4df652f5184" + }, + "schema": "dvidia.piper-control-readback-audit/v1", + "scope": "Read-only reanalysis of the same 14 frozen attempts, no simulation/timing reruns. Original fail-closed 64 MiB reader report is preserved.", + "script": { + "bytes": 14619, + "sha256": "874bf579d302219bdda7f30112e7cccc57f9aa2e7589b37b77067a3eb729ffe4" + }, + "self_tests": { + "checks": 9, + "passed": true + }, + "timing_adoption_rule": { + "minimum_pooled_mean_reduction_percent_each_clock": 5.0, + "on_failure": "Do not promote; retain as negative or inconclusive. No post-result tuning.", + "required_exact_equivalence": true, + "required_lower_candidate_mean_each_pair": [ + "step_wall", + "step_cpu" + ] + }, + "timing_runs": [ + { + "id": "timing-00-baseline", + "setup_reset_wall_s": 0.367916792165488, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.012059000000000042, + "mean_s": 0.0014127509999999999, + "over_nominal_20ms": 0, + "p50_s": 0.0011879999999999669, + "p95_s": 0.002274649999999988, + "p99_s": 0.002799599999999997, + "sum_s": 1.4127509999999999 + }, + "step_wall": { + "count": 1000, + "max_s": 0.012310583610087633, + "mean_s": 0.0014684963417239487, + "over_nominal_20ms": 0, + "p50_s": 0.0012011665385216475, + "p95_s": 0.002542358636856079, + "p99_s": 0.003286311766132712, + "sum_s": 1.4684963417239487 + }, + "variant": "baseline" + }, + { + "id": "timing-01-candidate", + "setup_reset_wall_s": 0.28948104102164507, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.008727999999999847, + "mean_s": 0.0012205139999999996, + "over_nominal_20ms": 0, + "p50_s": 0.0010654999999999415, + "p95_s": 0.0018321999999998393, + "p99_s": 0.0024473500000000855, + "sum_s": 1.2205139999999997 + }, + "step_wall": { + "count": 1000, + "max_s": 0.010767832864075899, + "mean_s": 0.0012328931200318038, + "over_nominal_20ms": 0, + "p50_s": 0.0010669578332453966, + "p95_s": 0.001886604120954871, + "p99_s": 0.0025030286889523266, + "sum_s": 1.2328931200318038 + }, + "variant": "candidate" + }, + { + "id": "timing-02-candidate", + "setup_reset_wall_s": 0.25960616674274206, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.00799099999999986, + "mean_s": 0.0012937500000000002, + "over_nominal_20ms": 0, + "p50_s": 0.001134999999999775, + "p95_s": 0.001920649999999912, + "p99_s": 0.0027079999999999327, + "sum_s": 1.2937500000000002 + }, + "step_wall": { + "count": 1000, + "max_s": 0.008000124711543322, + "mean_s": 0.0013053270056843757, + "over_nominal_20ms": 0, + "p50_s": 0.0011370624415576458, + "p95_s": 0.0019861731911078096, + "p99_s": 0.00277163659222424, + "sum_s": 1.3053270056843758 + }, + "variant": "candidate" + }, + { + "id": "timing-03-baseline", + "setup_reset_wall_s": 0.3268076251260936, + "status": "complete", + "step_cpu": { + "count": 1000, + "max_s": 0.018218000000000067, + "mean_s": 0.001463971000000003, + "over_nominal_20ms": 0, + "p50_s": 0.0012210000000000276, + "p95_s": 0.002330100000000157, + "p99_s": 0.0029761600000003073, + "sum_s": 1.463971000000003 + }, + "step_wall": { + "count": 1000, + "max_s": 0.02005712501704693, + "mean_s": 0.0015423414260149002, + "over_nominal_20ms": 1, + "p50_s": 0.001237792195752263, + "p95_s": 0.0026063663186505436, + "p99_s": 0.003594016665592789, + "sum_s": 1.5423414260149002 + }, + "variant": "baseline" + } + ], + "total_dense_snapshots_compared": 200010 +} diff --git a/benchmarks/piper-control-profile-v01/readback_audit.py b/benchmarks/piper-control-profile-v01/readback_audit.py new file mode 100644 index 0000000..6eca187 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/readback_audit.py @@ -0,0 +1,307 @@ +"""Independent, bounded readback of already frozen evidence; no native imports. + +This does not rerun a simulation, modify a source, replace the original failed +64 MiB reader report, or change the predeclared adoption rule. It streams the +same canonical contact/scorer JSON with a 512 MiB expanded bound and checks +saved arrays, cadence, active forces, contact histograms and cumulative scorer. +""" +from __future__ import annotations +import argparse +import gzip +from hashlib import sha256 +import io +import json +import math +from pathlib import Path +import tempfile + +MAX_BYTES = 512*1024*1024 +CASES = ('original', 'open-jaw', 'hold', 'feedback-no-pulse', 'feedback-pulse') +EPISODE_FIELDS = ('config', 'seed', 'valid', 'success', 'reason', 'error', 'simulated_seconds', + 'initial_object_position_m', 'compiled_model_sha256', 'trajectory', 'actions', 'metrics') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def unique(rows): + result = {} + for key, value in rows: + if key in result: + raise ValueError('Duplicate JSON key') + result[key] = value + return result + + +def nonfinite(token): + raise ValueError('Nonfinite JSON token: '+token) + + +DECODER = json.JSONDecoder(object_pairs_hook=unique, parse_constant=nonfinite) + + +def read(path): + return json.loads(Path(path).read_bytes(), object_pairs_hook=unique, parse_constant=nonfinite) + + +def file_id(path): + digest, size = sha256(), 0 + with Path(path).open('rb') as stream: + while chunk := stream.read(1024*1024): + digest.update(chunk) + size += len(chunk) + return {'bytes': size, 'sha256': digest.hexdigest()} + + +def expanded_id(path): + digest, size = sha256(), 0 + with gzip.open(path, 'rb') as stream: + while chunk := stream.read(65536): + size += len(chunk) + if size > MAX_BYTES: + raise ValueError('Expanded evidence exceeds 512 MiB') + digest.update(chunk) + return {'bytes': size, 'sha256': digest.hexdigest()} + + +def rows(path, chunk_size=65536): + """Incrementally decode one bounded top-level JSON array without loading it.""" + with gzip.open(path, 'rt', encoding='utf-8', errors='strict') as stream: + buffer, total, ended = '', 0, False + + def fill(): + nonlocal buffer, total, ended + chunk = stream.read(chunk_size) + if not chunk: + ended = True + return False + total += len(chunk.encode('utf-8')) + if total > MAX_BYTES: + raise ValueError('Expanded evidence exceeds 512 MiB') + buffer += chunk + return True + + def available(): + nonlocal buffer + buffer = buffer.lstrip() + while not buffer and not ended: + fill() + buffer = buffer.lstrip() + if not buffer: + raise ValueError('Truncated JSON array') + + available() + if buffer[0] != '[': + raise ValueError('Expected top-level JSON array') + buffer = buffer[1:] + may_end = True + while True: + available() + if buffer[0] == ']': + if not may_end: + raise ValueError('Trailing array comma') + buffer = buffer[1:] + while not ended: + fill() + if buffer.strip(): + raise ValueError('Data after JSON array') + return + while True: + try: + item, offset = DECODER.raw_decode(buffer) + break + except json.JSONDecodeError: + if len(buffer) > 1024*1024 or not fill(): + raise ValueError('Malformed or oversized row') from None + if type(item) is not dict: + raise ValueError('Expected object row') + canonical(item) + buffer = buffer[offset:] + yield item + available() + if buffer[0] == ',': + buffer = buffer[1:] + may_end = False + elif buffer[0] == ']': + may_end = True + else: + raise ValueError('Expected array delimiter') + + +def scorer_check(path, config): + scorer = dict(elapsed=0., steps=0, dwell=0., lift_dwell=0., max_lift=0., max_pad_force=0., + critical_contact_ticks=0, lift_seen=False, grasp_seen=False, carry_complete=False, + retention_lost=False, carry_contact_loss=0., max_carry_contact_loss=0., + carry_contact_loss_events=0, first_success_time=None, reason='running', done=False) + histogram, count = {}, 0 + for index, row in enumerate(rows(path)): + assert row['native_tick'] == index + assert abs(row['scorer']['elapsed']-index*.001) < 1e-8 + assert row['warnings'] == [] and all(item['number'] == 0 for item in row['native_warnings']) + contacts = row['contacts'] + active = [] + for raw in contacts['raw']: + assert all(math.isfinite(value) for value in raw['force_native']) + if raw['force_native'][0] > .01: + active.append({'geom_ids': sorted(set(raw['geom_ids'])), + 'normal_force_n': raw['force_native'][0]}) + assert canonical(active) == canonical(contacts['active_pairs']) + if index: + for pair in {tuple(item['geom_ids']) for item in active}: + name = ':'.join(map(str, pair)) + histogram[name] = histogram.get(name, 0)+1 + x = row['scorer_inputs'] + dt = config['timestep'] + scorer['elapsed'] += dt + scorer['steps'] = (index-1)//20 + scorer['max_pad_force'] = max(scorer['max_pad_force'], x['pad_force']) + scorer['critical_contact_ticks'] += bool(x['critical']) + scorer['grasp_seen'] |= x['both'] + scorer['max_lift'] = max(scorer['max_lift'], x['lift']) + loaded = x['both'] and not x['supported'] and x['lift'] >= config['min_lift_m'] + scorer['lift_dwell'] = scorer['lift_dwell']+dt if loaded else 0. + scorer['lift_seen'] |= scorer['lift_dwell']+1e-12 >= config['min_lift_dwell_s'] + if scorer['lift_seen'] and not scorer['carry_complete']: + scorer['carry_complete'] |= bool(x['both'] and x['supported'] and x['distance'] <= config['goal_position_tolerance_m']) + if not scorer['carry_complete']: + if x['both']: + scorer['carry_contact_loss'] = 0. + else: + before = scorer['carry_contact_loss'] + scorer['carry_contact_loss'] += dt + scorer['max_carry_contact_loss'] = max(scorer['max_carry_contact_loss'], scorer['carry_contact_loss']) + if before+1e-12 < .04 <= scorer['carry_contact_loss']+1e-12: + scorer['carry_contact_loss_events'] += 1 + scorer['retention_lost'] = True + qualifies = (scorer['lift_seen'] and scorer['grasp_seen'] and scorer['carry_complete'] + and not scorer['retention_lost'] and x['released'] and x['supported'] + and x['settled'] and x['clear'] and x['distance'] <= config['goal_position_tolerance_m'] + and scorer['critical_contact_ticks'] == 0) + scorer['dwell'] = scorer['dwell']+dt if qualifies else 0. + if scorer['dwell']+1e-12 >= config['goal_dwell_s'] and scorer['first_success_time'] is None: + scorer['first_success_time'] = scorer['elapsed'] + assert canonical(scorer) == canonical(row['scorer']), ('scorer', index) + else: + assert row['scorer_inputs'] is None + assert canonical(scorer) == canonical(row['scorer']) + assert histogram == row['contact_pair_native_ticks'], ('histogram', index) + count += 1 + assert count == 20001 + return {'rows': count, 'native_cadence_s': .001, 'scorer_recomputed_exact': True, + 'active_pairs_recomputed_from_raw_forces': True, + 'contact_pair_native_histogram_exact': True, 'warnings': 0} + + +def arrays_check(left, right): + import numpy as np + fields = [] + with np.load(left, allow_pickle=False) as a, np.load(right, allow_pickle=False) as b: + assert set(a.files) == set(b.files) + for name in a.files: + x, y = a[name], b[name] + assert x.dtype.kind in 'iufb' and y.dtype == x.dtype + assert x.shape == y.shape and x.shape[0] == 20001 + assert np.isfinite(x).all() and np.isfinite(y).all() + assert x.tobytes() == y.tobytes(), name + if name in ('qfrc_applied', 'xfrc_applied'): + assert not np.any(x) + fields.append({'name': name, 'shape': list(x.shape), 'dtype': str(x.dtype)}) + times = a['native_time'] + assert np.max(np.abs(times-times[0]-np.arange(20001)*.001)) < 1e-8 + return {'exact_bitwise_match': True, 'rows': 20001, 'all_finite': True, + 'external_forces_zero': True, 'native_clock_cadence_s': .001, 'fields': fields} + + +def read_episode(path): + with gzip.open(path, 'rt') as stream: + value = json.load(stream, object_pairs_hook=unique, parse_constant=nonfinite) + canonical(value) + return value + + +def self_test(): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary)/'rows.gz' + value = [{'native_tick': n, 'x': [0., {'value': 'text'}]} for n in range(5)] + path.write_bytes(gzip.compress(canonical(value))) + for size in (1, 7, 65536): + assert list(rows(path, size)) == value + assert expanded_id(path) == {'bytes': len(canonical(value)), 'sha256': sha256(canonical(value)).hexdigest()} + for raw in (b'[{"x":1},]', b'[{"x":1}]junk', b'[{"x":1}', b'[{"x":NaN}]', b'[{"x":1,"x":2}]'): + path.write_bytes(gzip.compress(raw)) + try: + list(rows(path, 2)) + except ValueError: + pass + else: + raise AssertionError('Malformed input accepted') + return {'passed': True, 'checks': 9} + + +def main(): + import sys + if sys.flags.optimize: + raise RuntimeError('Run this assertion-based research auditor without -O or PYTHONOPTIMIZE.') + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parent) + parser.add_argument('--output', type=Path, help='New audit path; defaults to ROOT/readback-audit.json') + parser.add_argument('--self-test', action='store_true') + args = parser.parse_args() + if args.self_test: + print(json.dumps(self_test())) + return + root = args.root.resolve() + script = Path(__file__).resolve() + output = args.output if args.output is not None else root/'readback-audit.json' + if output.exists(): + raise ValueError('Never overwrite an audit') + baseline_report = read(root/'comparison.json') + protocol = read(root/'protocol.json') + frozen_files = {str(path.relative_to(root)): file_id(path) for path in sorted(root.rglob('*')) + if path.is_file() and path.name not in ('readback-audit.json',)} + checks = [] + for case in CASES: + a, b = root/('dense-'+case+'-baseline'), root/('dense-'+case+'-candidate') + ra, rb = read(a/'result.json'), read(b/'result.json') + assert ra['status'] == rb['status'] == 'complete' + ea, eb = read_episode(a/'episode.json.gz'), read_episode(b/'episode.json.gz') + assert all(canonical(ea[name]) == canonical(eb[name]) for name in EPISODE_FIELDS) + assert all(canonical(ra[name]) == canonical(rb[name]) for name in ('model', 'runtime', 'assets', 'score')) + left_id, right_id = expanded_id(a/'native-contacts-scorer.json.gz'), expanded_id(b/'native-contacts-scorer.json.gz') + assert left_id == right_id + arrays = arrays_check(a/'native-arrays.npz', b/'native-arrays.npz') + # Canonical payload identity was checked independently for both files; + # parsing one identical payload suffices for cadence/scorer invariants. + scorer = scorer_check(a/'native-contacts-scorer.json.gz', protocol['config']) + expected_success = case not in ('open-jaw', 'hold') + assert ra['score']['success'] is expected_success + checks.append({'case': case, 'exact_boundaries_model_runtime_assets_score': True, + 'canonical_contact_scorer_payload_identity': left_id, + 'arrays': arrays, 'scorer': scorer, 'score_success': expected_success}) + print(json.dumps({'case': case, 'exact': True, 'rows': scorer['rows'], 'expanded_bytes': left_id['bytes']}), flush=True) + assert all(row['returncode'] == 0 and 'error' not in row for row in baseline_report['attempts']) + assert all(row['exact_match'] for row in baseline_report['comparisons'] if row['kind'] == 'timing-pair') + for relative, expected in frozen_files.items(): + assert file_id(root/relative) == expected, ('changed evidence', relative) + rule = baseline_report['adoption'] + timing_eligible = (all(row[clock] for row in rule['paired_lower_mean'] for clock in ('step_wall', 'step_cpu')) + and all(row['candidate_mean_reduction_percent'] >= 5. for row in rule['pooled'].values())) + result = {'schema': 'dvidia.piper-control-readback-audit/v1', 'passed': True, + 'scope': 'Read-only reanalysis of the same 14 frozen attempts, no simulation/timing reruns. Original fail-closed 64 MiB reader report is preserved.', + 'script': file_id(script), 'self_tests': self_test(), + 'original_report': file_id(root/'comparison.json'), 'protocol': file_id(root/'protocol.json'), + 'expanded_bound_bytes': MAX_BYTES, 'frozen_evidence': frozen_files, + 'evidence_unchanged': True, 'cases': checks, + 'total_dense_snapshots_compared': 20001*len(CASES)*2, + 'timing_runs': baseline_report['timing_runs'], 'timing_adoption_rule': protocol['adoption_rule'], + 'eligible_under_predeclared_rule_after_complete_readback': timing_eligible, + 'interpretation': 'Observed two-pair shared-host CPU improvement with exact saved mechanics. Not a general latency guarantee, accuracy gain, hardware test, or recovery-robustness evaluation; the fixed pulse also succeeds.'} + with output.open('x') as stream: + json.dump(result, stream, sort_keys=True, indent=2, allow_nan=False) + stream.write('\n') + print(json.dumps({'passed': True, 'eligible': timing_eligible, 'audit': file_id(output)})) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/piper-control-profile-v01/summary.json b/benchmarks/piper-control-profile-v01/summary.json new file mode 100644 index 0000000..3ec0358 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/summary.json @@ -0,0 +1,78 @@ +{ + "archive": { + "bytes": 127941580, + "filename": "piper-control-profile-v01-evidence.zip", + "logical_bytes": 333173510, + "logical_files": 186, + "sha256": "382eea1a7c4e82bae14b12271cb658a32cfe1562bfc211a8f31f80a911d588fa", + "unique_blob_bytes": 130261531, + "unique_blobs": 107 + }, + "historical_learned_heldout_result": { + "attempts": 12, + "successes": 5, + "unchanged": true + }, + "limitations": [ + "Two matched shared-host pairs on one fixed synthetic command tape.", + "No learned inference in timed environment-step measurements.", + "Dense diagnostics are separate from timing runs.", + "No new held-out accuracy, physical calibration, recovery-robustness or real-time claim.", + "Scheduled jaw pulse also succeeds; it is perturbation-equivalence coverage." + ], + "new_wheel": false, + "original_report": { + "all_passed": false, + "bytes": 17508, + "reason": "Original dense reader rejected three case pairs exceeding its 64 MiB expansion bound.", + "sha256": "11351eebac7d9d615f9c34db3a8fa72133ddbc1d8b7186e501ee9dfe0d3ff934" + }, + "paired_timing": [ + { + "baseline_sum_s": 1.4684963417239487, + "candidate_p95_reduction_percent": 25.79315547361659, + "candidate_sum_s": 1.2328931200318038, + "candidate_total_step_reduction_percent": 16.043841240731815, + "pair": 0 + }, + { + "baseline_sum_s": 1.5423414260149002, + "candidate_p95_reduction_percent": 23.795316993807734, + "candidate_sum_s": 1.3053270056843758, + "candidate_total_step_reduction_percent": 15.367182410636659, + "pair": 1 + } + ], + "physical_robot_ready": false, + "planned_attempts": 14, + "production_promotion": false, + "schema": "dvidia.piper-control-profile-public-summary/v1", + "scope": "One previously checked synthetic PiPER scenario, preserved commands and privileged native simulator state. Allocation-only source comparison, not new skill training, held-out success, hardware latency, calibrated contact or a real-time guarantee.", + "source_branch": "feat/piper-control-profile-v01", + "supplemental_readback": { + "bytes": 45061, + "eligible_under_predeclared_rule_after_complete_readback": true, + "evidence_unchanged": true, + "expanded_bound_bytes": 536870912, + "passed": true, + "sha256": "1a4f4a1093961b59c03a3a244d123afed35f5a5f77c98d1467a393a20c2d6fd4", + "total_dense_snapshots_compared": 200010 + }, + "timing": { + "step_cpu": { + "candidate_mean_reduction_percent": 12.599688117239072, + "mean_s": { + "baseline": 0.0014383610000000013, + "candidate": 0.001257132 + } + }, + "step_wall": { + "candidate_mean_reduction_percent": 15.69721381493122, + "mean_s": { + "baseline": 0.0015054188838694244, + "candidate": 0.0012691100628580898 + } + } + }, + "timing_pairs": 2 +} diff --git a/benchmarks/piper-control-profile-v01/test_package_evidence.py b/benchmarks/piper-control-profile-v01/test_package_evidence.py new file mode 100644 index 0000000..21da477 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/test_package_evidence.py @@ -0,0 +1,126 @@ +"""Archive integrity, deduplication and reconstruction boundaries, no native work.""" +from hashlib import sha256 +import importlib.util +import io +import json +from pathlib import Path +import stat +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +spec = importlib.util.spec_from_file_location('dvidia_package_evidence', Path(__file__).with_name('package_evidence.py')) +package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package) +helper = {'__name__': 'dvidia_test_verifier'} +exec(compile(package.VERIFY_SOURCE, 'verify_extract.py', 'exec'), helper) + + +class EvidencePackageTests(unittest.TestCase): + def fixture(self, root): + source = root/'original.txt' + source.write_bytes(b'original scientific bytes\n') + files = {'comparison/a.txt': source, 'comparison/b.txt': source} + generated = {'README.md': b'Readme\n', 'LICENSE': b'Existing license\n', + 'tools/verify_extract.py': package.VERIFY_SOURCE.encode()} + return files, generated + + def mutate(self, archive, output, change): + with zipfile.ZipFile(archive) as original: + content = {name: original.read(name) for name in original.namelist()} + change(content) + with zipfile.ZipFile(output, 'x') as result: + for name, value in content.items(): result.writestr(name, value) + + def test_deterministic_deduplication_verified_reconstruction_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); files, generated = self.fixture(root) + first, second = root/'a.zip', root/'b.zip' + manifest = package.build_archive(first, files, generated, []) + package.build_archive(second, files, generated, []) + self.assertEqual(first.read_bytes(), second.read_bytes()) + result = helper['verify'](first, root/'restored') + self.assertEqual(result['logical_files'], 5) + self.assertEqual(result['unique_blobs'], 4) + self.assertEqual((root/'restored/comparison/a.txt').read_bytes(), files['comparison/a.txt'].read_bytes()) + self.assertEqual((root/'restored/comparison/b.txt').read_bytes(), files['comparison/b.txt'].read_bytes()) + with self.assertRaises(ValueError): helper['verify'](first, root/'restored') + self.assertEqual(manifest['files'][0]['path'], 'LICENSE') + + def test_traversal_absolute_backslash_and_directory_collisions_reject_before_extraction(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); files, generated = self.fixture(root) + archive = root/'original.zip'; package.build_archive(archive, files, generated, []) + for index, bad in enumerate(('../escape', '/absolute', 'a/../b', 'a\\b', 'a//b', 'C:/drive')): + def change(content, bad=bad): + manifest = json.loads(content['manifest.json']);manifest['files'][0]['path'] = bad + content['manifest.json'] = json.dumps(manifest).encode() + candidate = root/f'bad-{index}.zip';self.mutate(archive, candidate, change) + with self.assertRaises(ValueError):helper['verify'](candidate, root/f'restored-{index}') + self.assertFalse((root/f'restored-{index}').exists()) + for bad in ('../escape', '/absolute', 'a/../b', 'a\\b'): + with self.assertRaises(ValueError):package.safe_path(bad) + + def test_blob_tampering_missing_payload_extra_member_and_symlink_reject(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary);files,generated=self.fixture(root);archive=root/'original.zip' + package.build_archive(archive, files, generated, []) + def tamper(content): + name=next(name for name in content if name.startswith('blobs/'));content[name]+=b'changed' + def missing(content):del content[next(name for name in content if name.startswith('blobs/'))] + def extra(content):content['unexpected.txt']=b'not listed' + for index,change in enumerate((tamper,missing,extra)): + candidate=root/f'bad-{index}.zip';self.mutate(archive,candidate,change) + with self.assertRaises(ValueError):helper['verify'](candidate) + with zipfile.ZipFile(archive,'a') as bundle: + entry=zipfile.ZipInfo('blobs/'+'0'*64);entry.create_system=3 + entry.external_attr=(stat.S_IFLNK|0o777)<<16;bundle.writestr(entry,b'target') + with self.assertRaises(ValueError):helper['verify'](archive) + + def test_scan_detects_cross_chunk_private_paths_and_credential_shapes(self): + private=b'/'+b'Users/'+b'person/' + with self.assertRaises(ValueError):package.scan_stream(io.BytesIO(b'a'*65534+private)) + with self.assertRaises(ValueError):package.scan_stream(io.BytesIO(b'hf_'+b'a'*24)) + self.assertEqual(package.scan_stream(io.BytesIO(b'public hashes and counts')),24) + with self.assertRaises(ValueError):package.scan_stream(io.BytesIO(b'x'*101),100) + + def test_prepare_rejects_mutated_frozen_scientific_payload_before_output(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);comparison=root/'comparison';comparison.mkdir() + original=comparison/'comparison.json';original.write_bytes(b'{"all_passed":false}') + native=comparison/'native.npz';native.write_bytes(b'original-native-bytes') + frozen={p.name:package.file_identity(p) for p in (original,native)} + audit={'passed':True,'evidence_unchanged':True, + 'eligible_under_predeclared_rule_after_complete_readback':True, + 'original_report':package.file_identity(original),'frozen_evidence':frozen} + raw=json.dumps(audit).encode();(comparison/'readback-audit.json').write_bytes(raw) + # Same length mutation must still be rejected by its audited digest. + native.write_bytes(b'mutated!-native-bytes') + with patch.object(package,'AUDIT_SHA',sha256(raw).hexdigest()): + with self.assertRaisesRegex(ValueError,'pinned audit'): + package.prepare(root,comparison,root/'unused-profile',root/'release') + self.assertFalse((root/'release').exists()) + + def test_frozen_roster_rejects_unknown_and_missing_files(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);source=root/'native.npz';source.write_bytes(b'fixed') + (root/'readback-audit.json').write_bytes(b'{}') + audit={'frozen_evidence':{'native.npz':package.file_identity(source)}} + self.assertEqual(package.check_frozen_evidence(root,audit),audit['frozen_evidence']) + extra=root/'unreviewed.json';extra.write_bytes(b'{}') + with self.assertRaisesRegex(ValueError,'roster'):package.check_frozen_evidence(root,audit) + extra.unlink();source.unlink() + with self.assertRaisesRegex(ValueError,'roster'):package.check_frozen_evidence(root,audit) + + def test_archive_indexing_remains_bound_to_audited_identity_after_precheck(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);files,generated=self.fixture(root) + expected={name:package.file_identity(source) for name,source in files.items()} + next(iter(files.values())).write_bytes(b'changed after precheck\n') + with self.assertRaisesRegex(ValueError,'before archive indexing'): + package.build_archive(root/'bad.zip',files,generated,[],expected) + self.assertFalse((root/'bad.zip').exists()) + + +if __name__ == '__main__':unittest.main() diff --git a/benchmarks/piper-control-profile-v01/test_profile.py b/benchmarks/piper-control-profile-v01/test_profile.py new file mode 100644 index 0000000..62bf994 --- /dev/null +++ b/benchmarks/piper-control-profile-v01/test_profile.py @@ -0,0 +1,45 @@ +"""Pure correctness checks for the development diagnostic, no native stepping.""" +from copy import deepcopy +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location('piper_control_profile', Path(__file__).with_name('profile.py')) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ProfileProtocolTests(unittest.TestCase): + def test_strict_json_rejects_duplicate_nonfinite_and_overflow(self): + for raw in (b'{"x":0,"x":1}', b'{"x":NaN}', b'{"x":1e999}'): + with self.assertRaises(ValueError): module.decode(raw) + + def test_comparison_catches_contacts_commands_and_outcomes_but_not_timing(self): + original = {key: None for key in module.COMPARISON_FIELDS} + original.update(trajectory=[{'grasp_contact_counts': {'left': 1, 'right': 1}}], + actions=[{'gripper_position_m': .01}], metrics={'contact_pair_native_ticks': {'1:2': 3}}) + changed = deepcopy(original) + changed['timing'] = {'wall_seconds': 999.} + self.assertTrue(module.compare_episode(original, changed)['exact_match']) + for key, value in (('trajectory', []), ('actions', []), ('metrics', {}), ('success', True)): + altered = deepcopy(changed); altered[key] = value + self.assertIn(key, module.compare_episode(original, altered)['different_fields']) + altered = deepcopy(original); del altered['error'] + self.assertFalse(module.compare_episode(original, altered)['exact_match']) + + def test_comparison_distinguishes_boolean_and_numeric_json(self): + original = {key: None for key in module.COMPARISON_FIELDS} + original['valid'] = False + altered = dict(original, valid=0) + self.assertFalse(module.compare_episode(original, altered)['exact_match']) + + def test_latency_summary_is_fixed_linear_quantile_and_strict_budget(self): + summary = module.summary([.0, .01, .02, .03]) + self.assertEqual(summary['count'], 4) + self.assertEqual(summary['over_nominal_20ms'], 1) + self.assertAlmostEqual(summary['p50_s'], .015) + self.assertAlmostEqual(summary['p95_s'], .0285) + self.assertEqual(module.summary([]), {'count': 0}) + + +if __name__ == '__main__': unittest.main() diff --git a/publishing/huggingface/publish_piper_control_profile_v01.py b/publishing/huggingface/publish_piper_control_profile_v01.py new file mode 100644 index 0000000..1f73dd0 --- /dev/null +++ b/publishing/huggingface/publish_piper_control_profile_v01.py @@ -0,0 +1,144 @@ +"""Add one bounded public benchmark dataset; never modify shared Spaces. + +Uses the existing named Keychain client only during publish. Credentials remain +in process memory; remote failures produce fixed diagnostics, not raw responses. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from pathlib import Path +import sys + +from publish import PublicationError, client +from publish_task_environments_v02 import digest, encoded, plain, public_client, download, scan + +REPO = 'Dvidia/piper-control-profile-v01' +SCHEMA = 'dvidia.piper-control-profile-publication/v1' +NAMES = {'README.md', 'LICENSE', 'piper-control-profile-v01-evidence.zip', + 'summary.json', 'comparison.json', 'readback-audit.json'} +ARCHIVE = 'piper-control-profile-v01-evidence.zip' + + +def read(path): + path = plain(path) + limit = 256*1024*1024 if path.name == ARCHIVE else 4*1024*1024 + if not path.is_file() or path.stat().st_size > limit: + raise PublicationError('Missing or oversized reviewed publication file.') + with path.open('rb') as stream: + raw = stream.read(limit+1) + if len(raw) > limit: + raise PublicationError('Publication file changed beyond its byte limit.') + return raw + + +def files(root): + root = plain(root) + paths = sorted(root.iterdir()) + if {p.name for p in paths} != NAMES: + raise PublicationError('Dataset must contain exactly the reviewed six files.') + inventory = [] + for path in paths: + raw = read(path) + scan(raw) + inventory.append({'path': path.name, **digest(raw), + 'representation': 'lfs' if path.name == ARCHIVE else 'git'}) + return inventory + + +def stage(root, plan): + root, plan = plain(root), plain(plan) + inventory = files(root) + if {r['path'] for r in inventory} != NAMES: + raise PublicationError('Dataset must contain exactly the reviewed six files.') + value = {'schema': SCHEMA, 'repo_id': REPO, 'repo_type': 'dataset', + 'private': False, 'root': str(root), 'files': inventory} + with plan.open('xb') as stream: + stream.write(encoded(value)) + return {'plan_sha256': sha256(read(plan)).hexdigest(), 'files': len(inventory), + 'bytes': sum(r['bytes'] for r in inventory)} + + +def checked(plan, expected_digest): + raw = read(plan) + if sha256(raw).hexdigest() != expected_digest: + raise PublicationError('Reviewed plan digest changed.') + value = json.loads(raw) + if (set(value) != {'schema', 'repo_id', 'repo_type', 'private', 'root', 'files'} + or value['schema'] != SCHEMA or value['repo_id'] != REPO + or value['repo_type'] != 'dataset' or value['private'] is not False + or {r['path'] for r in value['files']} != NAMES + or files(value['root']) != value['files']): + raise PublicationError('Reviewed dataset identity or bytes changed.') + return value + + +def publish(plan, expected_digest, cache): + value = checked(plan, expected_digest) + payloads = [] + for row in value['files']: + raw = read(Path(value['root'])/row['path']) + if digest(raw) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Payload changed while freezing reviewed publication bytes.') + payloads.append((row['path'], raw)) + api, token = client(cache) + from huggingface_hub import CommitOperationAdd + from huggingface_hub.errors import RepositoryNotFoundError + try: + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + except RepositoryNotFoundError: + api.create_repo(REPO, repo_type='dataset', private=False, exist_ok=False) + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + if info.private is not False or not info.sha or not info.siblings: + raise PublicationError('Expected an initialized public dataset.') + existing = {item.rfilename for item in info.siblings} + if existing != {'.gitattributes'}: + raise PublicationError('Dataset is not empty. Verify the existing publication; do not overwrite it.') + # Use the exact immutable bytes verified before credential/remote access. + operations = [CommitOperationAdd(path_in_repo=name, path_or_fileobj=raw) + for name, raw in payloads] + commit = api.create_commit(REPO, repo_type='dataset', operations=operations, + parent_commit=info.sha, + commit_message='Publish PiPER control profiling, paired compute and exact mechanics evidence') + del token, api + return {'repo_id': REPO, 'commit': commit.oid, 'files_added': len(operations)} + + +def verify(plan, expected_digest, cache): + value = checked(plan, expected_digest) + api = public_client() + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True, token=False) + if info.private is not False or {r.rfilename for r in info.siblings} != NAMES | {'.gitattributes'}: + raise PublicationError('Public dataset visibility or roster differs.') + for row in value['files']: + raw = download(REPO, row['path'], 'dataset', info.sha, cache) + if digest(raw) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Anonymous public bytes differ from the reviewed artifacts.') + return {'repo_id': REPO, 'commit': info.sha, 'anonymous_files_verified': len(value['files'])} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + s = commands.add_parser('stage'); s.add_argument('root'); s.add_argument('--plan', required=True) + for name in ('publish', 'verify'): + command = commands.add_parser(name) + command.add_argument('--plan', required=True) + command.add_argument('--sha256', required=True) + command.add_argument('--cache', required=True) + args = parser.parse_args() + try: + result = (stage(args.root, args.plan) if args.command == 'stage' else + globals()[args.command](args.plan, args.sha256, Path(args.cache))) + print(json.dumps(result, indent=2)) + return 0 + except PublicationError as exc: + print(str(exc), file=sys.stderr) + except Exception: + print('Publication operation failed; no credential or remote response details displayed.', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/huggingface/test_publish_piper_control_profile_v01.py b/publishing/huggingface/test_publish_piper_control_profile_v01.py new file mode 100644 index 0000000..b4bb8b4 --- /dev/null +++ b/publishing/huggingface/test_publish_piper_control_profile_v01.py @@ -0,0 +1,92 @@ +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import publish_piper_control_profile_v01 as publisher + + +class PublicationTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.dataset = self.root/'dataset'; self.dataset.mkdir() + for name in publisher.NAMES: + (self.dataset/name).write_bytes(b'synthetic publication fixture') + self.plan = self.root/'plan.json' + self.receipt = publisher.stage(self.dataset, self.plan) + + def test_staging_and_checking_are_credential_free(self): + with patch.object(publisher, 'client', side_effect=AssertionError('Credential accessed')): + result = publisher.checked(self.plan, self.receipt['plan_sha256']) + self.assertEqual(result['repo_id'], 'Dvidia/piper-control-profile-v01') + self.assertEqual(len(result['files']), 6) + + def test_mutated_payload_and_unreviewed_extra_files_fail(self): + (self.dataset/'README.md').write_bytes(b'changed') + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, self.receipt['plan_sha256']) + (self.dataset/'README.md').write_bytes(b'synthetic publication fixture') + (self.dataset/'unreviewed.json').write_text('{}') + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, self.receipt['plan_sha256']) + + def test_resealed_changed_destination_is_rejected(self): + value = json.loads(self.plan.read_bytes()) + for field, replacement in [('repo_id', 'Dvidia/robot-simulation-lab'), + ('repo_type', 'space'), ('private', True)]: + changed = deepcopy(value); changed[field] = replacement + raw = publisher.encoded(changed); self.plan.write_bytes(raw) + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, sha256(raw).hexdigest()) + + def test_existing_dataset_cannot_be_overwritten(self): + class Info: + private = False + sha = 'a'*40 + siblings = [type('File', (), {'rfilename': 'README.md'})()] + with patch.object(publisher, 'client') as get_client: + api = unittest.mock.Mock() + api.repo_info.return_value = Info() + get_client.return_value = api, 'synthetic-fixture' + with self.assertRaisesRegex(publisher.PublicationError, 'not empty'): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root/'cache') + api.create_commit.assert_not_called() + + def test_mutation_at_payload_freeze_is_rejected_before_credentials(self): + original = publisher.read + reads = 0 + def changed(path, *args, **kwargs): + nonlocal reads + if Path(path).name == 'README.md': + reads += 1 + if reads == 2: + return b'changed after review' + return original(path, *args, **kwargs) + with patch.object(publisher, 'read', side_effect=changed), patch.object(publisher, 'client') as client: + with self.assertRaisesRegex(publisher.PublicationError, 'freezing'): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root/'cache') + client.assert_not_called() + + def test_oversized_metadata_and_archive_are_rejected_before_reading(self): + for name, size in [('summary.json', 4*1024*1024+1), + (publisher.ARCHIVE, 256*1024*1024+1)]: + path = self.dataset/name + with path.open('wb') as stream: + stream.truncate(size) + with self.assertRaisesRegex(publisher.PublicationError, 'oversized'): + publisher.read(path) + + def test_symlink_payload_is_rejected(self): + target = self.root/'unreviewed'; target.write_text('synthetic') + path = self.dataset/'README.md'; path.unlink(); path.symlink_to(target) + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, self.receipt['plan_sha256']) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/dvidia_training/piper_task.py b/src/dvidia_training/piper_task.py index 779e6ee..96668df 100644 --- a/src/dvidia_training/piper_task.py +++ b/src/dvidia_training/piper_task.py @@ -220,8 +220,13 @@ def _contacts(self): mj, np = sim._native_imports() counts, forces, table, critical, pairs = {'left': 0, 'right': 0}, {'left': 0., 'right': 0.}, 0, 0, [] pads = set(sum(self.pad_geoms.values(), [])) + nonpad_robot = self.robot_geoms-pads + support_geoms = {self.table_geom, self.floor_geom} + # Scratch storage is private to this call. Every contact still queries + # fresh native forces; no result survives a physics step or caller edit. + force = np.zeros(6) for index, contact in enumerate(self.data.contact[:self.data.ncon]): - force = np.zeros(6) + force.fill(0.) mj.mj_contactForce(self.model, self.data, index, force) if force[0] <= .01: continue @@ -233,8 +238,8 @@ def _contacts(self): if pair.intersection(self.pad_geoms[side]): counts[side] += 1 forces[side] += float(force[0]) - critical += int(bool(pair.intersection(self.robot_geoms-pads)) or self.floor_geom in pair) - elif pair.intersection({self.table_geom, self.floor_geom}) and pair.intersection(self.moving_geoms): + critical += int(bool(pair.intersection(nonpad_robot)) or self.floor_geom in pair) + elif pair.intersection(support_geoms) and pair.intersection(self.moving_geoms): critical += 1 elif pair.intersection(self.arm_geoms) and pair.issubset(self.robot_geoms): critical += 1 diff --git a/tests/test_piper_contact_scratch.py b/tests/test_piper_contact_scratch.py new file mode 100644 index 0000000..37051f2 --- /dev/null +++ b/tests/test_piper_contact_scratch.py @@ -0,0 +1,73 @@ +"""Contact semantics at force thresholds and between mutable native reads.""" +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import piper_task + + +class ContactReadTests(unittest.TestCase): + def fixture(self, rows): + env = piper_task.PiperTaskEnv('unused') + env.model = object() + env.pad_geoms = {'left': [3, 4], 'right': [5, 6]} + env.robot_geoms = set(range(1, 7)) + env.moving_geoms = set(range(2, 7)) + env.arm_geoms = {2} + env.block_geom, env.table_geom, env.floor_geom = 7, 8, 9 + env.data = SimpleNamespace( + contact=[SimpleNamespace(geom1=a, geom2=b) for a, b, _ in rows], + ncon=len(rows), + ) + + def read_force(model, data, index, destination): + # A fresh force is provided for every query, as for native contact. + destination[:] = [rows[index][2], 0., 0., 0., 0., 0.] + + native = SimpleNamespace(mj_contactForce=read_force) + return env, native + + def test_strict_threshold_keeps_order_and_all_contact_classes(self): + rows = [(7, 3, .009), (7, 4, .01), (7, 3, .011), + (5, 7, .02), (7, 8, .03), (7, 2, .04), + (7, 9, .05), (2, 8, .06), (2, 3, .07), + (1, 9, .08), (3, 5, .09)] + env, native = self.fixture(rows) + with patch.object(piper_task.sim, '_native_imports', return_value=(native, np)): + counts, forces, table, critical, pairs = env._contacts() + self.assertEqual(counts, {'left': 1, 'right': 1}) + self.assertEqual(forces, {'left': .011, 'right': .02}) + self.assertEqual(table, 1) + self.assertEqual(critical, 4) + # Logs include active pairs even when they are neither grasp nor critical. + self.assertEqual(pairs, [{'geom_ids': sorted([a, b]), 'normal_force_n': f} + for a, b, f in rows[2:]]) + + def test_another_read_sees_changed_contacts_without_mutating_old_result(self): + rows = [(3, 7, .02), (5, 7, .03)] + env, native = self.fixture(rows) + with patch.object(piper_task.sim, '_native_imports', return_value=(native, np)): + first = env._contacts() + rows[0] = (3, 7, 0.) + rows[1] = (5, 7, .06) + second = env._contacts() + env.data.ncon = 0 + third = env._contacts() + self.assertEqual(first[:2], ({'left': 1, 'right': 1}, {'left': .02, 'right': .03})) + self.assertEqual(second[:2], ({'left': 0, 'right': 1}, {'left': 0., 'right': .06})) + self.assertEqual(third, ({'left': 0, 'right': 0}, {'left': 0., 'right': 0.}, 0, 0, [])) + + def test_geometry_membership_change_is_visible_on_next_call(self): + env, native = self.fixture([(7, 3, .02)]) + with patch.object(piper_task.sim, '_native_imports', return_value=(native, np)): + self.assertEqual(env._contacts()[3], 0) + env.pad_geoms['left'] = [4] + result = env._contacts() + self.assertEqual(result[0], {'left': 0, 'right': 0}) + self.assertEqual(result[3], 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_piper_control_compare.py b/tests/test_piper_control_compare.py new file mode 100644 index 0000000..cb5fd23 --- /dev/null +++ b/tests/test_piper_control_compare.py @@ -0,0 +1,157 @@ +"""Pure evidence/protocol tests; never start a native comparison.""" +import gzip +import importlib.util +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import contextlib +import io +import subprocess + +import numpy as np + + +_PATH = Path(__file__).resolve().parents[1]/'benchmarks/piper-control-profile-v01/compare.py' +_SPEC = importlib.util.spec_from_file_location('piper_control_compare', _PATH) +compare = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(compare) + + +class ComparisonProtocolTests(unittest.TestCase): + def test_fixed_balanced_schedule_and_no_single_pair(self): + self.assertEqual(compare.schedule(2), ['baseline', 'candidate', 'candidate', 'baseline']) + self.assertEqual(compare.schedule(4), compare.schedule(2)*2) + for invalid in (True, 0, 1, 3, 10): + with self.assertRaises(ValueError): + compare.schedule(invalid) + + def test_source_change_and_undeclared_difference_fail_closed(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + folder = root/'src/dvidia_training' + folder.mkdir(parents=True) + for name in compare.SOURCE_FILES: + (folder/name).write_text(name) + frozen = compare.inventory(root) + compare.check_inventory(root, frozen) + (folder/'piper_task.py').write_text('candidate') + candidate = compare.inventory(root) + self.assertEqual(compare.source_differences(frozen, candidate), ['piper_task.py']) + with self.assertRaises(ValueError): + compare.check_inventory(root, frozen) + (folder/'reference_task.py').write_text('changed scorer') + with self.assertRaises(ValueError): + compare.source_differences(frozen, compare.inventory(root)) + + def test_document_comparison_catches_contact_and_scorer_mismatch(self): + left = {'contacts': {'left': 1}, 'scorer': {'retention_lost': False}} + self.assertTrue(compare.exact_document(left, left)['exact_match']) + right = {'contacts': {'left': 0}, 'scorer': {'retention_lost': True}} + self.assertEqual(compare.exact_document(left, right)['different_fields'], ['contacts', 'scorer']) + + def test_bitwise_array_comparison_catches_signed_zero_and_shape(self): + with tempfile.TemporaryDirectory() as temporary: + left, right = Path(temporary)/'a.npz', Path(temporary)/'b.npz' + np.savez(left, qpos=np.array([[0., 1.], [2., 3.]])) + np.savez(right, qpos=np.array([[0., 1.], [2., 3.]])) + self.assertTrue(compare.compare_arrays(left, right)['exact_match']) + np.savez(right, qpos=np.array([[-0., 1.], [2., 3.]])) + result = compare.compare_arrays(left, right) + self.assertFalse(result['exact_match']) + self.assertEqual(result['differences'][0]['first_row'], 0) + np.savez(right, qpos=np.array([[0., 1.]])) + self.assertEqual(compare.compare_arrays(left, right)['differences'][0]['reason'], 'shape_or_dtype') + + def test_reject_executable_npz_and_duplicate_nonfinite_json(self): + with tempfile.TemporaryDirectory() as temporary: + left, right = Path(temporary)/'a.npz', Path(temporary)/'b.npz' + np.savez(left, qpos=np.array([{'unsafe': True}], dtype=object)) + np.savez(right, qpos=np.array([{'unsafe': True}], dtype=object)) + with self.assertRaises(ValueError): + compare.compare_arrays(left, right) + for text in ('{"x":1,"x":2}', '{"x":NaN}', '{"x":1e999}'): + with self.assertRaises(ValueError): + compare.decode(text) + + def test_gzip_bound_and_write_exclusive(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary)/'rows.json.gz' + compare.write_gzip(target, {'value': [1, 2, 3]}) + self.assertEqual(compare.read_gzip(target), {'value': [1, 2, 3]}) + with self.assertRaises(FileExistsError): + compare.write_gzip(target, {}) + target.write_bytes(gzip.compress(json.dumps({'large': 'a'*1000}).encode())) + with self.assertRaises(ValueError): + compare.read_gzip(target, 100) + + def test_foreign_tape_rejected_before_native_import(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary)/'episode.json.gz' + target.write_bytes(gzip.compress(b'{}')) + with self.assertRaisesRegex(ValueError, 'exact preserved'): + compare.load_tape(target) + + def test_dense_case_contract_and_step_statistics(self): + self.assertEqual(len(compare.CASES), 5) + self.assertIn('feedback-no-pulse', compare.CASES) + self.assertIn('feedback-pulse', compare.CASES) + self.assertIn('qfrc_applied', compare.NATIVE_ARRAYS) + self.assertIn('retention_lost', compare.SCORER_FIELDS) + result = compare.summary([.01, .02, .03]) + self.assertEqual(result['p50_s'], .02) + self.assertEqual(result['over_nominal_20ms'], 1) + + def test_predeclared_adoption_requires_both_clocks_and_both_pairs(self): + def row(variant, wall, cpu): + return {'variant': variant, 'status': 'complete', + 'step_wall': {'mean_s': wall, 'sum_s': wall*1000, 'count': 1000}, + 'step_cpu': {'mean_s': cpu, 'sum_s': cpu*1000, 'count': 1000}} + runs = [row('baseline', .01, .01), row('candidate', .009, .009), + row('candidate', .009, .009), row('baseline', .01, .01)] + self.assertTrue(compare.adoption_result(runs, True)['eligible_under_predeclared_rule']) + self.assertFalse(compare.adoption_result(runs, False)['eligible_under_predeclared_rule']) + runs[2] = row('candidate', .0101, .008) + self.assertFalse(compare.adoption_result(runs, True)['eligible_under_predeclared_rule']) + runs[2] = row('candidate', .009, .011) + self.assertFalse(compare.adoption_result(runs, True)['eligible_under_predeclared_rule']) + runs[2] = row('candidate', .0099, .0099) + runs[1] = row('candidate', .0099, .0099) + self.assertFalse(compare.adoption_result(runs, True)['eligible_under_predeclared_rule']) + + def test_every_timeout_retained_without_retries_or_native_imports(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + roots = {} + for variant in ('baseline', 'candidate'): + roots[variant] = root/variant + folder = roots[variant]/'src/dvidia_training' + folder.mkdir(parents=True) + for name in compare.SOURCE_FILES: + (folder/name).write_text(variant if name == 'piper_task.py' else name) + episode = root/'old-episode.gz' + episode.write_bytes(b'immutable-tape') + python = root/'venv-python' + python.write_bytes(b'immutable-python') + args = SimpleNamespace(output=root/'evidence', pairs=2, + baseline_root=roots['baseline'], candidate_root=roots['candidate'], + episode=episode, python=python, assets=root/'assets', worker_timeout=1.) + tape = ({'seed': 0, 'config': {}}, [], compare.identity(episode.read_bytes())) + with patch.object(compare, 'load_tape', return_value=tape), \ + patch.object(compare.subprocess, 'run', side_effect=subprocess.TimeoutExpired('worker', 1)) as runner, \ + contextlib.redirect_stdout(io.StringIO()): + report = compare.run(args) + self.assertEqual(runner.call_count, 14) + self.assertEqual(len(report['attempts']), 14) + self.assertTrue(all('TimeoutExpired' in row['error'] for row in report['attempts'])) + self.assertFalse(report['all_passed']) + self.assertFalse(report['adoption']['eligible_under_predeclared_rule']) + self.assertEqual(len(list(args.output.glob('*/started.json'))), 14) + self.assertEqual(len(list(args.output.glob('*/process.json'))), 14) + self.assertTrue((args.output/'comparison.json').is_file()) + + +if __name__ == '__main__': + unittest.main()