diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index be6a67e90..6928255d9 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -358,14 +358,13 @@ jobs: # the build through the CUDA prover path. NOTE: requires this PR's bench_abba.sh change # (the BENCH_FEATURES env) to be on main — i.e. it only takes effect after merge. # REBUILD=1: each Vast box is fresh, GPU-specific hardware — always rebuild both - # binaries (PTX is compiled for the detected arch); never trust a cached binary. - # CUDARC_PIN: pin cudarc to a fixed CUDA version (cuda-12080 = CUDA 12.8, matching the - # cuda_max_good>=12.8 offer floor) and drop fallback-latest, so cudarc binds a known - # symbol set instead of its newest. With fallback-latest cudarc requested a symbol the - # box's driver doesn't export (e.g. cuDevSmResourceSplit) -> runtime panic. This is the - # too-new end of the same compatibility window that MIN_DRIVER>=580 guards at the - # too-old end (older drivers lack cuCtxGetDevice_v2 and the GPU path falls back to CPU). - # nvidia-smi is logged for diagnosing driver issues. + # binaries (cubin is compiled for the detected arch); never trust a cached binary. + # CUDARC_PIN: compat shim for pre-pin baseline shas. cudarc's CUDA version is now pinned + # permanently in crypto/math-cuda/Cargo.toml (cuda-12080), so this no-ops on shas that + # carry the pin and only rewrites older baselines (where fallback-latest could request a + # symbol the box's driver doesn't export, e.g. cuDevSmResourceSplit -> runtime panic). + # MIN_DRIVER>=580 still guards the too-old end (older drivers lack cuCtxGetDevice_v2 and + # the GPU path falls back to CPU). nvidia-smi is logged for diagnosing driver issues. REMOTE="set -e; cd /workspace/lambda_vm; \ command -v python3 >/dev/null || { apt-get update -qq && apt-get install -y -qq python3; }; \ nvidia-smi || true; \ diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index 1a1f9a2b1..ddcce0ee3 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -287,7 +287,8 @@ jobs: ''|*[!A-Za-z0-9._/-]*) echo "::error::invalid ref: '$REF'"; exit 1 ;; esac # Check out the ref under test on the box, then run the CUDA test groups. - # gpu_test.sh owns the CUDARC_PIN / SYSROOT_DIR defaults — don't duplicate them here. + # gpu_test.sh owns the SYSROOT_DIR default — don't duplicate it here. (cudarc's CUDA + # version is pinned in crypto/math-cuda/Cargo.toml, so no CUDARC_PIN is needed.) REMOTE="set -e; cd /workspace/lambda_vm; \ git fetch --force origin '$REF'; \ git checkout -f FETCH_HEAD; \ diff --git a/README.md b/README.md index 2a9e3ed6e..0967f34d6 100644 --- a/README.md +++ b/README.md @@ -233,10 +233,18 @@ The CUDA test groups run only on a machine with an NVIDIA GPU and `nvcc`: - `make test-prover-cuda` — the prover/stark/crypto/ecsm suite with the GPU path enabled - `make test-prover-comprehensive-cuda` — the comprehensive all-instructions prove on the GPU path -The kernels are compiled by `nvcc` into PTX that the driver JIT-compiles at load, so the GPU's -driver must be new enough for the toolkit — an older driver rejects the PTX with -`CUDA_ERROR_UNSUPPORTED_PTX_VERSION`. These groups run automatically on a rented GPU in the merge -queue via `.github/workflows/gpu-tests.yml` (which filters offers on `cuda_max_good`). +The kernels are AOT-compiled by `nvcc` into native cubin (SASS) for the host GPU's real arch +(detected via `nvidia-smi`, or overridden with `CUDARC_NVCC_ARCH`), not PTX. This sidesteps the +PTX-ISA JIT version check, so a CUDA toolkit *newer* than the driver still loads and runs — no +`CUDA_ERROR_UNSUPPORTED_PTX_VERSION` and no need to hand-match the toolkit to the driver. The only +requirement is that the toolkit knows the GPU's compute capability (a too-old toolkit fails loudly +at `nvcc` build time). cudarc's host-side driver-API symbol set is likewise pinned to a safe floor +(`cuda-12080`) in `crypto/math-cuda/Cargo.toml`, so no `CUDARC_CUDA_VERSION` env is needed either. +That pin makes the GPU path require a driver of CUDA >= 12.8 (driver branch 570+ — any +Blackwell-capable driver qualifies); on an older driver cudarc's eager symbol resolution aborts at +CUDA init rather than falling back to CPU. +These groups run automatically on a rented GPU in the merge queue via +`.github/workflows/gpu-tests.yml` (which filters offers on `cuda_max_good`). ## Benchmarking & Profiling diff --git a/crypto/math-cuda/Cargo.toml b/crypto/math-cuda/Cargo.toml index df4ae6770..7a28498da 100644 --- a/crypto/math-cuda/Cargo.toml +++ b/crypto/math-cuda/Cargo.toml @@ -6,12 +6,26 @@ edition = "2024" license.workspace = true [dependencies] +# cudarc CUDA version is PINNED to `cuda-12080` (CUDA 12.8) — do NOT restore +# `cuda-version-from-build-system` + `fallback-latest`. Rationale: +# * That auto-detect binds the newest symbol set the *build toolkit* knows +# (e.g. a CUDA 13.1 toolkit pulls in `cuDevSmResourceSplit`, gated behind +# `cuda-13010`/`cuda-13020`). cudarc eagerly resolves those symbols at CUDA +# init; a driver that predates them (e.g. 580.x = CUDA 13.0 max) has no such +# export, so the `dynamic-loading` resolver `.expect()`s and PANICS. +# * This crate's cudarc surface is entirely CUDA-11-era +# (CudaContext/CudaFunction/CudaSlice/CudaStream/LaunchConfig/PushKernelArg/ +# DriverError/Ptx — no green contexts). The 12.8 symbol set is a strict +# subset every >=12.8 driver exports, so pinning it resolves cleanly on any +# supported driver and a *newer* driver loses nothing we use. +# * This replaces the fragile per-script `sed` pin in scripts/gpu_test.sh. +# To move the floor (e.g. to use a newer driver-API symbol), bump this one +# feature deliberately — see crypto/math-cuda/build.rs and README "GPU Tests". cudarc = { version = "0.19", default-features = false, features = [ "driver", "nvrtc", "std", - "cuda-version-from-build-system", - "fallback-latest", + "cuda-12080", "dynamic-loading", ] } math = { path = "../math" } diff --git a/crypto/math-cuda/build.rs b/crypto/math-cuda/build.rs index 316a9c7ed..7bd7c04cc 100644 --- a/crypto/math-cuda/build.rs +++ b/crypto/math-cuda/build.rs @@ -15,38 +15,64 @@ fn nvcc_path() -> PathBuf { } /// Query `nvidia-smi` for the local GPU's compute capability (e.g. "12.0" -/// for Blackwell). Returns a `compute_XX` target on success, falling back -/// to `compute_89` (Ada) when no GPU is visible or the query fails. +/// for Blackwell) and return a *real* arch (`sm_XX`) suitable for cubin +/// (SASS) generation. Hard-fails the build when no GPU is visible and the +/// query fails: a cubin is arch-locked, so there is no safe default — any +/// guess produces a binary that loads on exactly one GPU model and silently +/// CPU-falls-back everywhere else. Failing here is loud and fixable (set +/// `CUDARC_NVCC_ARCH` or build on the target host); a guess is neither. +/// +/// This is only reached when `nvcc` is present but the arch can't be detected +/// (a toolkit-installed host with no visible GPU). A host without `nvcc` takes +/// the empty-stub path in `compile_kernel` and never calls this. fn detect_arch() -> String { - const FALLBACK: &str = "compute_89"; + detect_arch_from_smi().unwrap_or_else(|| { + panic!( + "math-cuda: nvcc is present but no GPU arch could be detected via nvidia-smi, \ + and a cubin must target a concrete arch. Set CUDARC_NVCC_ARCH=sm_XX (e.g. sm_120 \ + for RTX 5090, sm_86 for RTX 3090) or build on the target GPU host." + ) + }) +} + +/// Parse the compute capability out of `nvidia-smi` and format it as a real +/// `sm_XX` arch. Returns `None` on every path where no capability can be read +/// (nvidia-smi missing, command failed, or unparsable output) so the caller +/// warns before falling back. +fn detect_arch_from_smi() -> Option { let output = match Command::new("nvidia-smi") .args(["--query-gpu=compute_cap", "--format=csv,noheader"]) .output() { Ok(o) if o.status.success() => o, - _ => return FALLBACK.to_string(), - }; - let line = match std::str::from_utf8(&output.stdout) { - Ok(s) => s, - Err(_) => return FALLBACK.to_string(), + _ => return None, }; + let line = std::str::from_utf8(&output.stdout).ok()?; // First line, first comma-separated value (covers multi-GPU hosts). - let cap = match line.lines().next() { - Some(l) => l.split(',').next().unwrap_or("").trim(), - None => return FALLBACK.to_string(), - }; - let (major, minor) = match cap.split_once('.') { - Some((m, n)) => (m.trim(), n.trim()), - None => return FALLBACK.to_string(), - }; + let cap = line.lines().next()?.split(',').next().unwrap_or("").trim(); + let (major, minor) = cap.split_once('.')?; + let (major, minor) = (major.trim(), minor.trim()); if major.chars().all(|c| c.is_ascii_digit()) && minor.chars().all(|c| c.is_ascii_digit()) { - format!("compute_{major}{minor}") + Some(format!("sm_{major}{minor}")) + } else { + None + } +} + +/// Normalize a user-supplied `CUDARC_NVCC_ARCH` override to a *real* arch +/// (`sm_XX`). cubin (SASS) generation rejects the *virtual* `compute_XX` +/// form, but we accept it (and a bare `XX`) for backwards compatibility. +fn to_real_arch(arch: &str) -> String { + if let Some(n) = arch.strip_prefix("compute_") { + format!("sm_{n}") + } else if arch.starts_with("sm_") { + arch.to_string() } else { - FALLBACK.to_string() + format!("sm_{arch}") } } -fn compile_ptx(src: &str, out_name: &str, have_nvcc: bool) { +fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool) { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let src_path = manifest_dir.join("kernels").join(src); @@ -57,30 +83,40 @@ fn compile_ptx(src: &str, out_name: &str, have_nvcc: bool) { println!("cargo:rerun-if-env-changed=CUDA_PATH"); println!("cargo:rerun-if-env-changed=CUDARC_NVCC_ARCH"); - // When nvcc is missing from PATH, emit an empty PTX stub so the crate - // still compiles. include_str! in src/device.rs needs the file to exist - // at build time. Any runtime kernel call panics in cudarc when loading - // the empty module. We can't run GPU code without nvcc on the build - // host anyway. + // When nvcc is missing from PATH, emit an empty cubin stub so the crate + // still compiles. include_bytes! in src/device.rs needs the file to exist + // at build time. Any runtime kernel call fails to load the empty module and + // the caller falls back to CPU. We can't run GPU code without nvcc on the + // build host anyway. if !have_nvcc { - fs::write(&out_path, "").expect("failed to write empty PTX stub"); + fs::write(&out_path, "").expect("failed to write empty cubin stub"); return; } - // Emit PTX for a virtual architecture; the CUDA driver JIT-compiles it for the - // actual GPU at load time. Override with CUDARC_NVCC_ARCH to pin a specific - // compute capability. If unset, try `nvidia-smi` to match the host GPU - // (avoids JIT failures like nvcc-13.0 PTX rejected on Blackwell drivers); - // fall back to compute_89 (Ada) when detection fails. + // AOT-compile each kernel to a native cubin (SASS) for the host GPU's real + // arch, NOT to PTX. This sidesteps the driver's PTX-ISA JIT version check: + // a toolkit's PTX ISA is fixed by its CUDA version (e.g. CUDA 13.1 emits PTX + // .version 9.1), and a driver older than that toolkit rejects the module at + // load with CUDA_ERROR_UNSUPPORTED_PTX_VERSION -> every kernel silently + // falls back to CPU. A cubin carries pre-compiled SASS for a real arch, so + // the driver loads it directly as long as it supports that GPU (which the + // driver installed for that GPU always does) — regardless of the toolkit's + // CUDA version. See README "GPU Tests". // - // NOTE: this `-arch` only sets the *virtual arch*, not the PTX ISA version, which is - // fixed by this nvcc's CUDA toolkit. The runtime driver must support that toolkit's CUDA - // version or it rejects the PTX with CUDA_ERROR_UNSUPPORTED_PTX_VERSION — i.e. the box's - // driver CUDA must be >= the build toolkit's CUDA. See README "GPU Tests". - let arch = env::var("CUDARC_NVCC_ARCH").unwrap_or_else(|_| detect_arch()); + // Trade-off: a cubin is arch-specific (an `sm_120` cubin runs only on + // `sm_120`). We build+run on the same GPU box in every flow and detect the + // arch from that box's `nvidia-smi`, so this is exactly right. Override with + // CUDARC_NVCC_ARCH (compute_XX / sm_XX / bare XX all accepted) to + // cross-compile for a different arch. If nvcc is present but no GPU is + // detectable and no override is given, `detect_arch` hard-fails rather than + // guessing an arch that would load on one GPU model and CPU-fall-back on + // every other. + let arch = env::var("CUDARC_NVCC_ARCH") + .map(|a| to_real_arch(&a)) + .unwrap_or_else(|_| detect_arch()); let status = Command::new(nvcc_path()) - .args(["--ptx", "-O3", "-std=c++17", "-arch", &arch, "-o"]) + .args(["--cubin", "-O3", "-std=c++17", "-arch", &arch, "-o"]) .arg(&out_path) .arg(&src_path) .status() @@ -107,19 +143,19 @@ fn main() { .unwrap_or(false); if !have_nvcc { println!( - "cargo:warning=math-cuda: nvcc not found at {} — emitting empty PTX stubs. \ - Runtime GPU calls will panic. Install CUDA and rebuild for a working backend.", + "cargo:warning=math-cuda: nvcc not found at {} — emitting empty cubin stubs. \ + Runtime GPU calls fall back to CPU. Install CUDA and rebuild for a working backend.", nvcc_path().display() ); } - compile_ptx("arith.cu", "arith.ptx", have_nvcc); - compile_ptx("ntt.cu", "ntt.ptx", have_nvcc); - compile_ptx("keccak.cu", "keccak.ptx", have_nvcc); - compile_ptx("barycentric.cu", "barycentric.ptx", have_nvcc); - compile_ptx("deep.cu", "deep.ptx", have_nvcc); - compile_ptx("fri.cu", "fri.ptx", have_nvcc); - compile_ptx("inverse.cu", "inverse.ptx", have_nvcc); - compile_ptx("logup.cu", "logup.ptx", have_nvcc); - compile_ptx("constraint_interp.cu", "constraint_interp.ptx", have_nvcc); + compile_kernel("arith.cu", "arith.cubin", have_nvcc); + compile_kernel("ntt.cu", "ntt.cubin", have_nvcc); + compile_kernel("keccak.cu", "keccak.cubin", have_nvcc); + compile_kernel("barycentric.cu", "barycentric.cubin", have_nvcc); + compile_kernel("deep.cu", "deep.cubin", have_nvcc); + compile_kernel("fri.cu", "fri.cubin", have_nvcc); + compile_kernel("inverse.cu", "inverse.cubin", have_nvcc); + compile_kernel("logup.cu", "logup.cubin", have_nvcc); + compile_kernel("constraint_interp.cu", "constraint_interp.cubin", have_nvcc); } diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs index bf75b696e..2bebe2cc0 100644 --- a/crypto/math-cuda/src/device.rs +++ b/crypto/math-cuda/src/device.rs @@ -90,16 +90,21 @@ impl Drop for PinnedStaging { } } -const ARITH_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/arith.ptx")); -const NTT_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/ntt.ptx")); -const KECCAK_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/keccak.ptx")); -const BARY_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/barycentric.ptx")); -const DEEP_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/deep.ptx")); -const FRI_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/fri.ptx")); -const INVERSE_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/inverse.ptx")); -const LOGUP_PTX: &str = include_str!(concat!(env!("OUT_DIR"), "/logup.ptx")); -const CONSTRAINT_INTERP_PTX: &str = - include_str!(concat!(env!("OUT_DIR"), "/constraint_interp.ptx")); +// Kernels are AOT-compiled to native cubin (SASS) by build.rs, embedded here, +// and loaded via `Ptx::from_binary` (cubin bytes -> cuModuleLoadData). This +// avoids the PTX-ISA/driver-version JIT check — see build.rs `compile_kernel`. +// An empty slice (nvcc-less stub build) fails to load at runtime and the caller +// falls back to CPU. +const ARITH_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/arith.cubin")); +const NTT_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ntt.cubin")); +const KECCAK_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/keccak.cubin")); +const BARY_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/barycentric.cubin")); +const DEEP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/deep.cubin")); +const FRI_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/fri.cubin")); +const INVERSE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/inverse.cubin")); +const LOGUP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/logup.cubin")); +const CONSTRAINT_INTERP_CUBIN: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/constraint_interp.cubin")); /// Number of CUDA streams in the pool. Larger pools let many rayon-parallel /// callers overlap on the GPU without serializing on stream ownership. The @@ -125,7 +130,7 @@ pub struct Backend { /// [`detect_vram_budget_bytes`]. vram_budget_bytes: u64, - // arith.ptx + // arith.cubin pub vector_add_u64: CudaFunction, pub gl_add: CudaFunction, pub gl_sub: CudaFunction, @@ -135,7 +140,7 @@ pub struct Backend { pub ext3_add: CudaFunction, pub ext3_sub: CudaFunction, - // ntt.ptx + // ntt.cubin pub bit_reverse_permute: CudaFunction, pub ntt_dit_level: CudaFunction, pub ntt_dit_8_levels: CudaFunction, @@ -152,7 +157,7 @@ pub struct Backend { pub pointwise_mul_row_major: CudaFunction, pub matrix_transpose_strided: CudaFunction, - // keccak.ptx + // keccak.cubin pub keccak256_leaves_base_row_major_row_pair: CudaFunction, pub keccak256_leaves_base_batched: CudaFunction, pub keccak256_leaves_base_row_pair_batched: CudaFunction, @@ -162,7 +167,7 @@ pub struct Backend { pub keccak_merkle_level: CudaFunction, pub merkle_gather_paths: CudaFunction, - // barycentric.ptx + // barycentric.cubin pub barycentric_base_batched: CudaFunction, pub barycentric_ext3_batched: CudaFunction, pub barycentric_base_batched_strided: CudaFunction, @@ -170,14 +175,14 @@ pub struct Backend { pub gather_rows_base: CudaFunction, pub gather_rows_ext3: CudaFunction, - // deep.ptx + // deep.cubin pub deep_composition_ext3_row: CudaFunction, - // fri.ptx + // fri.cubin pub fri_fold_ext3: CudaFunction, pub fri_update_twiddles: CudaFunction, - // inverse.ptx + // inverse.cubin pub compute_denoms_ext3: CudaFunction, pub block_inclusive_scan_fwd_ext3: CudaFunction, pub apply_block_offsets_fwd_ext3: CudaFunction, @@ -192,7 +197,7 @@ pub struct Backend { pub logup_finalize_accum_ext3: CudaFunction, pub logup_assemble_aux_ext3: CudaFunction, - // constraint_interp.ptx + // constraint_interp.cubin pub constraint_interp_kernel: CudaFunction, pub constraint_composition_kernel: CudaFunction, @@ -291,15 +296,16 @@ impl Backend { // we keep the current behaviour. retain_default_mempool(&ctx); - let arith = ctx.load_module(Ptx::from_src(ARITH_PTX))?; - let ntt = ctx.load_module(Ptx::from_src(NTT_PTX))?; - let keccak = ctx.load_module(Ptx::from_src(KECCAK_PTX))?; - let bary = ctx.load_module(Ptx::from_src(BARY_PTX))?; - let deep = ctx.load_module(Ptx::from_src(DEEP_PTX))?; - let fri = ctx.load_module(Ptx::from_src(FRI_PTX))?; - let inverse = ctx.load_module(Ptx::from_src(INVERSE_PTX))?; - let logup = ctx.load_module(Ptx::from_src(LOGUP_PTX))?; - let constraint_interp = ctx.load_module(Ptx::from_src(CONSTRAINT_INTERP_PTX))?; + let arith = ctx.load_module(Ptx::from_binary(ARITH_CUBIN.to_vec()))?; + let ntt = ctx.load_module(Ptx::from_binary(NTT_CUBIN.to_vec()))?; + let keccak = ctx.load_module(Ptx::from_binary(KECCAK_CUBIN.to_vec()))?; + let bary = ctx.load_module(Ptx::from_binary(BARY_CUBIN.to_vec()))?; + let deep = ctx.load_module(Ptx::from_binary(DEEP_CUBIN.to_vec()))?; + let fri = ctx.load_module(Ptx::from_binary(FRI_CUBIN.to_vec()))?; + let inverse = ctx.load_module(Ptx::from_binary(INVERSE_CUBIN.to_vec()))?; + let logup = ctx.load_module(Ptx::from_binary(LOGUP_CUBIN.to_vec()))?; + let constraint_interp = + ctx.load_module(Ptx::from_binary(CONSTRAINT_INTERP_CUBIN.to_vec()))?; let mut streams = Vec::with_capacity(STREAM_POOL_SIZE); for _ in 0..STREAM_POOL_SIZE { @@ -504,7 +510,26 @@ pub fn backend() -> Result<&'static Backend> { if let Some(b) = BACKEND.get() { return Ok(b); } - let b = Backend::init()?; + let b = match Backend::init() { + Ok(b) => b, + Err(e) => { + // Backend init failing means every GPU entry point silently falls + // back to CPU. That is expected on a GPU-less host, but it also + // fires when the AOT cubins won't load — most often a build-host vs + // run-host GPU-arch mismatch (cubins are compiled for the detected + // `sm_XX`) or an empty nvcc-less stub. Warn once so the fallback is + // never silent: rebuild on the run host, or set `CUDARC_NVCC_ARCH`. + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + eprintln!( + "math-cuda: GPU backend unavailable ({e}) — running on CPU. \ + If a GPU is present this is likely a kernel-cubin arch mismatch; \ + rebuild on the run host or set CUDARC_NVCC_ARCH to its sm_XX." + ); + }); + return Err(e); + } + }; let _ = BACKEND.set(b); Ok(BACKEND.get().expect("backend just initialised")) } diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index c4a8eeb7d..be0313bd5 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -86,6 +86,25 @@ where pub type DeepPolynomialEvaluations = (Vec>, Vec>); +/// Deep-composition sums that are identical across all FRI queries of a +/// single proof (see `compute_query_invariant_deep_terms`). +pub struct QueryInvariantDeepTerms +where + FieldExtension: Send + Sync + IsField, +{ + /// `ood_row_sum[row] = sum_col trace_term_coeffs[col][row] * ood(row, col)`, + /// over the reconstructed full OOD grid (g·z-pruned positions are zero). + ood_row_sum: Vec>, + /// Width of the reconstructed full OOD grid (= full trace width). + ood_width: usize, + /// Derived from `proof.composition_poly_parts_ood_evaluation().len()`. + number_of_parts: usize, + /// `challenges.z.pow(number_of_parts)`. + z_pow: FieldElement, + /// `sum_j composition_poly_parts_ood_evaluation[j] * challenges.gammas[j]`. + h_sum_zpow: FieldElement, +} + // The verifier reads proofs in place from their rkyv archive; archived field // elements are viewed as native ones, which is only valid on little-endian. #[cfg(not(target_endian = "little"))] @@ -390,14 +409,13 @@ pub trait IsStarkVerifier< step_size, &next_row_cols, ); - let next_row_flags = crate::ood::next_row_col_flags(ood_full.width, &next_row_cols); let (deep_poly_evaluations, deep_poly_evaluations_sym) = match Self::reconstruct_deep_composition_poly_evaluations_for_all_queries( challenges, domain, proof, &ood_full, - &next_row_flags, + &next_row_cols, step_size, ) { Some(pair) => pair, @@ -737,13 +755,83 @@ pub trait IsStarkVerifier< openings_ok & terminal_ok } - #[allow(clippy::too_many_arguments)] + /// Sums that depend only on `challenges` and proof-level OOD/gamma data — + /// identical for every FRI query — computed once instead of once per + /// query. + /// + /// g·z pruning: the trace OOD values come from the reconstructed full grid + /// `ood_full` (current-row block plus the scattered next-row window, zeros + /// elsewhere), not from `proof.trace_ood_evaluations()` which now carries + /// only the current-row block. Pruned positions are zero in both the grid + /// and `trace_term_coeffs`, so next rows sum only the window columns. + fn compute_query_invariant_deep_terms( + challenges: &Challenges, + proof: StarkProofView<'_, Field, FieldExtension, PI>, + ood_full: &Table, + next_row_cols: &[usize], + step_size: usize, + ) -> Option> { + let ood_evaluations_table_height = ood_full.height; + let ood_evaluations_table_width = ood_full.width; + let ood_data = ood_full.row_major_data(); + let trace_term_coeffs = &challenges.trace_term_coeffs; + + if trace_term_coeffs.is_empty() + || trace_term_coeffs.len() * trace_term_coeffs[0].len() + != ood_evaluations_table_height * ood_evaluations_table_width + { + return None; + } + + let mut ood_row_sum = Vec::with_capacity(ood_evaluations_table_height); + for row_idx in 0..ood_evaluations_table_height { + let ood_row = &ood_data[row_idx * ood_evaluations_table_width + ..(row_idx + 1) * ood_evaluations_table_width]; + let mut sum = FieldElement::::zero(); + if row_idx < step_size { + for col_idx in 0..ood_evaluations_table_width { + sum += &trace_term_coeffs[col_idx][row_idx] * &ood_row[col_idx]; + } + } else { + // Next-row row: off-window columns contribute coeff·0 with a + // zero coeff too, so the window-only sum is exact. + for &col_idx in next_row_cols { + sum += &trace_term_coeffs[col_idx][row_idx] * &ood_row[col_idx]; + } + } + ood_row_sum.push(sum); + } + + let composition_parts_ood = proof.composition_poly_parts_ood_evaluation(); + let number_of_parts = composition_parts_ood.len(); + let z_pow = challenges.z.pow(number_of_parts); + + // A malformed proof/challenge set can advertise more composition + // parts than sampled gammas; reject rather than silently truncate + // the sum below. + if challenges.gammas.len() < number_of_parts { + return None; + } + let mut h_sum_zpow = FieldElement::::zero(); + for (h_i_zpower, gamma) in composition_parts_ood.iter().zip(challenges.gammas.iter()) { + h_sum_zpow += h_i_zpower * gamma; + } + + Some(QueryInvariantDeepTerms { + ood_row_sum, + ood_width: ood_evaluations_table_width, + number_of_parts, + z_pow, + h_sum_zpow, + }) + } + fn reconstruct_deep_composition_poly_evaluations_for_all_queries( challenges: &Challenges, domain: &VerifierDomain, proof: StarkProofView<'_, Field, FieldExtension, PI>, ood_full: &Table, - next_row_flags: &[bool], + next_row_cols: &[usize], step_size: usize, ) -> Option> { let num_queries = challenges.iotas.len(); @@ -768,6 +856,14 @@ pub trait IsStarkVerifier< let primitive_root = &Field::get_primitive_root_of_unity(domain.root_order as u64) .expect("verifier domain root_order is a valid power of two"); + let query_invariant_terms = Self::compute_query_invariant_deep_terms( + challenges, + proof, + ood_full, + next_row_cols, + step_size, + )?; + for (i, iota) in challenges.iotas.iter().enumerate() { let opening = proof.deep_poly_opening(i); @@ -786,22 +882,6 @@ pub trait IsStarkVerifier< .map(|a| a.evaluations()) .unwrap_or(&[]); - let evaluation_point = Self::query_challenge_to_evaluation_point(*iota, false, domain); - deep_poly_evaluations.push(Self::reconstruct_deep_composition_poly_evaluation( - proof, - &evaluation_point, - primitive_root, - challenges, - lde_precomputed, - lde_main, - lde_aux, - opening.composition_poly().evaluations(), - ood_full, - next_row_flags, - step_size, - )?); - - // Mirror for the symmetric query point. let lde_precomputed_sym: &[FieldElement] = opening .precomputed_trace_polys() .map(|p| p.evaluations_sym()) @@ -813,50 +893,67 @@ pub trait IsStarkVerifier< .map(|a| a.evaluations_sym()) .unwrap_or(&[]); - let evaluation_point = Self::query_challenge_to_evaluation_point(*iota, true, domain); - deep_poly_evaluations_sym.push(Self::reconstruct_deep_composition_poly_evaluation( - proof, - &evaluation_point, - primitive_root, - challenges, - lde_precomputed_sym, - lde_main_sym, - lde_aux_sym, - opening.composition_poly().evaluations_sym(), - ood_full, - next_row_flags, - step_size, - )?); + let evaluation_point = Self::query_challenge_to_evaluation_point(*iota, false, domain); + let evaluation_point_sym = + Self::query_challenge_to_evaluation_point(*iota, true, domain); + let (evaluation, evaluation_sym) = + Self::reconstruct_deep_composition_poly_evaluation_pair( + &evaluation_point, + &evaluation_point_sym, + primitive_root, + challenges, + &query_invariant_terms, + next_row_cols, + step_size, + lde_precomputed, + lde_main, + lde_aux, + opening.composition_poly().evaluations(), + lde_precomputed_sym, + lde_main_sym, + lde_aux_sym, + opening.composition_poly().evaluations_sym(), + )?; + deep_poly_evaluations.push(evaluation); + deep_poly_evaluations_sym.push(evaluation_sym); } Some((deep_poly_evaluations, deep_poly_evaluations_sym)) } + /// Reconstructs the deep composition polynomial evaluation at a query's + /// point and its symmetric counterpart together. Rewriting the per-element + /// trace term `coeff*(base-ood)*denom` as `denom*(coeff*base - coeff*ood)` + /// isolates `coeff*ood` (identical for both points, hoisted into + /// `query_invariant_terms`) from `coeff*base` (per-point), so both points + /// share the OOD walk and a single batch-inverse for their denominators. + /// g·z pruning restricts next rows (`row_idx >= step_size`) to the + /// transition-window columns `next_row_cols` — all other next-row + /// coefficients are zero, so those terms vanish from both sums. #[allow(clippy::too_many_arguments)] - fn reconstruct_deep_composition_poly_evaluation<'b>( - proof: StarkProofView<'_, Field, FieldExtension, PI>, + fn reconstruct_deep_composition_poly_evaluation_pair<'b>( evaluation_point: &FieldElement, + evaluation_point_sym: &FieldElement, primitive_root: &FieldElement, challenges: &Challenges, + query_invariant_terms: &QueryInvariantDeepTerms, + next_row_cols: &[usize], + step_size: usize, lde_trace_precomputed_evaluations: &'b [FieldElement], lde_trace_main_evaluations: &'b [FieldElement], lde_trace_aux_evaluations: &[FieldElement], lde_composition_poly_parts_evaluation: &[FieldElement], - ood_full: &Table, - next_row_flags: &[bool], - step_size: usize, - ) -> Option> { - // g·z pruning: read from the reconstructed full grid (current-row block - // plus the scattered next-row window, zeros elsewhere), not from - // `proof.trace_ood_evaluations()` which now carries only the current-row - // block. Flatten once for the hot loop below. - let ood_evaluations_table_height = ood_full.height; - let ood_evaluations_table_width = ood_full.width; - let ood_data = ood_full.row_major_data(); + lde_trace_precomputed_evaluations_sym: &'b [FieldElement], + lde_trace_main_evaluations_sym: &'b [FieldElement], + lde_trace_aux_evaluations_sym: &[FieldElement], + lde_composition_poly_parts_evaluation_sym: &[FieldElement], + ) -> Option<(FieldElement, FieldElement)> { + let ood_evaluations_table_height = query_invariant_terms.ood_row_sum.len(); + let ood_evaluations_table_width = query_invariant_terms.ood_width; let trace_term_coeffs = &challenges.trace_term_coeffs; // Base columns are supplied as two slices (precomputed ‖ main) that the - // prover concatenated in this order; `num_base` is their combined width - // and `base_at` indexes into them as if concatenated, without allocating. + // prover concatenated in this order; `num_base`/`base_at` index into + // them as if concatenated, without allocating. let num_precomputed = lde_trace_precomputed_evaluations.len(); let num_base = num_precomputed + lde_trace_main_evaluations.len(); let base_at = move |col: usize| -> &'b FieldElement { @@ -866,76 +963,115 @@ pub trait IsStarkVerifier< &lde_trace_main_evaluations[col - num_precomputed] } }; + let num_precomputed_sym = lde_trace_precomputed_evaluations_sym.len(); + let num_base_sym = num_precomputed_sym + lde_trace_main_evaluations_sym.len(); + let base_at_sym = move |col: usize| -> &'b FieldElement { + if col < num_precomputed_sym { + &lde_trace_precomputed_evaluations_sym[col] + } else { + &lde_trace_main_evaluations_sym[col - num_precomputed_sym] + } + }; - // Runtime guard: a malformed proof may supply opening evaluations whose - // column count does not match the OOD table width, or whose composition - // poly parts count does not match the proof's `composition_poly_parts_ood_evaluation`. - // Without these checks the indexing below would panic in release builds. - if num_base + lde_trace_aux_evaluations.len() != ood_evaluations_table_width { + // Runtime guards: a malformed proof may supply opening evaluations + // whose column count does not match the OOD table width, or whose + // regular/symmetric base-column split disagree. Without these checks + // the indexing below would panic in release builds. + if num_base != num_base_sym { return None; } - if trace_term_coeffs.is_empty() - || trace_term_coeffs.len() * trace_term_coeffs[0].len() - != ood_evaluations_table_height * ood_evaluations_table_width + if num_base + lde_trace_aux_evaluations.len() != ood_evaluations_table_width + || num_base + lde_trace_aux_evaluations_sym.len() != ood_evaluations_table_width { return None; } - let mut denoms_trace = Vec::with_capacity(ood_evaluations_table_height); + // Build both denominator sets (regular, then symmetric) and invert + // them together in a single batch. + let mut denoms = Vec::with_capacity(2 * ood_evaluations_table_height); + let mut current_z = challenges.z.clone(); + for _ in 0..ood_evaluations_table_height { + denoms.push(evaluation_point - ¤t_z); + current_z = primitive_root * ¤t_z; + } let mut current_z = challenges.z.clone(); for _ in 0..ood_evaluations_table_height { - denoms_trace.push(evaluation_point - ¤t_z); + denoms.push(evaluation_point_sym - ¤t_z); current_z = primitive_root * ¤t_z; } // A malformed proof can land an OOD evaluation point on the LDE coset, reject. - FieldElement::inplace_batch_inverse(&mut denoms_trace).ok()?; - - let trace_term = (0..ood_evaluations_table_width) - .zip(&challenges.trace_term_coeffs) - .fold(FieldElement::zero(), |trace_terms, (col_idx, coeff_row)| { - let opened_next_row = next_row_flags[col_idx]; - let trace_i = (0..ood_evaluations_table_height).zip(coeff_row).fold( - FieldElement::zero(), - |trace_t, (row_idx, coeff)| { - // g·z pruning: the next-row block opens only transition- - // window columns. Skip every other next-row opening — its - // coefficient is zero, so the term is vacuous, and skipping - // it is where the verifier/recursion cycle saving lands. - if row_idx >= step_size && !opened_next_row { - return trace_t; - } - let ood_val = &ood_data[row_idx * ood_evaluations_table_width + col_idx]; - // Stay in base when we can: F: IsSubFieldOf gives F - E -> E. - let diff: FieldElement = if col_idx < num_base { - base_at(col_idx) - ood_val - } else { - &lde_trace_aux_evaluations[col_idx - num_base] - ood_val - }; - let poly_evaluation = diff * &denoms_trace[row_idx]; - trace_t + &poly_evaluation * coeff - }, - ); - trace_terms + trace_i - }); + FieldElement::inplace_batch_inverse(&mut denoms).ok()?; + let (denoms_trace, denoms_trace_sym) = denoms.split_at(ood_evaluations_table_height); + + let mut trace_term = FieldElement::::zero(); + let mut trace_term_sym = FieldElement::::zero(); + for row_idx in 0..ood_evaluations_table_height { + let ood_row_sum = &query_invariant_terms.ood_row_sum[row_idx]; + let mut base_row_sum = FieldElement::::zero(); + let mut base_row_sum_sym = FieldElement::::zero(); + if row_idx < step_size { + for (col_idx, coeff_col) in trace_term_coeffs.iter().enumerate() { + let coeff = &coeff_col[row_idx]; + if col_idx < num_base { + // F: IsSubFieldOf gives the cheap asymmetric F * E -> E product. + base_row_sum += base_at(col_idx) * coeff; + base_row_sum_sym += base_at_sym(col_idx) * coeff; + } else { + let aux_idx = col_idx - num_base; + base_row_sum += coeff * &lde_trace_aux_evaluations[aux_idx]; + base_row_sum_sym += coeff * &lde_trace_aux_evaluations_sym[aux_idx]; + } + } + } else { + // g·z pruning: the next-row block opens only transition-window + // columns; every other column's coefficient is zero + // (`build_pruned_trace_term_coeffs`), so summing the window + // alone is exact — and skipping the rest is where the + // verifier/recursion cycle saving lands. + for &col_idx in next_row_cols { + let coeff = &trace_term_coeffs[col_idx][row_idx]; + if col_idx < num_base { + base_row_sum += base_at(col_idx) * coeff; + base_row_sum_sym += base_at_sym(col_idx) * coeff; + } else { + let aux_idx = col_idx - num_base; + base_row_sum += coeff * &lde_trace_aux_evaluations[aux_idx]; + base_row_sum_sym += coeff * &lde_trace_aux_evaluations_sym[aux_idx]; + } + } + } + trace_term += &denoms_trace[row_idx] * &(&base_row_sum - ood_row_sum); + trace_term_sym += &denoms_trace_sym[row_idx] * &(&base_row_sum_sym - ood_row_sum); + } - let composition_parts_ood = proof.composition_poly_parts_ood_evaluation(); - let number_of_parts = lde_composition_poly_parts_evaluation.len(); - let z_pow = &challenges.z.pow(number_of_parts); - - // A malformed proof can make evaluation_point == z^N, reject. - let denom_composition = (evaluation_point - z_pow).inv().ok()?; - let mut h_terms = FieldElement::zero(); - for (j, h_i_upsilon) in lde_composition_poly_parts_evaluation.iter().enumerate() { - // Bounds-check via `.get(j)?`: a malformed opening may have more - // parts than the proof header advertises. - let h_i_zpower = composition_parts_ood.get(j)?; - let gamma = challenges.gammas.get(j)?; - let h_i_term = (h_i_upsilon - h_i_zpower) * gamma; - h_terms += h_i_term; + let number_of_parts = query_invariant_terms.number_of_parts; + // Also rejects a per-query opening length that disagrees with the + // proof-level `number_of_parts`, not just a regular/symmetric mismatch. + if lde_composition_poly_parts_evaluation.len() != number_of_parts + || lde_composition_poly_parts_evaluation_sym.len() != number_of_parts + { + return None; + } + let z_pow = &query_invariant_terms.z_pow; + + // A malformed proof can make evaluation_point == z_pow, reject. + let mut denom_composition_pair = [evaluation_point - z_pow, evaluation_point_sym - z_pow]; + FieldElement::inplace_batch_inverse(&mut denom_composition_pair).ok()?; + let [denom_composition, denom_composition_sym] = denom_composition_pair; + + let mut h_sum = FieldElement::::zero(); + let mut h_sum_sym = FieldElement::::zero(); + for j in 0..number_of_parts { + let h_i_upsilon = &lde_composition_poly_parts_evaluation[j]; + let h_i_upsilon_sym = &lde_composition_poly_parts_evaluation_sym[j]; + let gamma = &challenges.gammas[j]; + h_sum += h_i_upsilon * gamma; + h_sum_sym += h_i_upsilon_sym * gamma; } - h_terms *= denom_composition; + let h_terms = (&h_sum - &query_invariant_terms.h_sum_zpow) * denom_composition; + let h_terms_sym = (&h_sum_sym - &query_invariant_terms.h_sum_zpow) * denom_composition_sym; - Some(trace_term + h_terms) + Some((trace_term + h_terms, trace_term_sym + h_terms_sym)) } /// Verifies one or more STARK proofs with their corresponding AIRs. diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 821b2771d..15344d138 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -36,8 +36,8 @@ tiny-keccak = { version = "2.0", features = ["keccak"] } # `compute_precomputed_commitment_for_testing`. Only active under cargo test/bench. stark = { path = "../crypto/stark", features = ["test-utils"] } # Device-resident LDE handles for the cuda-gated GPU constraint-interp parity -# test (`tests/gpu_constraint_interp_real.rs`). Its build.rs stubs empty PTX when -# nvcc is absent, so this compiles on CPU-only hosts too; the test itself is +# test (`tests/gpu_constraint_interp_real.rs`). Its build.rs stubs an empty cubin +# when nvcc is absent, so this compiles on CPU-only hosts too; the test itself is # `#[cfg(feature = "cuda")]` and only runs with a GPU. math-cuda = { path = "../crypto/math-cuda" } diff --git a/scripts/gpu_test.sh b/scripts/gpu_test.sh index 661339f11..1c5458a67 100755 --- a/scripts/gpu_test.sh +++ b/scripts/gpu_test.sh @@ -14,12 +14,10 @@ # group failed, which fails the workflow job and blocks the merge. # # Env: -# CUDARC_PIN cudarc CUDA-version feature to pin (default cuda-12080). See the sed below. # SYSROOT_DIR rv64 sysroot (default /opt/lambda-vm-sysroot, provisioned by the template). set -euo pipefail -CUDARC_PIN="${CUDARC_PIN:-cuda-12080}" export SYSROOT_DIR="${SYSROOT_DIR:-/opt/lambda-vm-sysroot}" log() { printf '\n=== %s ===\n' "$*"; } @@ -37,26 +35,9 @@ nvcc --version | tail -n 2 nvidia-smi nvidia-smi --query-gpu=name,driver_version,compute_cap --format=csv,noheader -# --- Pin cudarc so it binds a fixed driver-symbol set -------------------------- -# crypto/math-cuda/Cargo.toml uses `cuda-version-from-build-system` + `fallback-latest`; -# when detection falls back to "latest", cudarc requests symbols some boxes' driver doesn't -# export (e.g. cuDevSmResourceSplit / cuCtxGetDevice_v2) -> runtime panic. Pinning to a fixed, -# conservative CUDA version binds a known driver-symbol set instead. (This is cudarc's -# host-side driver-API floor — independent of the PTX/driver version the offer filter targets.) -log "pinning cudarc to $CUDARC_PIN" -# Guard the sed anchors: if math-cuda's cudarc features are ever renamed/reformatted, a silent -# no-op here would bring the fallback-latest driver-symbol panic back with a confusing signature. -for anchor in '"cuda-version-from-build-system"' '"fallback-latest"'; do - grep -qF "$anchor" crypto/math-cuda/Cargo.toml \ - || { echo "ERROR: sed anchor $anchor not found in crypto/math-cuda/Cargo.toml — update this script's cudarc pin" >&2; exit 1; } -done -# Restore the tracked file on exit so a manual run on a dev box doesn't leave the tree dirty -# (CI doesn't need this — the workflow re-checks-out before every run — but it's harmless there). -CUDARC_TOML_BACKUP="$(mktemp)" -cp crypto/math-cuda/Cargo.toml "$CUDARC_TOML_BACKUP" -trap 'cp "$CUDARC_TOML_BACKUP" crypto/math-cuda/Cargo.toml; rm -f "$CUDARC_TOML_BACKUP"' EXIT -sed -i "s/\"cuda-version-from-build-system\"/\"${CUDARC_PIN}\"/; /\"fallback-latest\"/d" \ - crypto/math-cuda/Cargo.toml +# cudarc's CUDA-version pin now lives permanently in crypto/math-cuda/Cargo.toml +# (feature `cuda-12080`), so this script no longer patches the manifest. Kernels +# are AOT-compiled to cubin by build.rs, so no PTX/driver-version juggling either. # --- Build the guest ELFs the tests prove --------------------------------------- # math-cuda parity needs none; cuda_path_integration / cuda_fallback prove an asm ELF; the