diff --git a/.claude/PERFORMANCE_LOG.md b/.claude/PERFORMANCE_LOG.md index c0ae194..e6f53e1 100644 --- a/.claude/PERFORMANCE_LOG.md +++ b/.claude/PERFORMANCE_LOG.md @@ -477,3 +477,42 @@ extend. and a scan that skips the rest of a chunk. - To confirm on x86 with `bench/sweep.py` (expect affine2p ultralow near or below C-native). + +## 2026-10-17: x86 prefetches: claim output lines instead (HPCBio/wfa2lib-rs#31) + +After #49, the Xeon Gold 6150 (Skylake-SP) was the only machine where +gap-affine 2-piece high-memory CIGAR wasn't faster than C (1.01×; banded 1.28×, +against 0.75–0.77× on the EPYC 7713). Rust's CIGAR-minus-score overhead there +was +32% against +8% for C, and +8–10% for Rust on the M2 and EPYC. + +- `perf` (`l2_rqsts.rfo_miss`, `resource_stalls.sb`, sampled per instruction): + the store buffer was full for 4.67 G cycles against C's 0.38 G, about 60% + of them on the AVX2 affine2p kernel's five output stores. Those stores + missed L2 for ownership (RFO) 158 M times against about 88 M in C's kernel. + In high memory the outputs are fresh lines. +- Ruled out by A/B or counters: per-component victim outputs (no change), + split stores (12% vs C's 10%), slab cache coloring (helps only banded on + the Xeon, costs up to 4% on the EPYC), and the input prefetches (not the + cause, but they didn't help either). +- Fix: drop all x86 input prefetches (the hardware prefetcher covers these + sequential streams) and issue `PREFETCHW` on the five affine2p outputs, + 32 diagonals (two lines) ahead. + +A/B at `45f419e`, best of 3, time relative to the unchanged build, with +output identical in all cases: + +| Case | Xeon: 32 | Xeon: 64 | Xeon: 128 | EPYC: 32 | +|---|---|---|---|---| +| affine2p CIGAR (m1k_5) | **0.88** | 0.98 | 1.08 | 0.96 | +| affine2p banded CIGAR (m1k_20) | **0.65** | 0.91 | 1.05 | 1.03 | +| affine2p CIGAR (l10k_5) | **0.89** | 0.92 | 0.96 | 1.00 | +| affine2p score | **0.93** | 0.95 | 0.97 | 0.94 | +| affine2p BiWFA | **0.95** | 0.98 | 1.03 | 0.96 | +| gap-affine CIGAR / score | 0.98 / 0.98 | | | 1.01 / 0.97 | +| edit (control) | 0.99 | | | 0.99 | + +- On the Xeon, store-buffer stalls on affine2p CIGAR dropped from 4.63 G to + 0.43 G cycles, about C's level. Against C: affine2p CIGAR 1.01× → 0.89×, + banded 1.28× → 0.83×, 10 kb 0.97× → 0.87×. +- On the EPYC it is about the same as just removing the prefetches (2–6% + faster on most affine2p cases, the rest within noise). ARM is unchanged. diff --git a/.claude/PROGRESS.md b/.claude/PROGRESS.md index c7fe60d..a21bb6f 100644 --- a/.claude/PROGRESS.md +++ b/.claude/PROGRESS.md @@ -172,4 +172,10 @@ reports, inbound C ports) is tracked in HPCBio/wfa2lib-rs#19. breakpoint checks were 73% of affine2p BiWFA time and scalar except the NEON M scan. affine2p ultralow is now 0.38–0.52× C on ARM (was ~1.0×); output unchanged. +- **2026-10-17:** x86 prefetches (HPCBio/wfa2lib-rs#31): on the Xeon + (Skylake-SP), affine2p high-memory CIGAR waited on ownership requests for + its fresh output lines (store buffer full). The input prefetches are removed + and the affine2p kernel issues `PREFETCHW` on its outputs. Xeon: 5–35% + faster on affine2p high memory, now 0.83–0.89× C; EPYC slightly faster; + output unchanged. diff --git a/src/aligner.rs b/src/aligner.rs index b79ce95..8402088 100644 --- a/src/aligner.rs +++ b/src/aligner.rs @@ -1712,24 +1712,6 @@ impl WavefrontAligner { // Allocation range (also resizes the null/victim wavefronts) let (alloc_lo, alloc_hi) = self.output_alloc_range(score, lo, hi); - // Prefetch input wavefront offset arrays into L1 cache. - #[cfg(target_arch = "x86_64")] - unsafe { - use std::arch::x86_64::{_mm_prefetch, _MM_HINT_T0}; - macro_rules! prefetch_wf { - ($ptr:expr, $lo:expr) => { - if !$ptr.is_null() { - let base = (*$ptr).centered_offsets.offset($lo as isize) as *const i8; - _mm_prefetch(base, _MM_HINT_T0); - _mm_prefetch(base.add(64), _MM_HINT_T0); - } - } - } - prefetch_wf!(ptr_m_misms, lo); - prefetch_wf!(ptr_m_open, lo); - prefetch_wf!(ptr_i1_ext, lo); - prefetch_wf!(ptr_d1_ext, lo); - } // Initialize input wavefront ends using pointers (no slab re-lookup) unsafe { @@ -1953,28 +1935,6 @@ impl WavefrontAligner { // Allocation range (also resizes the null/victim wavefronts) let (alloc_lo, alloc_hi) = self.output_alloc_range(score, lo, hi); - // Prefetch input wavefront offset arrays into L1 cache. - // These prefetches overlap with the init_ends work below, hiding L1 miss latency. - #[cfg(target_arch = "x86_64")] - unsafe { - use std::arch::x86_64::{_mm_prefetch, _MM_HINT_T0}; - macro_rules! prefetch_wf { - ($ptr:expr, $lo:expr) => { - if !$ptr.is_null() { - let base = (*$ptr).centered_offsets.offset($lo as isize) as *const i8; - _mm_prefetch(base, _MM_HINT_T0); - _mm_prefetch(base.add(64), _MM_HINT_T0); - } - } - } - prefetch_wf!(ptr_m_misms, lo); - prefetch_wf!(ptr_m_open1, lo); - prefetch_wf!(ptr_m_open2, lo); - prefetch_wf!(ptr_i1_ext, lo); - prefetch_wf!(ptr_i2_ext, lo); - prefetch_wf!(ptr_d1_ext, lo); - prefetch_wf!(ptr_d2_ext, lo); - } // Initialize input wavefront ends using pointers (no slab re-lookup) unsafe { diff --git a/src/compute/affine.rs b/src/compute/affine.rs index 6e39192..402654b 100644 --- a/src/compute/affine.rs +++ b/src/compute/affine.rs @@ -248,15 +248,8 @@ unsafe fn compute_affine_idm_avx2( ); let avx_end = lo + (count & !7); - const PF_DIST: isize = 16; while k < avx_end { - // Prefetch next chunk of all 4 input arrays - _mm_prefetch(m_open.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(i1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(d1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(m_misms.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - // I1: load m_open[k-1] and i1_ext[k-1] let v_m_open_km1 = _mm256_loadu_si256(m_open.offset(k as isize - 1) as *const __m256i); let v_i1_ext_km1 = _mm256_loadu_si256(i1_ext.offset(k as isize - 1) as *const __m256i); diff --git a/src/compute/affine2p.rs b/src/compute/affine2p.rs index c8176be..065c213 100644 --- a/src/compute/affine2p.rs +++ b/src/compute/affine2p.rs @@ -141,18 +141,19 @@ pub fn compute_affine2p_idm( ); let avx_end = lo + (count & !7); - // Prefetch distance: 16 diagonals ahead (2 AVX iterations = 64 bytes) - const PF_DIST: isize = 16; + const PFW_DIST: isize = 32; while k < avx_end { - // Prefetch next chunk of all 7 input arrays - _mm_prefetch(m_open1.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(m_open2.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(i1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(i2_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(d1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(d2_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(m_misms.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); + // Claim the output lines for writing ahead of the stores + // (PREFETCHW): in high memory they are fresh, and on + // Skylake their RFO misses fill the store buffer. + // PFW_DIST diagonals = 2 lines ahead: on a Xeon Gold 6150 + // 32 beat 64 and 128 (HPCBio/wfa2lib-rs#31). + _mm_prefetch(m_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(i1_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(i2_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(d1_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(d2_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); // I1, I2 (shifted by -1) let v_i1 = _mm256_add_epi32(_mm256_max_epi32(