From 9a68c6088f62825eb2d0ea8d3e48789ad3ecab6e Mon Sep 17 00:00:00 2001 From: Chris Fields Date: Sat, 17 Oct 2026 01:45:51 -0500 Subject: [PATCH] x86: claim affine2p output lines with PREFETCHW; drop input prefetches On a Xeon Gold 6150 (Skylake-SP), gap-affine 2-piece high-memory CIGAR was store-buffer bound: the AVX2 kernel's five output stores go to fresh lines, whose ownership requests (RFOs) missed L2 ~1.8x as often as in C's kernel, leaving 4.67 G cycles of store-buffer-full stalls against C's 0.38 G. The kernel now issues PREFETCHW on its five outputs 32 diagonals (two lines) ahead, and all x86 input prefetches are removed (the hardware prefetcher covers those sequential streams; they did not help in A/B). 32 beat 64 and 128 diagonals. Xeon, time vs before: affine2p CIGAR 0.88x, banded 0.65x, 10 kb 0.89x, score 0.93x, BiWFA 0.95x; store-buffer stalls 4.63 G -> 0.43 G cycles. EPYC 7713: 2-6% faster on most affine2p cases, the rest within noise. ARM is unchanged. Output identical in every A/B case on both machines. Co-Authored-By: Claude Opus 5.5 --- .claude/PERFORMANCE_LOG.md | 39 +++++++++++++++++++++++++++++++++++++ .claude/PROGRESS.md | 6 ++++++ src/aligner.rs | 40 -------------------------------------- src/compute/affine.rs | 7 ------- src/compute/affine2p.rs | 21 ++++++++++---------- 5 files changed, 56 insertions(+), 57 deletions(-) diff --git a/.claude/PERFORMANCE_LOG.md b/.claude/PERFORMANCE_LOG.md index c0ae194..e6f53e1 100644 --- a/.claude/PERFORMANCE_LOG.md +++ b/.claude/PERFORMANCE_LOG.md @@ -477,3 +477,42 @@ extend. and a scan that skips the rest of a chunk. - To confirm on x86 with `bench/sweep.py` (expect affine2p ultralow near or below C-native). + +## 2026-10-17: x86 prefetches: claim output lines instead (HPCBio/wfa2lib-rs#31) + +After #49, the Xeon Gold 6150 (Skylake-SP) was the only machine where +gap-affine 2-piece high-memory CIGAR wasn't faster than C (1.01×; banded 1.28×, +against 0.75–0.77× on the EPYC 7713). Rust's CIGAR-minus-score overhead there +was +32% against +8% for C, and +8–10% for Rust on the M2 and EPYC. + +- `perf` (`l2_rqsts.rfo_miss`, `resource_stalls.sb`, sampled per instruction): + the store buffer was full for 4.67 G cycles against C's 0.38 G, about 60% + of them on the AVX2 affine2p kernel's five output stores. Those stores + missed L2 for ownership (RFO) 158 M times against about 88 M in C's kernel. + In high memory the outputs are fresh lines. +- Ruled out by A/B or counters: per-component victim outputs (no change), + split stores (12% vs C's 10%), slab cache coloring (helps only banded on + the Xeon, costs up to 4% on the EPYC), and the input prefetches (not the + cause, but they didn't help either). +- Fix: drop all x86 input prefetches (the hardware prefetcher covers these + sequential streams) and issue `PREFETCHW` on the five affine2p outputs, + 32 diagonals (two lines) ahead. + +A/B at `45f419e`, best of 3, time relative to the unchanged build, with +output identical in all cases: + +| Case | Xeon: 32 | Xeon: 64 | Xeon: 128 | EPYC: 32 | +|---|---|---|---|---| +| affine2p CIGAR (m1k_5) | **0.88** | 0.98 | 1.08 | 0.96 | +| affine2p banded CIGAR (m1k_20) | **0.65** | 0.91 | 1.05 | 1.03 | +| affine2p CIGAR (l10k_5) | **0.89** | 0.92 | 0.96 | 1.00 | +| affine2p score | **0.93** | 0.95 | 0.97 | 0.94 | +| affine2p BiWFA | **0.95** | 0.98 | 1.03 | 0.96 | +| gap-affine CIGAR / score | 0.98 / 0.98 | | | 1.01 / 0.97 | +| edit (control) | 0.99 | | | 0.99 | + +- On the Xeon, store-buffer stalls on affine2p CIGAR dropped from 4.63 G to + 0.43 G cycles, about C's level. Against C: affine2p CIGAR 1.01× → 0.89×, + banded 1.28× → 0.83×, 10 kb 0.97× → 0.87×. +- On the EPYC it is about the same as just removing the prefetches (2–6% + faster on most affine2p cases, the rest within noise). ARM is unchanged. diff --git a/.claude/PROGRESS.md b/.claude/PROGRESS.md index c7fe60d..a21bb6f 100644 --- a/.claude/PROGRESS.md +++ b/.claude/PROGRESS.md @@ -172,4 +172,10 @@ reports, inbound C ports) is tracked in HPCBio/wfa2lib-rs#19. breakpoint checks were 73% of affine2p BiWFA time and scalar except the NEON M scan. affine2p ultralow is now 0.38–0.52× C on ARM (was ~1.0×); output unchanged. +- **2026-10-17:** x86 prefetches (HPCBio/wfa2lib-rs#31): on the Xeon + (Skylake-SP), affine2p high-memory CIGAR waited on ownership requests for + its fresh output lines (store buffer full). The input prefetches are removed + and the affine2p kernel issues `PREFETCHW` on its outputs. Xeon: 5–35% + faster on affine2p high memory, now 0.83–0.89× C; EPYC slightly faster; + output unchanged. diff --git a/src/aligner.rs b/src/aligner.rs index b79ce95..8402088 100644 --- a/src/aligner.rs +++ b/src/aligner.rs @@ -1712,24 +1712,6 @@ impl WavefrontAligner { // Allocation range (also resizes the null/victim wavefronts) let (alloc_lo, alloc_hi) = self.output_alloc_range(score, lo, hi); - // Prefetch input wavefront offset arrays into L1 cache. - #[cfg(target_arch = "x86_64")] - unsafe { - use std::arch::x86_64::{_mm_prefetch, _MM_HINT_T0}; - macro_rules! prefetch_wf { - ($ptr:expr, $lo:expr) => { - if !$ptr.is_null() { - let base = (*$ptr).centered_offsets.offset($lo as isize) as *const i8; - _mm_prefetch(base, _MM_HINT_T0); - _mm_prefetch(base.add(64), _MM_HINT_T0); - } - } - } - prefetch_wf!(ptr_m_misms, lo); - prefetch_wf!(ptr_m_open, lo); - prefetch_wf!(ptr_i1_ext, lo); - prefetch_wf!(ptr_d1_ext, lo); - } // Initialize input wavefront ends using pointers (no slab re-lookup) unsafe { @@ -1953,28 +1935,6 @@ impl WavefrontAligner { // Allocation range (also resizes the null/victim wavefronts) let (alloc_lo, alloc_hi) = self.output_alloc_range(score, lo, hi); - // Prefetch input wavefront offset arrays into L1 cache. - // These prefetches overlap with the init_ends work below, hiding L1 miss latency. - #[cfg(target_arch = "x86_64")] - unsafe { - use std::arch::x86_64::{_mm_prefetch, _MM_HINT_T0}; - macro_rules! prefetch_wf { - ($ptr:expr, $lo:expr) => { - if !$ptr.is_null() { - let base = (*$ptr).centered_offsets.offset($lo as isize) as *const i8; - _mm_prefetch(base, _MM_HINT_T0); - _mm_prefetch(base.add(64), _MM_HINT_T0); - } - } - } - prefetch_wf!(ptr_m_misms, lo); - prefetch_wf!(ptr_m_open1, lo); - prefetch_wf!(ptr_m_open2, lo); - prefetch_wf!(ptr_i1_ext, lo); - prefetch_wf!(ptr_i2_ext, lo); - prefetch_wf!(ptr_d1_ext, lo); - prefetch_wf!(ptr_d2_ext, lo); - } // Initialize input wavefront ends using pointers (no slab re-lookup) unsafe { diff --git a/src/compute/affine.rs b/src/compute/affine.rs index 6e39192..402654b 100644 --- a/src/compute/affine.rs +++ b/src/compute/affine.rs @@ -248,15 +248,8 @@ unsafe fn compute_affine_idm_avx2( ); let avx_end = lo + (count & !7); - const PF_DIST: isize = 16; while k < avx_end { - // Prefetch next chunk of all 4 input arrays - _mm_prefetch(m_open.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(i1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(d1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(m_misms.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - // I1: load m_open[k-1] and i1_ext[k-1] let v_m_open_km1 = _mm256_loadu_si256(m_open.offset(k as isize - 1) as *const __m256i); let v_i1_ext_km1 = _mm256_loadu_si256(i1_ext.offset(k as isize - 1) as *const __m256i); diff --git a/src/compute/affine2p.rs b/src/compute/affine2p.rs index c8176be..065c213 100644 --- a/src/compute/affine2p.rs +++ b/src/compute/affine2p.rs @@ -141,18 +141,19 @@ pub fn compute_affine2p_idm( ); let avx_end = lo + (count & !7); - // Prefetch distance: 16 diagonals ahead (2 AVX iterations = 64 bytes) - const PF_DIST: isize = 16; + const PFW_DIST: isize = 32; while k < avx_end { - // Prefetch next chunk of all 7 input arrays - _mm_prefetch(m_open1.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(m_open2.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(i1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(i2_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(d1_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(d2_ext.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); - _mm_prefetch(m_misms.offset(k as isize + PF_DIST) as *const i8, _MM_HINT_T0); + // Claim the output lines for writing ahead of the stores + // (PREFETCHW): in high memory they are fresh, and on + // Skylake their RFO misses fill the store buffer. + // PFW_DIST diagonals = 2 lines ahead: on a Xeon Gold 6150 + // 32 beat 64 and 128 (HPCBio/wfa2lib-rs#31). + _mm_prefetch(m_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(i1_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(i2_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(d1_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); + _mm_prefetch(d2_curr.wrapping_offset(k as isize + PFW_DIST) as *const i8, _MM_HINT_ET0); // I1, I2 (shifted by -1) let v_i1 = _mm256_add_epi32(_mm256_max_epi32(