From bef65f1a5139289002364fe670df36b365ffc1f8 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 13 May 2026 12:49:18 +0200 Subject: [PATCH] Comment about the RAID optimizations --- raid/x86.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 105 insertions(+), 16 deletions(-) diff --git a/raid/x86.c b/raid/x86.c index a6f7dd57..4254a007 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -5,7 +5,10 @@ #include "gf.h" #include "cpu.h" -/* +/* ================================================================ + * Optimizations notes + * ================================================================ + * * For x86 optimizations you can see: * * Software optimization resources @@ -13,15 +16,111 @@ * * x86, x64 Instruction Latency, Memory Latency and CPUID dumps * http://users.atw.hu/instlatx64/ + * + * Optimization notes: + * - Cache Line Sizing: Intentionally don't process more than 64 bytes + * because 64 is the typical cache block. Processing 128 bytes or more + * doesn't increase performance, and in some cases it even decreases it. + * - Register Pressure: Processing exactly 64 bytes perfectly balances + * the SIMD register file. It allows 128-bit architectures (SSE/NEON) + * to hold 4 data vectors while keeping all necessary GF constants and + * masks in the remaining registers, avoiding slow stack spills. + * - Write-Combining Buffers: The final write of the P,Q,R,S,T parities + * always uses non-temporal stores (movntdq), even for writes smaller + * than the cache line, as the CPU has enough write-combining buffers + * to aggregate the sequential streams and avoid the RFO (Read-For-Ownership) + * memory penalty. + * - Parity Accumulator Locality: The outer loop iterates by block size, + * and the inner loop iterates by disk. This guarantees that the running + * parity accumulators never leave the CPU's SIMD registers during the + * computation. + * - D0 Cauchy Bypass: The final disk processed (D0) always has a + * coefficient of 1 for all parities. It is handled separately to bypass + * the expensive Cauchy matrix lookup and `pshufb` multiplications. + * - Pipeline Dependency Breaking: Galois Field doubling (Horner's method + * multiplication by 2) is strategically placed relative to the XOR + * operations to break serial dependency chains. This allows the CPU's + * out-of-order execution engine to hide the latency of the GF math + * behind memory loads. + */ + +/* + * ================================================================ + * RAID Parity Generation: Q-Parity Horner's Method Implementations + * ================================================================ + * + * This file implements different strategies for computing the Q parity. + * Q parity requires scaling each data disk by a descending power of 2 + * in the Galois Field: + * + * Q = (D_l * 2^l) + (D_{l-1} * 2^{l-1}) + ... + (D_1 * 2^1) + (D_0 * 1) + * + * To efficiently calculate this sequentially using SIMD instructions, we + * use Horner's method. Depending on hardware architecture and whether we + * need to optimize out the final Cauchy matrix multiplication for D0, the + * "multiply by 2" operation can be positioned in three distinct ways: + * + * ------------------------------------------------------------------------- + * CASE 1: Multiplication BEFORE XOR, handling all disks inside the loop + * ------------------------------------------------------------------------- + * Used in GEN2 (RAID6) where all disks can be processed uniformly. + * The accumulator is doubled BEFORE adding the current disk. Because the + * loop runs all the way down to D0 (d >= 0), the final disk is XORed into + * the accumulator and the loop terminates. This naturally leaves D0 with + * a coefficient of 1, as it escapes the doubling step. + * + * Simplified Code: + * Q = D_l; + * for (d = l - 1; d >= 0; --d) { + * Q = Q * 2; // Scale accumulator + * Q = Q ^ D_d; // Add current disk + * } + * + * ------------------------------------------------------------------------- + * CASE 2: Multiplication BEFORE XOR, handling D0 separately + * ------------------------------------------------------------------------- + * Used in GEN3/GEN4 when the loop contains expensive operations (like + * SSSE3 pshufb for Cauchy matrices) that we want to skip for the very + * last disk (which only requires coefficient 1). The Q-parity logic is + * identical to CASE 1, but the final iteration for D0 is unrolled. + * + * Simplified Code: + * Q = D_l; + * for (d = l - 1; d > 0; --d) { + * Q = Q * 2; // Scale accumulator + * Q = Q ^ D_d; // Add current disk + * } + * + * // Final disk (D0) processed outside the loop to skip Cauchy math + * Q = Q * 2; + * Q = Q ^ D_0; + * + * ------------------------------------------------------------------------- + * CASE 3: Multiplication AFTER XOR, pre-scaling the accumulator + * ------------------------------------------------------------------------- + * Used in highly pipelined architectures (like AVX-512) to improve + * instruction-level parallelism. Placing the multiplication AFTER the XOR + * in the loop allows the CPU to load and XOR the next disk while the + * complex doubling logic is still finishing. To ensure the math aligns + * correctly, the initial disk (D_l) must be pre-multiplied before entering + * the loop. + * + * Simplified Code: + * Q = D_l; + * Q = Q * 2; // Pre-scale the initial disk + * + * for (d = l - 1; d > 0; --d) { + * Q = Q ^ D_d; // Add current disk FIRST + * Q = Q * 2; // Scale accumulator AFTER + * } + * + * // Final disk (D0) simply XORed, keeping coefficient 1 + * Q = Q ^ D_0; */ #if defined(CONFIG_X86) && defined(CONFIG_SSE2) /* * GEN1 (RAID5 with xor) SSE2 implementation - * - * Intentionally don't process more than 64 bytes because 64 is the typical - * cache block, and processing 128 bytes doesn't increase performance, and in - * some cases it even decreases it. */ void raid_gen1_sse2(int nd, size_t size, void **vv) { @@ -35,7 +134,6 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) raid_sse_begin(); - /* additional unrolling makes it slower */ for (i = 0; i < size; i += 64) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); @@ -60,10 +158,6 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) #if defined(CONFIG_X86) && defined(CONFIG_AVX2) /* * GEN1 (RAID5 with xor) AVX2 implementation - * - * Intentionally don't process more than 64 bytes because 64 is the typical - * cache block, and processing 128 bytes doesn't increase performance, and in - * some cases it even decreases it. */ void raid_gen1_avx2(int nd, size_t size, void **vv) { @@ -77,7 +171,6 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) raid_avx_begin(); - /* additional unrolling makes it slower */ for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); @@ -100,10 +193,6 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) * Note that in true AVX512F would suffice, but we don't want to add * specific support for AVX512F because this would be the only function * to benefit from that. - * - * Intentionally don't process more than 64 bytes because 64 is the typical - * cache block, and processing 128 bytes doesn't increase performance, and in - * some cases it even decreases it. */ void raid_gen1_avx512bw(int nd, size_t size, void **vv) { @@ -117,7 +206,6 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) raid_avx_begin(); - /* additional unrolling makes it slower */ for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[l][i])); for (d = l - 1; d >= 0; --d) { @@ -191,6 +279,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) asm volatile ("pxor %xmm4,%xmm2"); asm volatile ("pxor %xmm5,%xmm3"); } + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i]));