Inject variable page-offset noise to disk buffer displacement to mitigate prefetcher contention

raid/memory.c

@@ -42,6 +42,82 @@ unsigned raid_optimal_displacement(int n)
 	return 33 * 64;
 }
 
+/*
+ * The 4096 bytes represents a full 64-set * 64-byte L1 cache cycle.
+ */
+#define L1_WRAP_SIZE 4096
+
+/*
+ * PREFETCHER MITIGATION VIA PAGE-OFFSET RANDOMIZATION (WAY-SHIFTING)
+ *
+ * STRIDE_NOISE is a sequence of small, non-linear multipliers used to inject
+ * variable byte offsets into the physical distance between consecutive disk
+ * buffers during allocation. These multipliers are applied in 4096-byte chunks.
+ *
+ * The parity generation loops contain a massive cross-disk jump in the inner
+ * loop. If the distance between every disk is a constant delta (e.g., exactly
+ * 256KB + fixed displacement bytes), modern IP-based stride prefetchers
+ * (particularly on architectures like AMD Zen 5) will easily recognize the pattern.
+ *
+ * Because the prefetcher is biased to aggressively accelerate constant positive
+ * strides, it incorrectly assumes the program is executing a massive linear
+ * sweep of memory. It begins flooding the memory controller with speculative,
+ * useless requests for data far beyond the active disks. This "garbage traffic"
+ * saturates the memory bus, starving the AVX/SIMD execution engine of
+ * the actual data it needs and crippling throughput.
+ *
+ * We must blind the prefetcher by making the stride unpredictable, but we CANNOT
+ * alter the carefully calculated L1 cache set mapping (raid_optimal_displacement),
+ * otherwise we will cause catastrophic L1 cache thrashing.
+ *
+ * - The standard x86 L1 data cache has 64 sets, and a cache line is 64 bytes.
+ * A full wrap-around of the L1 cache is exactly 64 * 64 = 4096 bytes (which
+ * also perfectly aligns with a standard 4KB memory page).
+ *
+ * - By adding a random multiple of 4096 bytes to our jump:
+ * 1. The physical memory address changes wildly, breaking the constant stride.
+ * 2. The modulo-64 cache math is completely unaffected ((X + 4096) % 64 == X % 64).
+ *
+ * The IP-based stride prefetcher monitors the cross-disk jumps and sees a
+ * chaotic sequence of massive, fluctuating deltas (e.g., +275KB, +267KB, +288KB).
+ * Because the stride is never constant, the prefetcher's internal confidence
+ * counter never builds up. The prefetcher safely disables itself for the inner
+ * loop.
+ *
+ * With the noisy prefetcher silenced, the memory bus is cleared. The CPU's L2
+ * stream prefetcher is left alone to perfectly track the independent +64 byte
+ * forward reads for each individual disk. The AVX/SIMD execution engine receives
+ * 100% of the bandwidth, yielding maximum physical throughput.
+ *
+ * These are the results in MB/s with no stride noise on a Zen 5 CPU:
+ *
+ * RAID functions used for computing the parity with 'sync':
+ *             best    int8   int32   int64    sse2   sse2e   ssse3  ssse3e    avx2   avx2e  avx512    gfni gfni512
+ *     gen1  avx512           48947   75608   26465                           70482           73848
+ *     gen2  avx512            9989   19702   28144   34140                   42712           22374   30712   68590
+ *     genz   avx2e            5935   11653   17763   19398                           38789
+ *     gen3   avx2e    2214                                   20620   23859           33947   24109   20045   31187
+ *     gen4   avx2e    1622                                   13843   17071           28194   25679   18826   20780
+ *     gen5   avx2e    1360                                   11775   12980           20285   20377   17509   16149
+ *     gen6   avx2e    1133                                    8478   11189           19951   16831   18965   13912
+ *
+ * These are the results in MB/s with stride noise on a Zen 5 CPU:
+ *
+ * RAID functions used for computing the parity with 'sync':
+ *             best    int8   int32   int64    sse2   sse2e   ssse3  ssse3e    avx2   avx2e  avx512    gfni gfni512
+ *     gen1  avx512           48161   88091  110896                          122684          121917
+ *     gen2  avx512            9970   19776   43223   45989                   86221           68418  110524  112067
+ *     genz   avx2e            5953   11730   18598   19475                           39411
+ *     gen3   avx2e    2219                                   21784   24347           49416   43295   95555  100458
+ *     gen4   avx2e    1624                                   14216   17408           38422   29275   73658   84268
+ *     gen5   avx2e    1361                                   12369   13585           28328   22052   63672   68036
+ *     gen6   avx2e    1131                                    8399   11692           23918   17703   53620   56806
+ */
+static const unsigned STRIDE_NOISE[16] = {
+	0, 3, 1, 6, 2, 5, 7, 4,
+	1, 4, 0, 7, 3, 6, 2, 5
+};
+
 void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, ssize_t displacement_size, void **freeptr)
 {
 	void **v;
@@ -59,7 +135,13 @@ void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, s
 		/* LCOV_EXCL_STOP */
 	}
 
-	va = raid_malloc_align(n * (size + displacement_size), align_size, freeptr);
+	/*
+	 * The allocated buffer must safely hold the disk chunks, the L1 fixed displacement,
+	 * and the variable STRIDE_NOISE. Because the maximum noise multiplier in the array
+	 * is 7, reserving 8 * L1_WRAP_SIZE per disk guarantees the pointer will never overflow
+	 * the allocated memory block.
+	 */
+	va = raid_malloc_align(n * (size + displacement_size + 8 * L1_WRAP_SIZE), align_size, freeptr);
 	if (!va) {
 		/* LCOV_EXCL_START */
 		free(v);
@@ -69,7 +151,15 @@ void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, s
 
 	for (i = 0; i < n; ++i) {
 		v[i] = va;
-		va += size + displacement_size;
+
+		/* move past the active disk buffer */
+		va += size;
+
+		/* apply the optimal L1 cache spacing */
+		va += displacement_size;
+
+		/* inject the variable noise multiplier to blind the stride prefetcher */
+		va += STRIDE_NOISE[i % 16] * L1_WRAP_SIZE;
 	}
 
 	return v;