diff --git a/cmdline/speed.c b/cmdline/speed.c index 400b2b8e..ac0980f2 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -171,7 +171,7 @@ void speed(int period, int nd, int size) printf("\n"); printf("Speed test using %u data buffers of %u bytes, for a total of %u KiB.\n", nd, size, nd * size / KIBI); - printf("Memory blocks have a displacement of %u bytes to improve cache performance.\n", RAID_MALLOC_DISPLACEMENT); + printf("Memory blocks have a displacement of %u bytes to improve cache performance.\n", raid_optimal_displacement(nv)); printf("The reported values are the aggregate bandwidth of all data blocks in MB/s,\n"); printf("not counting parity blocks.\n"); printf("\n"); diff --git a/raid/memory.c b/raid/memory.c index b6476221..69fe503c 100644 --- a/raid/memory.c +++ b/raid/memory.c @@ -28,10 +28,21 @@ void *raid_malloc_align(size_t size, size_t align_size, void **freeptr) void *raid_malloc(size_t size, void **freeptr) { - return raid_malloc_align(size, RAID_MALLOC_ALIGN, freeptr); + return raid_malloc_align(size, RAID_MALLOC_ALIGN, freeptr); } -void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, size_t displacement_size, void **freeptr) +unsigned raid_optimal_displacement(int n) +{ + if (n <= 8) + return 24 * 64; + if (n <= 16) + return 28 * 64; + if (n <= 32) + return 30 * 64; + return 33 * 64; +} + +void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, ssize_t displacement_size, void **freeptr) { void **v; unsigned char *va; @@ -39,6 +50,8 @@ void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, s BUG_ON(n <= 0 || nd < 0); + (void)nd; + v = malloc(n * sizeof(void *)); if (!v) { /* LCOV_EXCL_START */ @@ -64,7 +77,7 @@ void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, s void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) { - return raid_malloc_vector_align(nd, n, size, RAID_MALLOC_ALIGN, RAID_MALLOC_DISPLACEMENT, freeptr); + return raid_malloc_vector_align(nd, n, size, RAID_MALLOC_ALIGN, raid_optimal_displacement(n), freeptr); } void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv) @@ -131,4 +144,3 @@ int raid_mtest_vector(int n, size_t size, void **vv) return 0; } - diff --git a/raid/memory.h b/raid/memory.h index 51dbb818..06f3155d 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -11,48 +11,60 @@ */ #define RAID_MALLOC_ALIGN 256 -/** - * Memory displacement to avoid cache address sharing on contiguous blocks, - * used by raid_malloc_vector(). +/* + * Memory displacement to avoid L1 cache set aliasing and 4K aliasing penalties + * on contiguous blocks, used by raid_malloc_vector(). * - * When allocating a sequence of blocks with a size of power of 2, - * there is the risk that the addresses of each block are mapped into the - * same cache line and prefetching predictor, resulting in a lot of cache - * sharing if you access all the blocks in parallel, from the start to the - * end. + * When allocating a sequence of disk buffers with a size that is a large + * power of 2 (or multiples of 4KB/2MB), their starting addresses naturally + * map to the exact same sets in the L1 Data Cache. If these blocks are + * accessed in parallel during SIMD parity generation, the memory streams + * will constantly evict each other, resulting in severe cache thrashing. + * * Furthermore, if buffers are separated by exact multiples of 4096 bytes, + * modern CPUs suffer a "4K Aliasing" penalty. The memory disambiguation unit + * misidentifies the lower 12-bits of the addresses as a match, attempting + * invalid Store-to-Load Forwarding which results in a ~20-cycle pipeline flush. * - * To avoid this effect, it's better if all the blocks are allocated - * with a fixed displacement trying to reduce the cache address sharing. + * To eliminate both bottlenecks, each disk buffer is allocated with a dynamic + * displacement of (X * 64 bytes). The optimal 'X' balances two hardware rules: + * 1. L1 Capacity: The cycle of utilized L1 sets must be >= the number of disks. + * 2. 4K Aliasing: 'X' should be as close to 32 (2048 bytes) as possible to + * maximize the distance from 4096-byte boundaries. * - * The selected displacement was chosen empirically with some speed tests - * with 8/12/16/20/24 data buffers of 256 KB. + * The multiplier 'X' scales dynamically based on the disk count (nd): + * - nd <= 8 -> 24 * 64: Cycle of 8. Safest distance from 4K boundaries. + * - nd <= 16 -> 28 * 64: Cycle of 16. Widely disperses memory streams across + * the cache, preventing hardware prefetcher clustering. + * - nd <= 32 -> 30 * 64: Cycle of 32. Tighter L1 packing. + * - nd <= 64 -> 33 * 64: Cycle of 64. 33 and 64 are coprime, guaranteeing up + * to 64 streams never collide in L1 or trigger 4K stalls. * * These are the results in MB/s with no displacement: * * RAID functions used for computing the parity with 'sync': * best int8 int32 int64 sse2 sse2e ssse3 ssse3e avx2 avx2e - * gen1 avx2 14389 26472 42343 64085 - * gen2 avx2 3788 7309 20930 22042 36916 + * gen1 avx2 14389 26472 42343 64085 + * gen2 avx2 3788 7309 20930 22042 36916 * genz avx2e 2368 4262 11791 11786 21770 * gen3 avx2e 809 11249 11913 21937 * gen4 avx2e 609 8857 9469 17401 * gen5 avx2e 488 7147 7465 14231 * gen6 avx2e 398 5828 6381 12196 * - * These are the results with displacement of 9*256, resulting in improvements - * in the order of 20% or more: + * These are the results with a strategic cache line displacement, + * demonstrating throughput improvements in the order of 20% or more: * - * RAID functions used for computing the parity with 'sync': + * RAID functions used for computing the parity with 'sync': * best int8 int32 int64 sse2 sse2e ssse3 ssse3e avx2 avx2e - * gen1 avx2 18435 34980 62786 78159 - * gen2 avx2 5326 10026 26334 27349 45012 - * genz avx2e 3141 5037 14607 14099 25757 - * gen3 avx2e 1298 13289 14381 27563 - * gen4 avx2e 974 10076 11213 21637 - * gen5 avx2e 796 8091 8863 17458 - * gen6 avx2e 663 6761 7634 14912 + * gen1 avx2 18550 34501 62950 78335 + * gen2 avx2 5410 10080 27334 28502 46921 + * genz avx2e 3161 5083 14741 14704 26404 + * gen3 avx2e 1296 12752 13902 26446 + * gen4 avx2e 975 9757 10893 20598 + * gen5 avx2e 799 7879 8612 16613 + * gen6 avx2e 663 6386 7283 13987 */ -#define RAID_MALLOC_DISPLACEMENT (7*256) +unsigned raid_optimal_displacement(int n); /** * Aligned malloc. @@ -77,7 +89,7 @@ void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr); /** * Arbitrary aligned vector allocation. */ -void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, size_t displacement_size, void **freeptr); +void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, ssize_t displacement_size, void **freeptr); /** * Fills the memory vector with pseudo-random data based on the specified seed. @@ -91,4 +103,3 @@ void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv); int raid_mtest_vector(int n, size_t size, void **vv); #endif - diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index bc07b9ab..7c261850 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -100,7 +100,7 @@ void speed(void) } printf("Speed test using %u data buffers of %u bytes, for a total of %u KiB.\n", nd, size, nd * size / 1024); - printf("Memory blocks have a displacement of %u bytes to improve cache performance.\n", RAID_MALLOC_DISPLACEMENT); + printf("Memory blocks have a displacement of %u bytes to improve cache performance.\n", raid_optimal_displacement(nv)); printf("The reported values are the aggregate bandwidth of all data blocks in MiB/s,\n"); printf("not counting parity blocks.\n"); printf("\n");