snapraid/raid/memory.c

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2013 Andrea Mazzoleni

#include "internal.h"
#include "cpu.h"
#include "memory.h"

void *raid_malloc_align(size_t size, size_t align_size, void **freeptr)
{
	uint8_t *ptr;
	uintptr_t offset;

	ptr = malloc(size + align_size);
	if (!ptr) {
		/* LCOV_EXCL_START */
		return 0;
		/* LCOV_EXCL_STOP */
	}

	*freeptr = ptr;

	offset = ((uintptr_t)ptr) % align_size;

	if (offset != 0)
		ptr += align_size - offset;

	return ptr;
}

void *raid_malloc(size_t size, void **freeptr)
{
	return raid_malloc_align(size, RAID_MALLOC_ALIGN, freeptr);
}

unsigned raid_optimal_displacement(int n)
{
	if (n <= 8)
		return 24 * 64;
	if (n <= 16)
		return 28 * 64;
	if (n <= 32)
		return 30 * 64;
	return 33 * 64;
}

/*
 * The 4096 bytes represents a full 64-set * 64-byte L1 cache cycle.
 */
#define RAID_WRAP_SIZE 4096

/*
 * PREFETCHER MITIGATION WITH 4K STRIDE PERTURBATION
 *
 * STRIDE_NOISE is a sequence of small, non-linear multipliers used to add
 * variable 4096-byte increments to the virtual distance between consecutive
 * disk buffers during allocation.
 *
 * The parity generation loops repeatedly access corresponding offsets in each
 * disk buffer. If the distance between buffers is constant, the inner loop
 * produces a regular cross-buffer stride in addition to the unit-stride access
 * within each buffer. A two-dimensional prefetcher can recognize this access
 * pattern as a regular grid of disk buffers and offsets.
 *
 * This additional prefetching is not necessarily useful. The cross-buffer
 * accesses already form independent forward streams, and excessive look-ahead
 * can cause cache pollution or pressure on cache-fill and memory-request
 * resources.
 *
 * raid_optimal_displacement() provides the fixed displacement that separates
 * buffer starts among L1 cache sets. This cache-set separation is the most
 * likely cause of the broad performance improvement over contiguous buffers.
 * STRIDE_NOISE is an additional perturbation and must retain that mapping.
 *
 * On common x86 L1 data caches there are 64 sets and cache lines are 64 bytes.
 * A full set-index cycle is therefore 64 * 64 = 4096 bytes, which also matches
 * the usual memory-page size.
 *
 * By adding a varying multiple of 4096 bytes to the buffer spacing:
 *
 * 1. The virtual cross-buffer stride varies instead of remaining constant.
 * 2. The L1 set index is unchanged because adding 4096 bytes preserves address
 *    bits 6 through 11.
 *
 * STRIDE_NOISE is deterministic, not random. Its non-uniform sequence can make
 * the cross-buffer pattern less suitable for stride-based prediction while
 * leaving the sequential access pattern of each buffer unchanged.
 *
 * With the fixed displacement already applied, STRIDE_NOISE had no material
 * effect on the tested Intel processor. On Zen 5, which has a two-dimensional
 * prefetcher, disabling it caused a large throughput reduction. This points to
 * two-dimensional prefetching as the likely trigger, rather than to an Intel
 * or AMD vendor difference.
 *
 * These are the results in MB/s with no stride noise on a Zen 5 CPU:
 *
 * RAID functions used for computing the parity with 'sync':
 *             best    int8   int32   int64    sse2   sse2e   ssse3  ssse3e    avx2   avx2e  avx512    gfni gfni512
 *     gen1  avx512           48947   75608   26465                           70482           73848
 *     gen2  avx512            9989   19702   28144   34140                   42712           22374   30712   68590
 *     genz   avx2e            5935   11653   17763   19398                           38789
 *     gen3   avx2e    2214                                   20620   23859           33947   24109   20045   31187
 *     gen4   avx2e    1622                                   13843   17071           28194   25679   18826   20780
 *     gen5   avx2e    1360                                   11775   12980           20285   20377   17509   16149
 *     gen6   avx2e    1133                                    8478   11189           19951   16831   18965   13912
 *
 * These are the results in MB/s with stride noise on a Zen 5 CPU:
 *
 * RAID functions used for computing the parity with 'sync':
 *             best    int8   int32   int64    sse2   sse2e   ssse3  ssse3e    avx2   avx2e  avx512    gfni gfni512
 *     gen1  avx512           48161   88091  110896                          122684          121917
 *     gen2  avx512            9970   19776   43223   45989                   86221           68418  110524  112067
 *     genz   avx2e            5953   11730   18598   19475                           39411
 *     gen3   avx2e    2219                                   21784   24347           49416   43295   95555  100458
 *     gen4   avx2e    1624                                   14216   17408           38422   29275   73658   84268
 *     gen5   avx2e    1361                                   12369   13585           28328   22052   63672   68036
 *     gen6   avx2e    1131                                    8399   11692           23918   17703   53620   56806
 */
static const unsigned STRIDE_NOISE[16] = {
	0, 3, 1, 6, 2, 5, 7, 4,
	1, 4, 0, 7, 3, 6, 2, 5
};

void **raid_malloc_vector_align(int n, size_t size, size_t align_size, size_t displacement_size, size_t wrap_size, void **freeptr)
{
	void **v;
	uint8_t *va;
	int i;

	BUG_ON(n <= 0);

	v = malloc(n * sizeof(void *));
	if (!v) {
		/* LCOV_EXCL_START */
		return 0;
		/* LCOV_EXCL_STOP */
	}

	/*
	 * The allocated buffer must safely hold the disk chunks, the L1 fixed displacement,
	 * and the variable STRIDE_NOISE. Because the maximum noise multiplier in the array
	 * is 7, reserving 8 * RAID_WRAP_SIZE per disk guarantees the pointer will never overflow
	 * the allocated memory block.
	 */
	va = raid_malloc_align(n * (size + displacement_size + 8 * wrap_size), align_size, freeptr);
	if (!va) {
		/* LCOV_EXCL_START */
		free(v);
		return 0;
		/* LCOV_EXCL_STOP */
	}

	for (i = 0; i < n; ++i) {
		v[i] = va;

		/* move past the active disk buffer */
		va += size;

		/* apply the optimal L1 cache spacing */
		va += displacement_size;

		/* inject the variable noise multiplier to blind the stride prefetcher */
		va += STRIDE_NOISE[i % 16] * wrap_size;
	}

	return v;
}

void **raid_malloc_vector(int n, size_t size, void **freeptr)
{
	return raid_malloc_vector_align(n, size, RAID_MALLOC_ALIGN, raid_optimal_displacement(n), RAID_WRAP_SIZE, freeptr);
}

void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv)
{
	uint8_t **v = (uint8_t **)vv;
	int i;
	size_t j;

	for (i = 0; i < n; ++i)
		for (j = 0; j < size; ++j) {
			/* basic C99/C11 linear congruential generator */
			seed = seed * 1103515245U + 12345U;

			v[i][j] = seed >> 16;
		}
}

void raid_mcache_flush(void *ptr, size_t size)
{
#ifdef CONFIG_X86
	uint8_t *p = ptr;
	size_t cache_line_size;
	size_t offset;
	int has_clflushopt;

	/* MFENCE is used to complete CLFLUSHOPT before the caller reads again. */
	if (size == 0 || !raid_cpu_has_sse2() || !raid_cpu_has_clflush())
		return;

	cache_line_size = raid_cpu_clflush_size();
	if (cache_line_size == 0)
		return;

	has_clflushopt = raid_cpu_has_clflushopt();
	if (has_clflushopt) {
		for (offset = 0; offset < size; offset += cache_line_size)
			asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + offset)));
		asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + size - 1)));
	} else {
		for (offset = 0; offset < size; offset += cache_line_size)
			asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + offset)));
		asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + size - 1)));
	}

	/* Cache invalidation must be globally visible before returning. */
	asm volatile ("mfence" : : : "memory");
#else
	(void)ptr;
	(void)size;
#endif
}

static uintptr_t raid_mtest_address_pattern(const void *ptr)
{
	uintptr_t value = (uintptr_t)ptr;

	/* Mix every address bit while retaining a one-to-one word mapping. */
	value ^= value >> 17;
	value *= (uintptr_t)0xed5ad4bbU;
	value ^= value >> 11;
	value *= (uintptr_t)0xac4c1b51U;
	value ^= value >> 15;

	return value;
}

int raid_mtest_vector(int n, size_t size, void **vv)
{
	static const uint8_t pattern[] = {
		0x00, 0xff, 0x03, 0xfc, 0x0a, 0xf5, 0x09, 0xf6,
		0x59, 0xa6, 0x5a, 0xa5, 0x53, 0xac, 0x50, 0xaf,
		0xf0, 0x0f, 0xf3, 0x0c, 0xfa, 0x05, 0xf9, 0x06,
		0xa9, 0x56, 0xaa, 0x55, 0xa3, 0x5c, 0xa0, 0x5f,
		0x95, 0x6a, 0x96, 0x69, 0x9f, 0x60, 0x9c, 0x63,
		0xcc, 0x33, 0xcf, 0x30, 0xc6, 0x39, 0xc5, 0x3a,
		0x65, 0x9a, 0x66, 0x99, 0x6f, 0x90, 0x6c, 0x93,
		0x3c, 0xc3, 0x3f, 0xc0, 0x36, 0xc9, 0x35, 0xca
	};
	uint8_t **v = (uint8_t **)vv;
	int i;
	size_t j;
	size_t k;
	uint8_t d;
	uint8_t p;

	/* fill with 0 */
	d = 0;
	for (i = 0; i < n; ++i)
		for (j = 0; j < size; ++j)
			v[i][j] = d;
	for (i = 0; i < n; ++i)
		raid_mcache_flush(v[i], size);

	/*
	 * Complementary pairs make each direction cover all two-bit combinations
	 * uniformly, and together they cover all three-bit combinations uniformly.
	 * Transitions between pairs change six bits, except for one four-bit change.
	 */
	for (k = 0; k < sizeof(pattern) / sizeof(pattern[0]); k += 2) {
		p = d;
		d = pattern[k];

		/* forward fill */
		for (i = 0; i < n; ++i) {
			for (j = 0; j < size; ++j) {
				if (v[i][j] != p) {
					/* LCOV_EXCL_START */
					return -1;
					/* LCOV_EXCL_STOP */
				}
				v[i][j] = d;
			}
		}
		for (i = 0; i < n; ++i)
			raid_mcache_flush(v[i], size);

		p = d;
		d = pattern[k + 1];

		/* backward fill */
		for (i = 0; i < n; ++i) {
			for (j = size; j > 0; --j) {
				if (v[i][j - 1] != p) {
					/* LCOV_EXCL_START */
					return -1;
					/* LCOV_EXCL_STOP */
				}
				v[i][j - 1] = d;
			}
		}
		for (i = 0; i < n; ++i)
			raid_mcache_flush(v[i], size);
	}

	/*
	 * Equal byte patterns cannot reveal two memory locations aliasing each
	 * other. Give every word a value derived from its address, then repeat
	 * the verification with the complement to exercise both bit values.
	 */
	for (i = 0; i < n; ++i) {
		for (j = 0; j < size; j += sizeof(uintptr_t)) {
			uintptr_t value = raid_mtest_address_pattern(v[i] + j);
			size_t run = size - j;

			if (run > sizeof(value))
				run = sizeof(value);
			memcpy(v[i] + j, &value, run);
		}
		raid_mcache_flush(v[i], size);
	}

	/* verify forward and replace every address pattern with its complement */
	for (i = 0; i < n; ++i) {
		for (j = 0; j < size; j += sizeof(uintptr_t)) {
			uintptr_t value = raid_mtest_address_pattern(v[i] + j);
			size_t run = size - j;

			if (run > sizeof(value))
				run = sizeof(value);
			if (memcmp(v[i] + j, &value, run) != 0) {
				/* LCOV_EXCL_START */
				return -1;
				/* LCOV_EXCL_STOP */
			}
			value = ~value;
			memcpy(v[i] + j, &value, run);
		}
		raid_mcache_flush(v[i], size);
	}

	/* verify backward and restore the zero-filled result expected by callers */
	for (i = 0; i < n; ++i) {
		j = size;
		while (j != 0) {
			uintptr_t value;
			size_t run = j % sizeof(uintptr_t);

			if (run == 0)
				run = sizeof(uintptr_t);
			j -= run;
			value = ~raid_mtest_address_pattern(v[i] + j);
			if (memcmp(v[i] + j, &value, run) != 0) {
				/* LCOV_EXCL_START */
				return -1;
				/* LCOV_EXCL_STOP */
			}
			memset(v[i] + j, 0, run);
		}
		raid_mcache_flush(v[i], size);
	}

	return 0;
}