snapraid/raid/memory.h

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2013 Andrea Mazzoleni

#ifndef __RAID_MEMORY_H
#define __RAID_MEMORY_H

/**
 * Memory allocation and benchmark layout helpers.
 *
 * This module provides optional utility functions to allocate SIMD-aligned and
 * cache-displaced memory buffers for test suites, benchmarks, and Direct I/O.
 * The core RAID engine (raid.h) remains zero-allocation and operates on
 * preallocated buffers.
 *
 * Like other internal helpers in this library, these functions intentionally do
 * not validate against arithmetic overflow. Callers are responsible for passing
 * reasonable and bounded arguments.
 */

/**
 * Memory alignment provided by raid_malloc().
 *
 * It should guarantee good cache performance everywhere.
 */
#define RAID_MALLOC_ALIGN 256

/*
 * Memory displacement to color buffer starts across L1 cache sets, used by
 * raid_malloc_vector().
 *
 * On the intended x86 L1 data cache layout, cache lines are 64 bytes and there
 * are 64 sets. Contiguous buffers whose size is a multiple of 4096 bytes have
 * the same L1 set index at corresponding offsets. Parallel SIMD streams can
 * then contend for the same set, reducing the useful prefetch distance and
 * causing conflict misses.
 *
 * A displacement of X * 64 bytes changes the starting set index of each
 * buffer. The cycle of distinct starting sets is 64 / gcd(X, 64).
 *
 * The multiplier X scales with the number of allocated buffers (n):
 * - n <= 8  -> 24 * 64: Cycle of 8.
 * - n <= 16 -> 28 * 64: Cycle of 16.
 * - n <= 32 -> 30 * 64: Cycle of 32.
 * - n > 32  -> 33 * 64: Cycle of 64.
 *
 * The 64-set cycle gives distinct starting set indexes for up to 64 buffers;
 * for larger vectors the starting set indexes repeat. It reduces, but does not
 * eliminate, all possible L1 cache conflicts.
 *
 * These are the results in MB/s with no displacement:
 *
 * RAID functions used for computing the parity with 'sync':
 *             best    int8   int32   int64    sse2   sse2e   ssse3  ssse3e    avx2   avx2e
 *     gen1    avx2           14389   26472   42343                           64085
 *     gen2    avx2            3788    7309   20930   22042                   36916
 *     genz   avx2e            2368    4262   11791   11786                           21770
 *     gen3   avx2e     809                                   11249   11913           21937
 *     gen4   avx2e     609                                    8857    9469           17401
 *     gen5   avx2e     488                                    7147    7465           14231
 *     gen6   avx2e     398                                    5828    6381           12196
 *
 * These are the results with the cache-line displacement, demonstrating
 * throughput improvements in the order of 20% or more. They were measured on
 * a machine without a two-dimensional prefetcher, where STRIDE_NOISE had no
 * material effect. They therefore measure the fixed displacement alone:
 *
 * RAID functions used for computing the parity with 'sync':
 *             best    int8   int32   int64    sse2   sse2e   ssse3  ssse3e    avx2   avx2e
 *     gen1    avx2           18550   34501   62950                           78335
 *     gen2    avx2            5410   10080   27334   28502                   46921
 *     genz   avx2e            3161    5083   14741   14704                           26404
 *     gen3   avx2e    1296                                   12752   13902           26446
 *     gen4   avx2e     975                                    9757   10893           20598
 *     gen5   avx2e     799                                    7879    8612           16613
 *     gen6   avx2e     663                                    6386    7283           13987
 */
unsigned raid_optimal_displacement(int n);

/**
 * Aligned malloc with default RAID alignment (256 bytes).
 *
 * Stores the base allocation address in @freeptr to be passed to free().
 */
void *raid_malloc(size_t size, void **freeptr);

/**
 * Arbitrary aligned malloc.
 *
 * Allocates a buffer of @size bytes aligned to @align_size bytes.
 * Stores the base allocation address in @freeptr to be passed to free().
 */
void *raid_malloc_align(size_t size, size_t align_size, void **freeptr);

/**
 * Cache-displaced vector allocation.
 *
 * Allocates an array of @n pointers, each one pointing to a block of
 * the specified @size with optimal L1 displacement and stride perturbation.
 *
 * The first block is aligned to RAID_MALLOC_ALIGN. If @size is a multiple of
 * 64, every block is aligned to at least 64 bytes because the displacement and
 * stride perturbation are also multiples of 64. Later blocks are deliberately
 * not realigned to RAID_MALLOC_ALIGN, as that would alter their cache-set
 * displacement.
 *
 * Freeing requires two calls: free(*freeptr) for the data buffer,
 * and free(v) for the pointer vector.
 */
void **raid_malloc_vector(int n, size_t size, void **freeptr);

/**
 * Base-aligned vector allocation with caller-defined spacing.
 *
 * Allocates an array of @n pointers. The first block, which is the base of the
 * usable data area, is aligned to @base_align_size bytes. Subsequent block
 * addresses are obtained by adding @size, @displacement_size, and an integer
 * multiple of @wrap_size; they are not individually realigned.
 *
 * All blocks retain @base_align_size alignment if:
 *
 *   (@size + @displacement_size) % @base_align_size == 0
 *   @wrap_size % @base_align_size == 0
 *
 * @base_align_size must be greater than 0. Callers requiring a specific
 * alignment for every block must select spacing values that satisfy these
 * conditions.
 *
 * Freeing requires two calls: free(*freeptr) for the data buffer,
 * and free(v) for the pointer vector.
 */
void **raid_malloc_vector_align(int n, size_t size, size_t base_align_size, size_t displacement_size, size_t wrap_size, void **freeptr);

/**
 * Fills the memory vector with pseudo-random data based on the specified seed.
 */
void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv);

/**
 * Writes back and invalidates the CPU cache lines covering the memory range.
 *
 * This operation is best effort. It does nothing on architectures without a
 * safe user-space cache invalidation instruction.
 */
void raid_mcache_flush(void *ptr, size_t size);

/**
 * Tests the memory vector for RAM problems.
 *
 * This test is destructive and overwrites the existing buffer contents.
 * On success, all buffers are guaranteed to be zero-filled.
 * If a memory error is detected, the function returns immediately and
 * buffers may be left partially modified.
 *
 * Returns 0 on success, or -1 if a memory problem is found.
 */
int raid_mtest_vector(int n, size_t size, void **vv);

#endif