From f69007eaf69501ea786539c05b8c9276daad9fb5 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 10 Sep 2026 08:41:14 +0200 Subject: [PATCH] Strengthen stream and vector memory tests --- cmdline/stream.c | 6 +- cmdline/util.c | 43 +-------------- cmdline/util.h | 5 -- raid/cpu.h | 52 ++++++++++++++++++ raid/memory.c | 140 +++++++++++++++++++++++++++++++++++++++++++++-- raid/memory.h | 8 +++ 6 files changed, 199 insertions(+), 55 deletions(-) diff --git a/cmdline/stream.c b/cmdline/stream.c index d5b4a861..ff535e75 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -59,7 +59,8 @@ STREAM* sopen_read(const char* file, int flags) s->buffer_size = STREAM_SIZE; if (s->buffer_size > s->size) s->buffer_size = s->size; /* allocate only what is necessary */ - s->buffer = malloc_nofail_test(s->buffer_size); + s->buffer = malloc_nofail(s->buffer_size); + mtest_vector(1, s->buffer_size, (void**)&s->buffer); s->pos = s->buffer; s->end = s->buffer; s->state = STREAM_STATE_READ; @@ -89,7 +90,8 @@ STREAM* sopen_multi_write(unsigned count, int flags) } s->buffer_size = STREAM_SIZE; - s->buffer = malloc_nofail_test(s->buffer_size); + s->buffer = malloc_nofail(s->buffer_size); + mtest_vector(1, s->buffer_size, (void**)&s->buffer); s->pos = s->buffer; s->end = s->buffer + s->buffer_size; s->state = STREAM_STATE_WRITE; diff --git a/cmdline/util.c b/cmdline/util.c index 199e52ff..ae0b0b5b 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -75,50 +75,9 @@ void** malloc_nofail_vector_direct(int n, size_t size, void** freeptr) return ptr; } -void* malloc_nofail_test(size_t size) -{ - void* ptr; - - ptr = malloc_nofail(size); - - mtest_vector(1, size, &ptr); - - return ptr; -} - -/** - * Fast memory test. - * - * It's similar at raid_mtest_vector() but faster because - * we have a lot of buffers to verify. - */ -static int fast_mtest_vector(int n, size_t size, void** vv) -{ - int i, j; - - /* test with all the bits */ - for (i = 0; i < 8; ++i) { - unsigned char value = 1 << i; - - /* fill first */ - memset(vv[0], value, size); - - /* copy to the next */ - for (j = 1; j < n; ++j) - memcpy(vv[j], vv[j - 1], size); - - /* compare with the previous */ - for (j = 1; j <= n; ++j) - if (memcmp(vv[j % n], vv[j - 1], size) != 0) - return -1; - } - - return 0; -} - void mtest_vector(int n, size_t size, void** vv) { - if (fast_mtest_vector(n, size, vv) != 0) { + if (raid_mtest_vector(n, size, vv) != 0) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n"); log_fatal(EINTERNAL, "Try running a memory test like http://www.memtest86.com/\n"); diff --git a/cmdline/util.h b/cmdline/util.h index fd2a3e7b..7166e08d 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -32,11 +32,6 @@ void** malloc_nofail_vector_align(int n, size_t size, void** freeptr); */ void** malloc_nofail_vector_direct(int n, size_t size, void** freeptr); -/** - * Safe allocation with memory test. - */ -void* malloc_nofail_test(size_t size); - /** * Test the memory vector for RAM problems. * If a problem is found, it crashes. diff --git a/raid/cpu.h b/raid/cpu.h index 51e7a6bf..0933133a 100644 --- a/raid/cpu.h +++ b/raid/cpu.h @@ -133,6 +133,58 @@ static inline int raid_cpu_has_sse2(void) return raid_cpu_match_sse(0, leaf_1_edx); } +static inline int raid_cpu_has_clflush(void) +{ + /* + * Intel 64 and IA-32 Architectures Software Developer's Manual + * CPUID - CPU Identification + * + * Check that the processor supports CLFLUSH (true if CPUID.01H:EDX.CLFSH[bit 19] = 1). + */ + uint32_t leaf_1_edx = 1 << 19; /* CLFLUSH */ + + return raid_cpu_match_sse(0, leaf_1_edx); +} + +static inline int raid_cpu_has_clflushopt(void) +{ + /* + * Intel 64 and IA-32 Architectures Software Developer's Manual + * CPUID - CPU Identification + * + * Check that the processor supports CLFLUSHOPT (true if CPUID.07H.00H:EBX.CLFLUSHOPT[bit 23] = 1). + */ + uint32_t reg[4]; + uint32_t leaf_7_ebx = 1 << 23; /* CLFLUSHOPT */ + + /* check that leaf 7 is supported */ + raid_cpuid(0, 0, reg); + if (reg[0] < 7) + return 0; + + raid_cpuid(7, 0, reg); + if ((reg[1] & leaf_7_ebx) != leaf_7_ebx) + return 0; + + return 1; +} + +static inline unsigned raid_cpu_clflush_size(void) +{ + /* + * Intel 64 and IA-32 Architectures Software Developer's Manual + * CPUID - CPU Identification + * + * CPUID.01H:EBX.CLFLUSH[bits 15:8] reports the CLFLUSH cache-line + * size in units of 8 bytes. + */ + uint32_t reg[4]; + + raid_cpuid(1, 0, reg); + + return ((reg[1] >> 8) & 0xff) * 8; +} + static inline int raid_cpu_has_ssse3(void) { /* diff --git a/raid/memory.c b/raid/memory.c index 70be3b6a..57aa248d 100644 --- a/raid/memory.c +++ b/raid/memory.c @@ -2,6 +2,7 @@ // Copyright (C) 2013 Andrea Mazzoleni #include "internal.h" +#include "cpu.h" #include "memory.h" void *raid_malloc_align(size_t size, size_t align_size, void **freeptr) @@ -184,12 +185,71 @@ void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv) } } +void raid_mcache_flush(void *ptr, size_t size) +{ +#ifdef CONFIG_X86 + uint8_t *p = ptr; + size_t cache_line_size; + size_t offset; + int has_clflushopt; + + /* MFENCE is used to complete CLFLUSHOPT before the caller reads again. */ + if (size == 0 || !raid_cpu_has_sse2() || !raid_cpu_has_clflush()) + return; + + cache_line_size = raid_cpu_clflush_size(); + if (cache_line_size == 0) + return; + + has_clflushopt = raid_cpu_has_clflushopt(); + if (has_clflushopt) { + for (offset = 0; offset < size; offset += cache_line_size) + asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + offset))); + asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + size - 1))); + } else { + for (offset = 0; offset < size; offset += cache_line_size) + asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + offset))); + asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + size - 1))); + } + + /* Cache invalidation must be globally visible before returning. */ + asm volatile ("mfence" : : : "memory"); +#else + (void)ptr; + (void)size; +#endif +} + +static uintptr_t raid_mtest_address_pattern(const void *ptr) +{ + uintptr_t value = (uintptr_t)ptr; + + /* Mix every address bit while retaining a one-to-one word mapping. */ + value ^= value >> 17; + value *= (uintptr_t)0xed5ad4bbU; + value ^= value >> 11; + value *= (uintptr_t)0xac4c1b51U; + value ^= value >> 15; + + return value; +} + int raid_mtest_vector(int n, size_t size, void **vv) { + static const uint8_t pattern[] = { + 0x00, 0xff, 0x03, 0xfc, 0x0a, 0xf5, 0x09, 0xf6, + 0x59, 0xa6, 0x5a, 0xa5, 0x53, 0xac, 0x50, 0xaf, + 0xf0, 0x0f, 0xf3, 0x0c, 0xfa, 0x05, 0xf9, 0x06, + 0xa9, 0x56, 0xaa, 0x55, 0xa3, 0x5c, 0xa0, 0x5f, + 0x95, 0x6a, 0x96, 0x69, 0x9f, 0x60, 0x9c, 0x63, + 0xcc, 0x33, 0xcf, 0x30, 0xc6, 0x39, 0xc5, 0x3a, + 0x65, 0x9a, 0x66, 0x99, 0x6f, 0x90, 0x6c, 0x93, + 0x3c, 0xc3, 0x3f, 0xc0, 0x36, 0xc9, 0x35, 0xca + }; uint8_t **v = (uint8_t **)vv; int i; size_t j; - unsigned k; + size_t k; uint8_t d; uint8_t p; @@ -198,11 +258,17 @@ int raid_mtest_vector(int n, size_t size, void **vv) for (i = 0; i < n; ++i) for (j = 0; j < size; ++j) v[i][j] = d; + for (i = 0; i < n; ++i) + raid_mcache_flush(v[i], size); - /* test with all the byte patterns */ - for (k = 1; k < 256; ++k) { + /* + * Complementary pairs make each direction cover all two-bit combinations + * uniformly, and together they cover all three-bit combinations uniformly. + * Transitions between pairs change six bits, except for one four-bit change. + */ + for (k = 0; k < sizeof(pattern) / sizeof(pattern[0]); k += 2) { p = d; - d = k; + d = pattern[k]; /* forward fill */ for (i = 0; i < n; ++i) { @@ -215,10 +281,13 @@ int raid_mtest_vector(int n, size_t size, void **vv) v[i][j] = d; } } + for (i = 0; i < n; ++i) + raid_mcache_flush(v[i], size); p = d; - d = ~p; - /* backward fill with complement */ + d = pattern[k + 1]; + + /* backward fill */ for (i = 0; i < n; ++i) { for (j = size; j > 0; --j) { if (v[i][j - 1] != p) { @@ -229,6 +298,65 @@ int raid_mtest_vector(int n, size_t size, void **vv) v[i][j - 1] = d; } } + for (i = 0; i < n; ++i) + raid_mcache_flush(v[i], size); + } + + /* + * Equal byte patterns cannot reveal two memory locations aliasing each + * other. Give every word a value derived from its address, then repeat + * the verification with the complement to exercise both bit values. + */ + for (i = 0; i < n; ++i) { + for (j = 0; j < size; j += sizeof(uintptr_t)) { + uintptr_t value = raid_mtest_address_pattern(v[i] + j); + size_t run = size - j; + + if (run > sizeof(value)) + run = sizeof(value); + memcpy(v[i] + j, &value, run); + } + raid_mcache_flush(v[i], size); + } + + /* verify forward and replace every address pattern with its complement */ + for (i = 0; i < n; ++i) { + for (j = 0; j < size; j += sizeof(uintptr_t)) { + uintptr_t value = raid_mtest_address_pattern(v[i] + j); + size_t run = size - j; + + if (run > sizeof(value)) + run = sizeof(value); + if (memcmp(v[i] + j, &value, run) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + value = ~value; + memcpy(v[i] + j, &value, run); + } + raid_mcache_flush(v[i], size); + } + + /* verify backward and restore the zero-filled result expected by callers */ + for (i = 0; i < n; ++i) { + j = size; + while (j != 0) { + uintptr_t value; + size_t run = j % sizeof(uintptr_t); + + if (run == 0) + run = sizeof(uintptr_t); + j -= run; + value = ~raid_mtest_address_pattern(v[i] + j); + if (memcmp(v[i] + j, &value, run) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + memset(v[i] + j, 0, run); + } + raid_mcache_flush(v[i], size); } return 0; diff --git a/raid/memory.h b/raid/memory.h index def9728a..165db5d0 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -115,6 +115,14 @@ void **raid_malloc_vector_align(int n, size_t size, size_t align_size, size_t di */ void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv); +/** + * Writes back and invalidates the CPU cache lines covering the memory range. + * + * This operation is best effort. It does nothing on architectures without a + * safe user-space cache invalidation instruction. + */ +void raid_mcache_flush(void *ptr, size_t size); + /** * Tests the memory vector for RAM problems. * If a problem is found, it crashes.