From 2f47d407ba661d59df76c30d4277a3b800ba82dd Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 10 Sep 2026 08:41:14 +0200 Subject: [PATCH] Strengthen stream and vector memory tests --- cmdline/stream.c | 6 +- cmdline/util.c | 43 +-------- cmdline/util.h | 5 -- raid/cpu.h | 52 +++++++++++ raid/memory.c | 224 ++++++++++++++++++++++++++++++++++++++++------- raid/memory.h | 16 +++- 6 files changed, 262 insertions(+), 84 deletions(-) diff --git a/cmdline/stream.c b/cmdline/stream.c index d5b4a861..ff535e75 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -59,7 +59,8 @@ STREAM* sopen_read(const char* file, int flags) s->buffer_size = STREAM_SIZE; if (s->buffer_size > s->size) s->buffer_size = s->size; /* allocate only what is necessary */ - s->buffer = malloc_nofail_test(s->buffer_size); + s->buffer = malloc_nofail(s->buffer_size); + mtest_vector(1, s->buffer_size, (void**)&s->buffer); s->pos = s->buffer; s->end = s->buffer; s->state = STREAM_STATE_READ; @@ -89,7 +90,8 @@ STREAM* sopen_multi_write(unsigned count, int flags) } s->buffer_size = STREAM_SIZE; - s->buffer = malloc_nofail_test(s->buffer_size); + s->buffer = malloc_nofail(s->buffer_size); + mtest_vector(1, s->buffer_size, (void**)&s->buffer); s->pos = s->buffer; s->end = s->buffer + s->buffer_size; s->state = STREAM_STATE_WRITE; diff --git a/cmdline/util.c b/cmdline/util.c index 4169083e..233656f8 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -75,50 +75,9 @@ void** malloc_nofail_vector_direct(int n, size_t size, void** freeptr) return ptr; } -void* malloc_nofail_test(size_t size) -{ - void* ptr; - - ptr = malloc_nofail(size); - - mtest_vector(1, size, &ptr); - - return ptr; -} - -/** - * Fast memory test. - * - * It's similar at raid_mtest_vector() but faster because - * we have a lot of buffers to verify. - */ -static int fast_mtest_vector(int n, size_t size, void** vv) -{ - int i, j; - - /* test with all the bits */ - for (i = 0; i < 8; ++i) { - unsigned char value = 1 << i; - - /* fill first */ - memset(vv[0], value, size); - - /* copy to the next */ - for (j = 1; j < n; ++j) - memcpy(vv[j], vv[j - 1], size); - - /* compare with the previous */ - for (j = 1; j <= n; ++j) - if (memcmp(vv[j % n], vv[j - 1], size) != 0) - return -1; - } - - return 0; -} - void mtest_vector(int n, size_t size, void** vv) { - if (fast_mtest_vector(n, size, vv) != 0) { + if (raid_mtest_vector(n, size, vv) != 0) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n"); log_fatal(EINTERNAL, "Try running a memory test like http://www.memtest86.com/\n"); diff --git a/cmdline/util.h b/cmdline/util.h index 7cc24bc9..76772c47 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -32,11 +32,6 @@ void** malloc_nofail_vector_align(int n, size_t size, void** freeptr); */ void** malloc_nofail_vector_direct(int n, size_t size, void** freeptr); -/** - * Safe allocation with memory test. - */ -void* malloc_nofail_test(size_t size); - /** * Test the memory vector for RAM problems. * If a problem is found, it crashes. diff --git a/raid/cpu.h b/raid/cpu.h index 51e7a6bf..0933133a 100644 --- a/raid/cpu.h +++ b/raid/cpu.h @@ -133,6 +133,58 @@ static inline int raid_cpu_has_sse2(void) return raid_cpu_match_sse(0, leaf_1_edx); } +static inline int raid_cpu_has_clflush(void) +{ + /* + * Intel 64 and IA-32 Architectures Software Developer's Manual + * CPUID - CPU Identification + * + * Check that the processor supports CLFLUSH (true if CPUID.01H:EDX.CLFSH[bit 19] = 1). + */ + uint32_t leaf_1_edx = 1 << 19; /* CLFLUSH */ + + return raid_cpu_match_sse(0, leaf_1_edx); +} + +static inline int raid_cpu_has_clflushopt(void) +{ + /* + * Intel 64 and IA-32 Architectures Software Developer's Manual + * CPUID - CPU Identification + * + * Check that the processor supports CLFLUSHOPT (true if CPUID.07H.00H:EBX.CLFLUSHOPT[bit 23] = 1). + */ + uint32_t reg[4]; + uint32_t leaf_7_ebx = 1 << 23; /* CLFLUSHOPT */ + + /* check that leaf 7 is supported */ + raid_cpuid(0, 0, reg); + if (reg[0] < 7) + return 0; + + raid_cpuid(7, 0, reg); + if ((reg[1] & leaf_7_ebx) != leaf_7_ebx) + return 0; + + return 1; +} + +static inline unsigned raid_cpu_clflush_size(void) +{ + /* + * Intel 64 and IA-32 Architectures Software Developer's Manual + * CPUID - CPU Identification + * + * CPUID.01H:EBX.CLFLUSH[bits 15:8] reports the CLFLUSH cache-line + * size in units of 8 bytes. + */ + uint32_t reg[4]; + + raid_cpuid(1, 0, reg); + + return ((reg[1] >> 8) & 0xff) * 8; +} + static inline int raid_cpu_has_ssse3(void) { /* diff --git a/raid/memory.c b/raid/memory.c index 70be3b6a..6303e709 100644 --- a/raid/memory.c +++ b/raid/memory.c @@ -2,6 +2,7 @@ // Copyright (C) 2013 Andrea Mazzoleni #include "internal.h" +#include "cpu.h" #include "memory.h" void *raid_malloc_align(size_t size, size_t align_size, void **freeptr) @@ -184,50 +185,205 @@ void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv) } } +void raid_mcache_flush(void *ptr, size_t size) +{ +#ifdef CONFIG_X86 + uint8_t *p = ptr; + size_t cache_line_size; + size_t offset; + int has_clflushopt; + + /* MFENCE is used to complete CLFLUSHOPT before the caller reads again. */ + if (size == 0 || !raid_cpu_has_sse2() || !raid_cpu_has_clflush()) + return; + + cache_line_size = raid_cpu_clflush_size(); + if (cache_line_size == 0) + return; + + has_clflushopt = raid_cpu_has_clflushopt(); + if (has_clflushopt) { + for (offset = 0; offset < size; offset += cache_line_size) + asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + offset))); + asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + size - 1))); + } else { + for (offset = 0; offset < size; offset += cache_line_size) + asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + offset))); + asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + size - 1))); + } + + /* Cache invalidation must be globally visible before returning. */ + asm volatile ("mfence" : : : "memory"); +#else + (void)ptr; + (void)size; +#endif +} + +static uintptr_t raid_mtest_address_pattern(const void *ptr) +{ + uintptr_t value = (uintptr_t)ptr; + + /* Mix every address bit while retaining a one-to-one word mapping. */ +#if UINTPTR_MAX > 0xffffffffU + value ^= value >> 33; + value *= (uintptr_t)0xff51afd7ed558ccdULL; + value ^= value >> 33; + value *= (uintptr_t)0xc4ceb9fe1a85ec53ULL; + value ^= value >> 33; +#else + value ^= value >> 17; + value *= (uintptr_t)0xed5ad4bbU; + value ^= value >> 11; + value *= (uintptr_t)0xac4c1b51U; + value ^= value >> 15; +#endif + + return value; +} + +/** + * Compare memory against a repeated byte value. + * + * Return <0 if ptr is less than value, 0 if equal, >0 if greater. + */ +static __always_inline int raid_membcmp(const void *ptr, uint8_t value, size_t size) +{ + const uint8_t *p = ptr; + + if (size == 0) + return 0; + if (p[0] != value) + return (int)p[0] - (int)value; + + /* + * If the buffer contains value up to index k-1, then at offset k-1 + * memcmp compares (p + 1)[k - 1] = p[k] against p[k - 1] = value. + * The first differing adjacent pair yields the exact sign (p[k] - value). + */ + return memcmp(p + 1, p, size - 1); +} + int raid_mtest_vector(int n, size_t size, void **vv) { + static const uint8_t pattern[] = { + 0x00, 0xff, /* solid bits: minimum vs maximum cell charge and VDD droop */ + 0x55, 0xaa, /* 1-bit alternate: worst-case crosstalk between adjacent lines */ + 0x33, 0xcc, /* 2-bit alternate: coupling across line pairs (00110011 / 11001100) */ + 0x0f, 0xf0 /* 4-bit alternate: coupling across nibbles */ + }; uint8_t **v = (uint8_t **)vv; - int i; - size_t j; - unsigned k; - uint8_t d; - uint8_t p; + size_t remainder = size % sizeof(uintptr_t); + size_t aligned_size = size - remainder; - /* fill with 0 */ - d = 0; - for (i = 0; i < n; ++i) - for (j = 0; j < size; ++j) - v[i][j] = d; + /* + * Complementary pairs exercise minimum/maximum cell charge and + * alternating bit coupling across 1-bit, 2-bit, and 4-bit pin strides. + */ + for (size_t k = 0; k < sizeof(pattern) / sizeof(pattern[0]); ++k) { + uint8_t d = pattern[k]; - /* test with all the byte patterns */ - for (k = 1; k < 256; ++k) { - p = d; - d = k; + for (int i = 0; i < n; ++i) + memset(v[i], d, size); - /* forward fill */ - for (i = 0; i < n; ++i) { - for (j = 0; j < size; ++j) { - if (v[i][j] != p) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - v[i][j] = d; + for (int i = 0; i < n; ++i) + raid_mcache_flush(v[i], size); + + for (int i = 0; i < n; ++i) { + if (raid_membcmp(v[i], d, size) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ } } + } - p = d; - d = ~p; - /* backward fill with complement */ - for (i = 0; i < n; ++i) { - for (j = size; j > 0; --j) { - if (v[i][j - 1] != p) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - v[i][j - 1] = d; + /* + * Equal byte patterns cannot reveal two memory locations aliasing each + * other. Give every word a value derived from its address, then repeat + * the verification with the complement to exercise both bit values. + */ + for (int i = 0; i < n; ++i) { + uint8_t *p = v[i]; + uint8_t *limit = p + aligned_size; + while (p < limit) { + uintptr_t value = raid_mtest_address_pattern(p); + memcpy(p, &value, sizeof(uintptr_t)); + p += sizeof(uintptr_t); + } + if (remainder) { + uintptr_t value = raid_mtest_address_pattern(p); + memcpy(p, &value, remainder); + } + + raid_mcache_flush(v[i], size); + } + + /* + * Verify forward pass: confirm that the initial address patterns are + * intact across all buffers, proving no cross-buffer aliasing occurred. + * Replace each word with its bitwise complement in-place to test bit + * transition faults (0 -> 1 and 1 -> 0) and catch stuck-at bits. + */ + for (int i = 0; i < n; ++i) { + uint8_t *p = v[i]; + uint8_t *limit = p + aligned_size; + while (p < limit) { + uintptr_t value = raid_mtest_address_pattern(p); + uintptr_t actual; + memcpy(&actual, p, sizeof(uintptr_t)); + if (actual != value) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ } + value = ~value; + memcpy(p, &value, sizeof(uintptr_t)); + p += sizeof(uintptr_t); + } + if (remainder) { + uintptr_t value = raid_mtest_address_pattern(p); + if (memcmp(p, &value, remainder) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + value = ~value; + memcpy(p, &value, remainder); + } + + raid_mcache_flush(v[i], size); + } + + /* + * Verify backward pass: traverse memory and buffers in strictly descending + * order to detect directional coupling faults and address-line bridging. + * Confirm that the inverted patterns survived the cache flush, then + * restore the zero-filled state required by callers. + */ + for (int i = n - 1; i >= 0; --i) { + uint8_t *p = v[i] + aligned_size; + uint8_t *limit = v[i]; + if (remainder) { + uintptr_t value = ~raid_mtest_address_pattern(p); + if (memcmp(p, &value, remainder) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + memset(p, 0, remainder); + } + while (p > limit) { + p -= sizeof(uintptr_t); + uintptr_t value = ~raid_mtest_address_pattern(p); + uintptr_t actual; + memcpy(&actual, p, sizeof(uintptr_t)); + if (actual != value) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + memset(p, 0, sizeof(uintptr_t)); } } diff --git a/raid/memory.h b/raid/memory.h index def9728a..a13d518b 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -115,9 +115,23 @@ void **raid_malloc_vector_align(int n, size_t size, size_t align_size, size_t di */ void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv); +/** + * Writes back and invalidates the CPU cache lines covering the memory range. + * + * This operation is best effort. It does nothing on architectures without a + * safe user-space cache invalidation instruction. + */ +void raid_mcache_flush(void *ptr, size_t size); + /** * Tests the memory vector for RAM problems. - * If a problem is found, it crashes. + * + * This test is destructive and overwrites the existing buffer contents. + * On success, all buffers are guaranteed to be zero-filled. + * If a memory error is detected, the function returns immediately and + * buffers may be left partially modified. + * + * Returns 0 on success, or -1 if a memory problem is found. */ int raid_mtest_vector(int n, size_t size, void **vv);