Strengthen stream and vector memory tests

cmdline/stream.c

@@ -59,7 +59,8 @@ STREAM* sopen_read(const char* file, int flags)
 	s->buffer_size = STREAM_SIZE;
 	if (s->buffer_size > s->size)
 		s->buffer_size = s->size; /* allocate only what is necessary */
-	s->buffer = malloc_nofail_test(s->buffer_size);
+	s->buffer = malloc_nofail(s->buffer_size);
+	mtest_vector(1, s->buffer_size, (void**)&s->buffer);
 	s->pos = s->buffer;
 	s->end = s->buffer;
 	s->state = STREAM_STATE_READ;
@@ -89,7 +90,8 @@ STREAM* sopen_multi_write(unsigned count, int flags)
 	}
 
 	s->buffer_size = STREAM_SIZE;
-	s->buffer = malloc_nofail_test(s->buffer_size);
+	s->buffer = malloc_nofail(s->buffer_size);
+	mtest_vector(1, s->buffer_size, (void**)&s->buffer);
 	s->pos = s->buffer;
 	s->end = s->buffer + s->buffer_size;
 	s->state = STREAM_STATE_WRITE;

cmdline/util.c

@@ -75,50 +75,9 @@ void** malloc_nofail_vector_direct(int n, size_t size, void** freeptr)
 	return ptr;
 }
 
-void* malloc_nofail_test(size_t size)
-{
-	void* ptr;
-
-	ptr = malloc_nofail(size);
-
-	mtest_vector(1, size, &ptr);
-
-	return ptr;
-}
-
-/**
- * Fast memory test.
- *
- * It's similar at raid_mtest_vector() but faster because
- * we have a lot of buffers to verify.
- */
-static int fast_mtest_vector(int n, size_t size, void** vv)
-{
-	int i, j;
-
-	/* test with all the bits */
-	for (i = 0; i < 8; ++i) {
-		unsigned char value = 1 << i;
-
-		/* fill first */
-		memset(vv[0], value, size);
-
-		/* copy to the next */
-		for (j = 1; j < n; ++j)
-			memcpy(vv[j], vv[j - 1], size);
-
-		/* compare with the previous */
-		for (j = 1; j <= n; ++j)
-			if (memcmp(vv[j % n], vv[j - 1], size) != 0)
-				return -1;
-	}
-
-	return 0;
-}
-
 void mtest_vector(int n, size_t size, void** vv)
 {
-	if (fast_mtest_vector(n, size, vv) != 0) {
+	if (raid_mtest_vector(n, size, vv) != 0) {
 		/* LCOV_EXCL_START */
 		log_fatal(EINTERNAL, "DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
 		log_fatal(EINTERNAL, "Try running a memory test like http://www.memtest86.com/\n");

cmdline/util.h

@@ -32,11 +32,6 @@ void** malloc_nofail_vector_align(int n, size_t size, void** freeptr);
  */
 void** malloc_nofail_vector_direct(int n, size_t size, void** freeptr);
 
-/**
- * Safe allocation with memory test.
- */
-void* malloc_nofail_test(size_t size);
-
 /**
  * Test the memory vector for RAM problems.
  * If a problem is found, it crashes.

raid/cpu.h

@@ -133,6 +133,58 @@ static inline int raid_cpu_has_sse2(void)
 	return raid_cpu_match_sse(0, leaf_1_edx);
 }
 
+static inline int raid_cpu_has_clflush(void)
+{
+	/*
+	 * Intel 64 and IA-32 Architectures Software Developer's Manual
+	 * CPUID - CPU Identification
+	 *
+	 * Check that the processor supports CLFLUSH (true if CPUID.01H:EDX.CLFSH[bit 19] = 1).
+	 */
+	uint32_t leaf_1_edx = 1 << 19; /* CLFLUSH */
+
+	return raid_cpu_match_sse(0, leaf_1_edx);
+}
+
+static inline int raid_cpu_has_clflushopt(void)
+{
+	/*
+	 * Intel 64 and IA-32 Architectures Software Developer's Manual
+	 * CPUID - CPU Identification
+	 *
+	 * Check that the processor supports CLFLUSHOPT (true if CPUID.07H.00H:EBX.CLFLUSHOPT[bit 23] = 1).
+	 */
+	uint32_t reg[4];
+	uint32_t leaf_7_ebx = 1 << 23; /* CLFLUSHOPT */
+
+	/* check that leaf 7 is supported */
+	raid_cpuid(0, 0, reg);
+	if (reg[0] < 7)
+		return 0;
+
+	raid_cpuid(7, 0, reg);
+	if ((reg[1] & leaf_7_ebx) != leaf_7_ebx)
+		return 0;
+
+	return 1;
+}
+
+static inline unsigned raid_cpu_clflush_size(void)
+{
+	/*
+	 * Intel 64 and IA-32 Architectures Software Developer's Manual
+	 * CPUID - CPU Identification
+	 *
+	 * CPUID.01H:EBX.CLFLUSH[bits 15:8] reports the CLFLUSH cache-line
+	 * size in units of 8 bytes.
+	 */
+	uint32_t reg[4];
+
+	raid_cpuid(1, 0, reg);
+
+	return ((reg[1] >> 8) & 0xff) * 8;
+}
+
 static inline int raid_cpu_has_ssse3(void)
 {
 	/*

raid/memory.c

@@ -2,6 +2,7 @@
 // Copyright (C) 2013 Andrea Mazzoleni
 
 #include "internal.h"
+#include "cpu.h"
 #include "memory.h"
 
 void *raid_malloc_align(size_t size, size_t align_size, void **freeptr)
@@ -184,12 +185,71 @@ void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv)
 		}
 }
 
+void raid_mcache_flush(void *ptr, size_t size)
+{
+#ifdef CONFIG_X86
+	uint8_t *p = ptr;
+	size_t cache_line_size;
+	size_t offset;
+	int has_clflushopt;
+
+	/* MFENCE is used to complete CLFLUSHOPT before the caller reads again. */
+	if (size == 0 || !raid_cpu_has_sse2() || !raid_cpu_has_clflush())
+		return;
+
+	cache_line_size = raid_cpu_clflush_size();
+	if (cache_line_size == 0)
+		return;
+
+	has_clflushopt = raid_cpu_has_clflushopt();
+	if (has_clflushopt) {
+		for (offset = 0; offset < size; offset += cache_line_size)
+			asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + offset)));
+		asm volatile ("clflushopt %0" : "+m" (*(volatile uint8_t *)(p + size - 1)));
+	} else {
+		for (offset = 0; offset < size; offset += cache_line_size)
+			asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + offset)));
+		asm volatile ("clflush %0" : "+m" (*(volatile uint8_t *)(p + size - 1)));
+	}
+
+	/* Cache invalidation must be globally visible before returning. */
+	asm volatile ("mfence" : : : "memory");
+#else
+	(void)ptr;
+	(void)size;
+#endif
+}
+
+static uintptr_t raid_mtest_address_pattern(const void *ptr)
+{
+	uintptr_t value = (uintptr_t)ptr;
+
+	/* Mix every address bit while retaining a one-to-one word mapping. */
+	value ^= value >> 17;
+	value *= (uintptr_t)0xed5ad4bbU;
+	value ^= value >> 11;
+	value *= (uintptr_t)0xac4c1b51U;
+	value ^= value >> 15;
+
+	return value;
+}
+
 int raid_mtest_vector(int n, size_t size, void **vv)
 {
+	static const uint8_t pattern[] = {
+		0x00, 0xff, 0x03, 0xfc, 0x0a, 0xf5, 0x09, 0xf6,
+		0x59, 0xa6, 0x5a, 0xa5, 0x53, 0xac, 0x50, 0xaf,
+		0xf0, 0x0f, 0xf3, 0x0c, 0xfa, 0x05, 0xf9, 0x06,
+		0xa9, 0x56, 0xaa, 0x55, 0xa3, 0x5c, 0xa0, 0x5f,
+		0x95, 0x6a, 0x96, 0x69, 0x9f, 0x60, 0x9c, 0x63,
+		0xcc, 0x33, 0xcf, 0x30, 0xc6, 0x39, 0xc5, 0x3a,
+		0x65, 0x9a, 0x66, 0x99, 0x6f, 0x90, 0x6c, 0x93,
+		0x3c, 0xc3, 0x3f, 0xc0, 0x36, 0xc9, 0x35, 0xca
+	};
 	uint8_t **v = (uint8_t **)vv;
 	int i;
 	size_t j;
-	unsigned k;
+	size_t k;
 	uint8_t d;
 	uint8_t p;
 
@@ -198,11 +258,17 @@ int raid_mtest_vector(int n, size_t size, void **vv)
 	for (i = 0; i < n; ++i)
 		for (j = 0; j < size; ++j)
 			v[i][j] = d;
+	for (i = 0; i < n; ++i)
+		raid_mcache_flush(v[i], size);
 
-	/* test with all the byte patterns */
-	for (k = 1; k < 256; ++k) {
+	/*
+	 * Complementary pairs make each direction cover all two-bit combinations
+	 * uniformly, and together they cover all three-bit combinations uniformly.
+	 * Transitions between pairs change six bits, except for one four-bit change.
+	 */
+	for (k = 0; k < sizeof(pattern) / sizeof(pattern[0]); k += 2) {
 		p = d;
-		d = k;
+		d = pattern[k];
 
 		/* forward fill */
 		for (i = 0; i < n; ++i) {
@@ -215,10 +281,13 @@ int raid_mtest_vector(int n, size_t size, void **vv)
 				v[i][j] = d;
 			}
 		}
+		for (i = 0; i < n; ++i)
+			raid_mcache_flush(v[i], size);
 
 		p = d;
-		d = ~p;
-		/* backward fill with complement */
+		d = pattern[k + 1];
+
+		/* backward fill */
 		for (i = 0; i < n; ++i) {
 			for (j = size; j > 0; --j) {
 				if (v[i][j - 1] != p) {
@@ -229,6 +298,65 @@ int raid_mtest_vector(int n, size_t size, void **vv)
 				v[i][j - 1] = d;
 			}
 		}
+		for (i = 0; i < n; ++i)
+			raid_mcache_flush(v[i], size);
+	}
+
+	/*
+	 * Equal byte patterns cannot reveal two memory locations aliasing each
+	 * other. Give every word a value derived from its address, then repeat
+	 * the verification with the complement to exercise both bit values.
+	 */
+	for (i = 0; i < n; ++i) {
+		for (j = 0; j < size; j += sizeof(uintptr_t)) {
+			uintptr_t value = raid_mtest_address_pattern(v[i] + j);
+			size_t run = size - j;
+
+			if (run > sizeof(value))
+				run = sizeof(value);
+			memcpy(v[i] + j, &value, run);
+		}
+		raid_mcache_flush(v[i], size);
+	}
+
+	/* verify forward and replace every address pattern with its complement */
+	for (i = 0; i < n; ++i) {
+		for (j = 0; j < size; j += sizeof(uintptr_t)) {
+			uintptr_t value = raid_mtest_address_pattern(v[i] + j);
+			size_t run = size - j;
+
+			if (run > sizeof(value))
+				run = sizeof(value);
+			if (memcmp(v[i] + j, &value, run) != 0) {
+				/* LCOV_EXCL_START */
+				return -1;
+				/* LCOV_EXCL_STOP */
+			}
+			value = ~value;
+			memcpy(v[i] + j, &value, run);
+		}
+		raid_mcache_flush(v[i], size);
+	}
+
+	/* verify backward and restore the zero-filled result expected by callers */
+	for (i = 0; i < n; ++i) {
+		j = size;
+		while (j != 0) {
+			uintptr_t value;
+			size_t run = j % sizeof(uintptr_t);
+
+			if (run == 0)
+				run = sizeof(uintptr_t);
+			j -= run;
+			value = ~raid_mtest_address_pattern(v[i] + j);
+			if (memcmp(v[i] + j, &value, run) != 0) {
+				/* LCOV_EXCL_START */
+				return -1;
+				/* LCOV_EXCL_STOP */
+			}
+			memset(v[i] + j, 0, run);
+		}
+		raid_mcache_flush(v[i], size);
 	}
 
 	return 0;

raid/memory.h

@@ -115,6 +115,14 @@ void **raid_malloc_vector_align(int n, size_t size, size_t align_size, size_t di
  */
 void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv);
 
+/**
+ * Writes back and invalidates the CPU cache lines covering the memory range.
+ *
+ * This operation is best effort. It does nothing on architectures without a
+ * safe user-space cache invalidation instruction.
+ */
+void raid_mcache_flush(void *ptr, size_t size);
+
 /**
  * Tests the memory vector for RAM problems.
  * If a problem is found, it crashes.