Optimizes the cache use for all the RAID computations.

HISTORY

@@ -19,6 +19,7 @@ SnapRAID HISTORY
  * Added a hash seed to make harder intentional collision attacks.
  * When insering files for the first time, sort them by their physical
    address to improve read performance.
+ * Optimized the cache use for the all the RAID computations.
  * Better selection of the RAID6 implementation for different CPUs.
  * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4.
    They are a little faster than the previous unroll by 2.

check.c

@@ -437,7 +437,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 	block_off_t i;
 	unsigned j;
 	void* buffer_alloc;
-	unsigned char* buffer_aligned;
 	unsigned char** buffer;
 	unsigned buffermax;
 	int ret;
@@ -455,11 +454,9 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 	/* we need disk + 2 for each parity level buffers + 1 zero buffer */
 	buffermax = diskmax + state->level * 2 + 1;
 
-	buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc);
-	buffer = malloc_nofail(buffermax * sizeof(void*));
-	for(i=0;i<buffermax;++i) {
-		buffer[i] = buffer_aligned + i * state->block_size;
-	}
+	buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc);
+
+	/* fill up the zero buffer */
 	memset(buffer[buffermax-1], 0, state->block_size);
 
 	failed = malloc_nofail(diskmax * sizeof(struct failed_struct));

scrub.c

@@ -44,7 +44,6 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 	block_off_t i;
 	unsigned j;
 	void* buffer_alloc;
-	unsigned char* buffer_aligned;
 	unsigned char** buffer;
 	unsigned buffermax;
 	data_off_t countsize;
@@ -67,11 +66,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 	/* we need disk + 2 for each parity level buffers */
 	buffermax = diskmax + state->level * 2;
 
-	buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc);
-	buffer = malloc_nofail(buffermax * sizeof(void*));
-	for(i=0;i<buffermax;++i) {
-		buffer[i] = buffer_aligned + i * state->block_size;
-	}
+	buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc);
 
 	error = 0;
 	silent_error = 0;

selftest.c

@@ -46,18 +46,15 @@ static struct hash_test_vector TEST_SPOOKY2[] = {
 static void raid5test(unsigned diskmax, unsigned block_size)
 {
 	void* buffer_alloc;
-	unsigned char* buffer_aligned;
 	unsigned char** buffer;
 	unsigned char* buffer_save;
 	unsigned buffermax;
 	unsigned i, j;
 
 	buffermax = diskmax + 1 + 1;
-	buffer_aligned = malloc_nofail_align(buffermax * block_size, &buffer_alloc);
-	buffer = malloc_nofail(buffermax * sizeof(void*));
-	for(i=0;i<buffermax;++i) {
-		buffer[i] = buffer_aligned + i * block_size;
-	}
+
+	buffer = malloc_nofail_vector_align(buffermax, block_size, &buffer_alloc);
+
 	buffer_save = buffer[buffermax-1];
 
 	/* fill with random */
@@ -92,7 +89,6 @@ static void raid5test(unsigned diskmax, unsigned block_size)
 static void raid6test(unsigned diskmax, unsigned block_size)
 {
 	void* buffer_alloc;
-	unsigned char* buffer_aligned;
 	unsigned char** buffer;
 	unsigned char* buffer_save0;
 	unsigned char* buffer_save1;
@@ -101,11 +97,9 @@ static void raid6test(unsigned diskmax, unsigned block_size)
 	unsigned i, j;
 
 	buffermax = diskmax + 2 + 2 + 1;
-	buffer_aligned = malloc_nofail_align(buffermax * block_size, &buffer_alloc);
-	buffer = malloc_nofail(buffermax * sizeof(void*));
-	for(i=0;i<buffermax;++i) {
-		buffer[i] = buffer_aligned + i * block_size;
-	}
+
+	buffer = malloc_nofail_vector_align(buffermax, block_size, &buffer_alloc);
+
 	buffer_save0 = buffer[buffermax-3];
 	buffer_save1 = buffer[buffermax-2];
 	buffer_zero = buffer[buffermax-1];

speed.c

@@ -64,13 +64,12 @@ void speed(void)
 	unsigned block_size = 256 * 1024;
 	unsigned diskmax = 8;
 	void* buffer_alloc;
-	unsigned char* buffer_aligned;
 	unsigned char** buffer;
 
-	buffer_aligned = malloc_nofail_align((diskmax + 2) * block_size, &buffer_alloc);
-	buffer = malloc_nofail((diskmax + 2) * sizeof(void*));
-	for(i=0;i<diskmax+2;++i) {
-		buffer[i] = buffer_aligned + i * block_size;
+	buffer = malloc_nofail_vector_align(diskmax + 3, block_size, &buffer_alloc);
+
+	/* initialize with fixed data */
+	for(i=0;i<diskmax+3;++i) {
 		memset(buffer[i], i, block_size);
 	}
 	for(i=0;i<HASH_SIZE;++i)

sync.c

@@ -44,7 +44,6 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 	block_off_t i;
 	unsigned j;
 	void* buffer_alloc;
-	unsigned char* buffer_aligned;
 	unsigned char** buffer;
 	unsigned buffermax;
 	data_off_t countsize;
@@ -69,11 +68,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 	/* we need disk + 1 for each parity level buffers */
 	buffermax = diskmax + state->level;
 
-	buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc);
-	buffer = malloc_nofail(buffermax * sizeof(void*));
-	for(i=0;i<buffermax;++i) {
-		buffer[i] = buffer_aligned + i * state->block_size;
-	}
+	buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc);
 
 	unrecoverable_error = 0;
 

util.c

@@ -1325,26 +1325,69 @@ void* malloc_nofail(size_t size)
 
 /**
  * Memory alignment provided by malloc_nofail_align().
+ *
  * It should guarantee good cache performance everywhere.
  */
-#define MALLOC_NOFAIL_ALIGN 256
+#define MALLOC_ALIGN 256
 
 void* malloc_nofail_align(size_t size, void** freeptr)
 {
-	unsigned char* ptr = malloc_nofail(size + MALLOC_NOFAIL_ALIGN);
+	unsigned char* ptr = malloc_nofail(size + MALLOC_ALIGN);
 	uintptr_t offset;
 
 	*freeptr = ptr;
 
-	offset = ((uintptr_t)ptr) % MALLOC_NOFAIL_ALIGN;
+	offset = ((uintptr_t)ptr) % MALLOC_ALIGN;
 
 	if (offset != 0) {
-		ptr += MALLOC_NOFAIL_ALIGN - offset;
+		ptr += MALLOC_ALIGN - offset;
 	}
 
 	return ptr;
 }
 
+/**
+ * Memory displacement to avoid cache collisions on contiguous blocks.
+ *
+ * When allocating a sequence of blocks with a size of power of 2,
+ * there is the risk that the start of each block is mapped into the same cache line,
+ * resulting in cache collisions if you access all the blocks in parallel,
+ * from the start to the end.
+ *
+ * The selected value was choosen empirically with some speed tests
+ * with 8/12/16/20/24 data buffers and 3 parity buffers
+ * for RAID-5/6/TP computation.
+ *
+ * With displacement (8 buffers, icore5, 32 bit):
+ *
+ * RAID5 sse2x4 21936 [MB/s]
+ * RAID6 sse2x2 11902 [MB/s]
+ * RAIDTP sse2x1 5838 [MB/s]
+ *
+ * Without displacement:
+ *
+ * RAID5 sse2x4 15368 [MB/s]
+ * RAID6 sse2x2 6814 [MB/s]
+ * RAIDTP sse2x1 3033 [MB/s]
+ */
+#define MALLOC_DISPLACEMENT (7*256)
+
+unsigned char** malloc_nofail_vector_align(size_t count, size_t size, void** freeptr)
+{
+	unsigned char** buffer;
+	unsigned char* buffer_aligned;
+	size_t i;
+
+	buffer = malloc_nofail(count * sizeof(void*));
+
+	buffer_aligned = malloc_nofail_align(count * (size + MALLOC_DISPLACEMENT), freeptr);
+
+	for(i=0;i<count;++i)
+		buffer[i] = buffer_aligned + i * (size + MALLOC_DISPLACEMENT);
+
+	return buffer;
+}
+
 char* strdup_nofail(const char* str)
 {
 	size_t size;

util.h

@@ -457,6 +457,13 @@ void* malloc_nofail(size_t size);
  */
 void* malloc_nofail_align(size_t size, void** freeptr);
 
+/**
+ * Safe aligned vector allocation.
+ * Returns a vector of "count" pointers, each one pointing to a block of the specified "size".
+ * If no memory is available, it aborts.
+ */
+unsigned char** malloc_nofail_vector_align(size_t count, size_t size, void** freeptr);
+
 /**
  * Safe strdup.
  * If no memory is available, it aborts.