diff --git a/HISTORY b/HISTORY index 9d45ae88..6c9003b9 100644 --- a/HISTORY +++ b/HISTORY @@ -19,6 +19,7 @@ SnapRAID HISTORY * Added a hash seed to make harder intentional collision attacks. * When insering files for the first time, sort them by their physical address to improve read performance. + * Optimized the cache use for the all the RAID computations. * Better selection of the RAID6 implementation for different CPUs. * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4. They are a little faster than the previous unroll by 2. diff --git a/check.c b/check.c index 2bd76db4..c94e87dc 100644 --- a/check.c +++ b/check.c @@ -437,7 +437,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, block_off_t i; unsigned j; void* buffer_alloc; - unsigned char* buffer_aligned; unsigned char** buffer; unsigned buffermax; int ret; @@ -455,11 +454,9 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* we need disk + 2 for each parity level buffers + 1 zero buffer */ buffermax = diskmax + state->level * 2 + 1; - buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc); - buffer = malloc_nofail(buffermax * sizeof(void*)); - for(i=0;iblock_size; - } + buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc); + + /* fill up the zero buffer */ memset(buffer[buffermax-1], 0, state->block_size); failed = malloc_nofail(diskmax * sizeof(struct failed_struct)); diff --git a/scrub.c b/scrub.c index b0ccb9f8..b15cc4cc 100644 --- a/scrub.c +++ b/scrub.c @@ -44,7 +44,6 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par block_off_t i; unsigned j; void* buffer_alloc; - unsigned char* buffer_aligned; unsigned char** buffer; unsigned buffermax; data_off_t countsize; @@ -67,11 +66,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par /* we need disk + 2 for each parity level buffers */ buffermax = diskmax + state->level * 2; - buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc); - buffer = malloc_nofail(buffermax * sizeof(void*)); - for(i=0;iblock_size; - } + buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc); error = 0; silent_error = 0; diff --git a/selftest.c b/selftest.c index a5a01c4e..84e982e4 100644 --- a/selftest.c +++ b/selftest.c @@ -46,18 +46,15 @@ static struct hash_test_vector TEST_SPOOKY2[] = { static void raid5test(unsigned diskmax, unsigned block_size) { void* buffer_alloc; - unsigned char* buffer_aligned; unsigned char** buffer; unsigned char* buffer_save; unsigned buffermax; unsigned i, j; buffermax = diskmax + 1 + 1; - buffer_aligned = malloc_nofail_align(buffermax * block_size, &buffer_alloc); - buffer = malloc_nofail(buffermax * sizeof(void*)); - for(i=0;ilevel; - buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc); - buffer = malloc_nofail(buffermax * sizeof(void*)); - for(i=0;iblock_size; - } + buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc); unrecoverable_error = 0; diff --git a/util.c b/util.c index adca959c..b46e7f33 100644 --- a/util.c +++ b/util.c @@ -1325,26 +1325,69 @@ void* malloc_nofail(size_t size) /** * Memory alignment provided by malloc_nofail_align(). + * * It should guarantee good cache performance everywhere. */ -#define MALLOC_NOFAIL_ALIGN 256 +#define MALLOC_ALIGN 256 void* malloc_nofail_align(size_t size, void** freeptr) { - unsigned char* ptr = malloc_nofail(size + MALLOC_NOFAIL_ALIGN); + unsigned char* ptr = malloc_nofail(size + MALLOC_ALIGN); uintptr_t offset; *freeptr = ptr; - offset = ((uintptr_t)ptr) % MALLOC_NOFAIL_ALIGN; + offset = ((uintptr_t)ptr) % MALLOC_ALIGN; if (offset != 0) { - ptr += MALLOC_NOFAIL_ALIGN - offset; + ptr += MALLOC_ALIGN - offset; } return ptr; } +/** + * Memory displacement to avoid cache collisions on contiguous blocks. + * + * When allocating a sequence of blocks with a size of power of 2, + * there is the risk that the start of each block is mapped into the same cache line, + * resulting in cache collisions if you access all the blocks in parallel, + * from the start to the end. + * + * The selected value was choosen empirically with some speed tests + * with 8/12/16/20/24 data buffers and 3 parity buffers + * for RAID-5/6/TP computation. + * + * With displacement (8 buffers, icore5, 32 bit): + * + * RAID5 sse2x4 21936 [MB/s] + * RAID6 sse2x2 11902 [MB/s] + * RAIDTP sse2x1 5838 [MB/s] + * + * Without displacement: + * + * RAID5 sse2x4 15368 [MB/s] + * RAID6 sse2x2 6814 [MB/s] + * RAIDTP sse2x1 3033 [MB/s] + */ +#define MALLOC_DISPLACEMENT (7*256) + +unsigned char** malloc_nofail_vector_align(size_t count, size_t size, void** freeptr) +{ + unsigned char** buffer; + unsigned char* buffer_aligned; + size_t i; + + buffer = malloc_nofail(count * sizeof(void*)); + + buffer_aligned = malloc_nofail_align(count * (size + MALLOC_DISPLACEMENT), freeptr); + + for(i=0;i