diff --git a/cmdline/check.c b/cmdline/check.c index 3fc19f8a..e39e9d96 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -90,7 +90,7 @@ * REP -> OLD data is represented * DELETED -> OLD data is represented * REBUILD -> data and hash remain CURRENT; only physical parity is untrusted - * + * * * REBUILD is orthogonal to the OLD/CURRENT sync-history distinction. Unlike * CHG, REP and DELETED, it does not represent a different data generation. @@ -355,7 +355,7 @@ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned d * If we checked something, and no block failed the check * recompute all the redundancy information */ - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 1; } @@ -365,12 +365,12 @@ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned d static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, unsigned i, void** buffer, void** buffer_recov) { /* recompute parity, note that we don't need parity over i */ - raid_gen(diskmax, i + 1, state->block_size, buffer); + raid_gen(diskmax, i + 1, state->block_size, buffer, 0); /* if the recovered parity block matches */ if (memcmp(buffer[diskmax + i], buffer_recov[i], state->block_size) == 0) { /* recompute all the redundancy information */ - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 1; } @@ -401,7 +401,7 @@ static int repair_step(struct snapraid_state* state, int rehash, block_off_t pos if (failed_count == 0) { /* LCOV_EXCL_START */ /* recompute only the parity */ - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 0; /* LCOV_EXCL_STOP */ } @@ -686,7 +686,7 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns * actual recovery to perform. */ if (failed_count == 0) { - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 0; } @@ -826,7 +826,7 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns if (!something_to_recover) { log_tag("recover_sync:%" PRIu64 ":%u: Skipped for already recovered\n", pos, n); - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 0; } @@ -2082,8 +2082,8 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna * A partial -S/-B fix operates only at block level, so keep FILE_IS_DAMAGED * but leave the file under its normal name when partial is set. */ - if (fix - && !file_flag_has(failed[j].file, FILE_IS_EXCLUDED) + if (fix + && !file_flag_has(failed[j].file, FILE_IS_EXCLUDED) && !(state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED)) && !partial ) { @@ -3053,3 +3053,4 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b return 0; } + diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 8dc3d313..529ba561 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -824,7 +824,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) { /* compute the parity */ - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 0); /* compare the parity */ for (l = 0; l < state->level; ++l) { diff --git a/cmdline/speed.c b/cmdline/speed.c index 2dd3446e..637a0900 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -248,14 +248,14 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8s", ""); SPEED_START { - raid_gen1_int32(nd, size, v); + raid_gen1_int32(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_gen1_int64(nd, size, v); + raid_gen1_int64(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -263,7 +263,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON SPEED_START { - raid_gen1_neon(nd, size, v); + raid_gen1_neon(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -271,7 +271,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen1_neon32(nd, size, v); + raid_gen1_neon32(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -281,7 +281,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { - raid_gen1_sse2(nd, size, v); + raid_gen1_sse2(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -299,7 +299,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms } if (raid_cpu_has_avx2()) { SPEED_START { - raid_gen1_avx2(nd, size, v); + raid_gen1_avx2(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -311,7 +311,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen1_avx512bw(nd, size, v); + raid_gen1_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -331,7 +331,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms fflush(stdout); SPEED_START { - raid_gen2_int8(nd, size, v); + raid_gen2_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -339,28 +339,28 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_int32_aes(nd, size, v); + raid_gen2_int32_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_gen2_int64_aes(nd, size, v); + raid_gen2_int64_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } else { SPEED_START { - raid_gen2_int32_raid(nd, size, v); + raid_gen2_int32_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_gen2_int64_raid(nd, size, v); + raid_gen2_int64_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -370,11 +370,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_neon_aes(nd, size, v); + raid_gen2_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_neon_raid(nd, size, v); + raid_gen2_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -384,11 +384,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_neon32_aes(nd, size, v); + raid_gen2_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_neon32_raid(nd, size, v); + raid_gen2_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -400,11 +400,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_sse2()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_sse2_aes(nd, size, v); + raid_gen2_sse2_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_sse2_raid(nd, size, v); + raid_gen2_sse2_raid(nd, size, v, 1); } SPEED_STOP } @@ -414,11 +414,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_sse2ext_aes(nd, size, v); + raid_gen2_sse2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_sse2ext_raid(nd, size, v); + raid_gen2_sse2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -435,11 +435,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx2()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx2_aes(nd, size, v); + raid_gen2_avx2_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_avx2_raid(nd, size, v); + raid_gen2_avx2_raid(nd, size, v, 1); } SPEED_STOP } @@ -449,11 +449,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx2ext_aes(nd, size, v); + raid_gen2_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_avx2ext_raid(nd, size, v); + raid_gen2_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -464,7 +464,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen2_avx512bw(nd, size, v); + raid_gen2_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -473,13 +473,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx2gfni_aes(nd, size, v); + raid_gen2_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen2_avx2gfni_raid(nd, size, v); + raid_gen2_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -489,13 +489,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx512gfni_aes(nd, size, v); + raid_gen2_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen2_avx512gfni_raid(nd, size, v); + raid_gen2_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -512,7 +512,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms fflush(stdout); SPEED_START { - raid_gen3_int8(nd, size, v); + raid_gen3_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -524,11 +524,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_neon_aes(nd, size, v); + raid_gen3_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_neon_raid(nd, size, v); + raid_gen3_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -538,11 +538,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_neon32_aes(nd, size, v); + raid_gen3_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_neon32_raid(nd, size, v); + raid_gen3_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -561,11 +561,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_ssse3_aes(nd, size, v); + raid_gen3_ssse3_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_ssse3_raid(nd, size, v); + raid_gen3_ssse3_raid(nd, size, v, 1); } SPEED_STOP } @@ -575,11 +575,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_ssse3ext_aes(nd, size, v); + raid_gen3_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_ssse3ext_raid(nd, size, v); + raid_gen3_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -592,11 +592,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_avx2ext_aes(nd, size, v); + raid_gen3_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_avx2ext_raid(nd, size, v); + raid_gen3_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -607,7 +607,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen3_avx512bw(nd, size, v); + raid_gen3_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -616,13 +616,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_avx2gfni_aes(nd, size, v); + raid_gen3_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen3_avx2gfni_raid(nd, size, v); + raid_gen3_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -632,13 +632,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_avx512gfni_aes(nd, size, v); + raid_gen3_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen3_avx512gfni_raid(nd, size, v); + raid_gen3_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -655,7 +655,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms fflush(stdout); SPEED_START { - raid_gen4_int8(nd, size, v); + raid_gen4_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -667,11 +667,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_neon_aes(nd, size, v); + raid_gen4_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_neon_raid(nd, size, v); + raid_gen4_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -681,11 +681,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_neon32_aes(nd, size, v); + raid_gen4_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_neon32_raid(nd, size, v); + raid_gen4_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -704,11 +704,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_ssse3_aes(nd, size, v); + raid_gen4_ssse3_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_ssse3_raid(nd, size, v); + raid_gen4_ssse3_raid(nd, size, v, 1); } SPEED_STOP } @@ -718,11 +718,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_ssse3ext_aes(nd, size, v); + raid_gen4_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_ssse3ext_raid(nd, size, v); + raid_gen4_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -735,11 +735,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_avx2ext_aes(nd, size, v); + raid_gen4_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_avx2ext_raid(nd, size, v); + raid_gen4_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -750,7 +750,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen4_avx512bw(nd, size, v); + raid_gen4_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -759,13 +759,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_avx2gfni_aes(nd, size, v); + raid_gen4_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen4_avx2gfni_raid(nd, size, v); + raid_gen4_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -775,13 +775,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_avx512gfni_aes(nd, size, v); + raid_gen4_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen4_avx512gfni_raid(nd, size, v); + raid_gen4_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -798,7 +798,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms fflush(stdout); SPEED_START { - raid_gen5_int8(nd, size, v); + raid_gen5_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -810,11 +810,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_neon_aes(nd, size, v); + raid_gen5_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_neon_raid(nd, size, v); + raid_gen5_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -824,11 +824,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_neon32_aes(nd, size, v); + raid_gen5_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_neon32_raid(nd, size, v); + raid_gen5_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -846,7 +846,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_RAID) { SPEED_START { - raid_gen5_ssse3_raid(nd, size, v); + raid_gen5_ssse3_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -858,11 +858,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_ssse3ext_aes(nd, size, v); + raid_gen5_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_ssse3ext_raid(nd, size, v); + raid_gen5_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -875,11 +875,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_avx2ext_aes(nd, size, v); + raid_gen5_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_avx2ext_raid(nd, size, v); + raid_gen5_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -890,7 +890,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen5_avx512bw(nd, size, v); + raid_gen5_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -899,13 +899,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_avx2gfni_aes(nd, size, v); + raid_gen5_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen5_avx2gfni_raid(nd, size, v); + raid_gen5_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -915,13 +915,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_avx512gfni_aes(nd, size, v); + raid_gen5_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen5_avx512gfni_raid(nd, size, v); + raid_gen5_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -938,7 +938,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms fflush(stdout); SPEED_START { - raid_gen6_int8(nd, size, v); + raid_gen6_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -950,11 +950,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_neon_aes(nd, size, v); + raid_gen6_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_neon_raid(nd, size, v); + raid_gen6_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -964,11 +964,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_neon32_aes(nd, size, v); + raid_gen6_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_neon32_raid(nd, size, v); + raid_gen6_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -986,7 +986,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_RAID) { SPEED_START { - raid_gen6_ssse3_raid(nd, size, v); + raid_gen6_ssse3_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -998,11 +998,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_ssse3ext_aes(nd, size, v); + raid_gen6_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_ssse3ext_raid(nd, size, v); + raid_gen6_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -1015,11 +1015,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_avx2ext_aes(nd, size, v); + raid_gen6_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_avx2ext_raid(nd, size, v); + raid_gen6_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -1030,7 +1030,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen6_avx512bw(nd, size, v); + raid_gen6_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1039,13 +1039,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_avx2gfni_aes(nd, size, v); + raid_gen6_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen6_avx2gfni_raid(nd, size, v); + raid_gen6_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1055,13 +1055,13 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_avx512gfni_aes(nd, size, v); + raid_gen6_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen6_avx512gfni_raid(nd, size, v); + raid_gen6_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1126,14 +1126,14 @@ void speed_genz(int nd, void** v, int size, int delta, int period) printf("%8s", ""); SPEED_START { - raid_genz_int32_raid(nd, size, v); + raid_genz_int32_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_genz_int64_raid(nd, size, v); + raid_genz_int64_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1141,7 +1141,7 @@ void speed_genz(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_genz_neon_raid(nd, size, v); + raid_genz_neon_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1149,7 +1149,7 @@ void speed_genz(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_genz_neon32_raid(nd, size, v); + raid_genz_neon32_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1159,7 +1159,7 @@ void speed_genz(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { - raid_genz_sse2_raid(nd, size, v); + raid_genz_sse2_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1167,7 +1167,7 @@ void speed_genz(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { - raid_genz_sse2ext_raid(nd, size, v); + raid_genz_sse2ext_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1184,7 +1184,7 @@ void speed_genz(int nd, void** v, int size, int delta, int period) printf("%8s", ""); #ifdef CONFIG_X86_64 SPEED_START { - raid_genz_avx2ext_raid(nd, size, v); + raid_genz_avx2ext_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1287,6 +1287,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3(1, id, ip, nd, size, v); } SPEED_STOP @@ -1295,6 +1297,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3ext(1, id, ip, nd, size, v); } SPEED_STOP @@ -1304,6 +1308,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2(1, id, ip, nd, size, v); } SPEED_STOP @@ -1313,6 +1319,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx2()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2ext(1, id, ip, nd, size, v); } SPEED_STOP @@ -1321,6 +1329,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512bw()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512bw(1, id, ip, nd, size, v); } SPEED_STOP @@ -1329,6 +1339,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP @@ -1337,6 +1349,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP diff --git a/cmdline/sync.c b/cmdline/sync.c index 5e1de52c..815e8979 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1516,7 +1516,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* update the parity only if really needed */ if (parity_needs_to_be_updated) { /* compute the parity */ - raid_gen(diskmax, state->level, state->block_size, buffer); + raid_gen(diskmax, state->level, state->block_size, buffer, 1); /* until now is raid */ state_usage_raid(state); diff --git a/raid/NOTES b/raid/NOTES index 4b11e520..7c407a14 100644 --- a/raid/NOTES +++ b/raid/NOTES @@ -57,28 +57,25 @@ Generation: Recovery: -* The normal one-data-disk recovery with P available is handled by dedicated - XOR-only 1-pass kernels on SSSE3, SSSE3 extended, AVX2, AVX2 extended, - AVX512BW, and GFNI (AVX2 and AVX512). Scalar, AArch64 NEON, and AArch32 NEON - continue to use raid_rec1of1(). +* The normal one-data-disk recovery with P available is handled via raid_rec1of1() + across all backends (Scalar, NEON, SSSE3, AVX2, AVX512BW, GFNI), invoking + raid_gen(nd, 1, size, v, 0) with temporal stores. * SIMD recovery has parity-count-specific REC1..REC6 entry points. Fused paths compute exactly the selected syndromes in one scan of the surviving data and reconstruct the missing blocks without full-size temporary syndrome buffers. -* AVX512BW and GFNI (AVX2 and AVX512) use dedicated XOR-only kernels for - rec1of1/P and fused syndrome generation for non-P REC1 and all REC2..REC6 - cases, including P+Q (rec2of2). AVX2 and AVX2 extended share a dedicated - XOR-only rec1of1/P kernel with temporal stores; AVX2 uses a specialized - 1-pass Horner kernel for Q, while both use fused selected-syndrome recovery - for the other non-P REC1 cases and fused recovery for all REC2..REC6 cases. - SSSE3 and SSSE3 extended likewise share a dedicated XOR-only rec1of1/P kernel - with temporal stores, use specialized 1-pass Horner kernels for Q-only REC1 - and P+Q REC2, and use fused selected-syndrome recovery for the remaining - REC1..REC6 cases. AArch64 NEON uses raid_rec1of1() for P and fused syndrome - generation for non-P REC1 and all REC2..REC6 cases. AArch32 NEON uses - raid_rec1of1() for P, fused syndrome generation for non-P REC1, all REC2 - cases except P+Q, and REC3..REC6. +* AVX512BW and GFNI (AVX2 and AVX512) use raid_rec1of1() for P and fused syndrome + generation for non-P REC1 and all REC2..REC6 cases, including P+Q (rec2of2). + AVX2 and AVX2 extended use raid_rec1of1() for P, use specialized 1-pass + Horner kernels for Q, and use fused selected-syndrome recovery for the other + non-P REC1 cases and fused recovery for all REC2..REC6 cases. + SSSE3 and SSSE3 extended likewise use raid_rec1of1() for P, use specialized + 1-pass Horner kernels for Q-only REC1 and P+Q REC2, and use fused + selected-syndrome recovery for the remaining REC1..REC6 cases. AArch64 NEON + uses raid_rec1of1() for P and fused syndrome generation for non-P REC1 and all + REC2..REC6 cases. AArch32 NEON uses raid_rec1of1() for P, fused syndrome + generation for non-P REC1, all REC2 cases except P+Q, and REC3..REC6. * Scalar non-P REC1 and scalar REC2, plus AArch32 NEON P+Q, continue to use raid_delta_gen(), while scalar REC3..REC6 uses fused syndrome recovery. @@ -160,43 +157,38 @@ Real workload assumptions in cache is therefore not necessarily useful. -MOVNT / non-temporal stores ---------------------------- +MOVNT / non-temporal stores vs cached generation +-------------------------------------------------- -* Generation backends use MOVNTDQ / VMOVNTDQ for parity output where applicable. +* raid_gen(..., streaming) allows the caller to select between non-temporal + streaming stores (streaming != 0, using MOVNTDQ/VMOVNTDQ + sfence on x86) and + normal temporal/cached stores (streaming == 0, using MOVDQA/VMOVDQA without sfence). + +* Generation backends select MOVNTDQ / VMOVNTDQ for parity output when + streaming != 0 is requested. * The synthetic speedtest naturally favors non-temporal stores because generated parity is not consumed afterwards. This benchmark therefore measures producer throughput, not the complete producer -> queue -> writer pipeline. -* However, the real SnapRAID queue makes the workload much closer to the - non-temporal case than to an immediate-consumer case. +* In the real SnapRAID sync pipeline, parity blocks are queued to disk writers, + which makes the workload non-temporal (streaming = 1). At typical queue depths, + cached parity output would occupy many MiB of cache and evict useful input data; + MOVNT avoids normal cache allocation/write-allocate traffic and reduces cache + pressure. -* Cached stores could potentially help only with very short queues, where the - writer consumes parity while it is still hot. Queue depths around 1..4 are the - most plausible area where this could happen. - -* Those very short queues were not representative of the fastest measured - SnapRAID disk pipeline in these tests. In that workload, increasing the queue - depth up to about 32 provided substantial throughput improvements. - -* At queue depths representative of those measurements, cached parity output - can occupy many MiB of cache and can evict useful input data. MOVNT avoids - normal cache allocation/write-allocate traffic for parity output and reduces - this source of cache pressure. - -* Conclusion: for the tested large-block, queued SnapRAID generation workload, - keep MOVNT as the default generation policy. Replacing MOVNT with cached - stores is not currently considered an open optimization. This conclusion - should be revisited if block size, queueing behavior, target CPU, or consumer - locality changes. +* Conversely, in-memory parity verification in scrub and check, delta parity + generation in raid_delta_gen(), and single-parity recovery in raid_rec1of1() + use cached generation (streaming = 0) so that computed parity or recovered + data remains hot in cache for immediate CPU consumption. Recovery temporal stores ------------------------ -* Recovery output has a different lifetime from generated parity. A reconstructed - data block is an intermediate result that is consumed again immediately. +* Recovery output has a different lifetime from generated parity written to disk. + A reconstructed data block is an intermediate result that is consumed again + immediately. * After raid_data() reconstructs a candidate, repair_step() immediately validates it. With a trustworthy hash, blockcmp()/memhash_block() reads the recovered @@ -208,22 +200,21 @@ Recovery temporal stores so keeping recently produced cache lines available can also help the following write path. -* For this reason optimized x86 REC1OF1/P kernels use ordinary temporal - MOVDQA/VMOVDQA stores for recovered data rather than MOVNTDQ/VMOVNTDQ. The - isolated REC1OF1 speed test may report lower producer-only throughput because - temporal stores allocate cache lines and create cache pressure, but that test - stops before the immediate consumers that motivate the policy. +* For this reason, recovery uses ordinary temporal MOVDQA/VMOVDQA stores for + recovered data rather than MOVNTDQ/VMOVNTDQ. Single-disk recovery with P + (raid_rec1of1()) invokes raid_gen(nd, 1, size, v, 0) with streaming = 0, + ensuring temporal stores and omitting sfence across all backends. * With typical 256 KiB blocks and roughly 8..12 data disks, the full recovery working set may fit in L2 on some CPUs and exceed it on others. Do not switch store policy at an architecture-specific disk-count/cache-size threshold: the recovered block itself has strong immediate reuse on all of these machines, - while a generated parity block is normally queued and consumed later. Temporal - recovery stores are therefore the general compromise. + while generated parity written during sync is normally queued and consumed later. + Temporal recovery stores are therefore the general compromise. -* Store policy is deliberately asymmetric: generation uses non-temporal stores - for final queued parity output, while recovery uses temporal stores for data - that immediately becomes input to validation and parity regeneration. +* Store policy is deliberately selectable: generation for disk writing (sync) + uses non-temporal stores (streaming = 1), while recovery and in-memory + verification use temporal stores (streaming = 0). Typical recovery case @@ -235,15 +226,12 @@ Typical recovery case * raid_rec() selects the first available parity. Therefore, for a single data failure with P available, recovery uses P. -* REC1 implementations on SSSE3, SSSE3 extended, AVX2, AVX2 extended, - AVX512BW, and GFNI (AVX2 and AVX512) use dedicated XOR-only kernels for the P - case and temporal stores for the recovered data. Scalar, AArch64 NEON, and - AArch32 NEON paths call raid_rec1of1(). +* All backends (Scalar, NEON, SSSE3, AVX2, AVX512BW, GFNI) route the P case + through raid_rec1of1(). * raid_rec1of1() remaps pointers so that P is used as an XOR input and the - failed data buffer is used as the GEN1 output, then calls raid_gen(nd, 1, ...). - This generic path therefore inherits the store policy of the selected GEN1 - backend; optimized x86 recovery avoids that coupling with dedicated P kernels. + failed data buffer is used as the GEN1 output, then calls raid_gen(nd, 1, ..., 0). + All backends use this path with temporal stores for optimal P single-disk recovery. * Therefore the common single-disk recovery path does NOT call raid_delta_gen(). @@ -276,13 +264,8 @@ REC1OF1: * The row uses ip[0] == 0. -* On SSSE3, SSSE3 extended, AVX2, AVX2 extended, AVX512BW, and GFNI (AVX2 - and AVX512), REC1 uses a dedicated XOR-only 1-pass kernel that streams P and - the surviving data blocks directly into the recovered block using temporal - stores. - -* Scalar, AArch64 NEON, and AArch32 NEON wrappers enter raid_rec1of1() to - recompute P using the generation backend. +* All backends use raid_rec1of1(), calling the GEN1 backend with streaming = 0 + to reconstruct the missing block directly into cache using temporal stores. * Because the row measures only the recovery producer and does not include the immediate validation/parity-regeneration consumers, temporal-store x86 @@ -371,10 +354,9 @@ raid_delta_gen() fusion according to register pressure; they do not allocate full-size syndrome buffers. -* The common rec1of1/P case on SSSE3, SSSE3 extended, AVX2, AVX2 extended, - AVX512BW, and GFNI (AVX2 and AVX512) executes in dedicated XOR-only streaming - kernels, with no syndrome or GF recovery setup. Scalar, AArch64 NEON, and - AArch32 NEON use the raid_rec1of1() pointer remapping path. +* The common rec1of1/P case executes via raid_rec1of1() across all backends, + calling the GEN1 generator with streaming = 0 to write the recovered data + block using temporal stores with no syndrome or GF recovery setup. * raid_delta_gen() remains in scalar non-P REC1, scalar REC2, and AArch32 NEON P+Q. @@ -462,8 +444,10 @@ Generation: coefficient residency Recent GEN2/GEN3 work established coefficient residency as a practical generation optimization, not only a theoretical one. -* Non-GFNI SIMD GEN2/GEN3 kernels preload useful low/high multiplication tables - into vector registers when the register file has enough space. +* Non-GFNI SIMD GEN2/GEN3 kernels (such as AVX512BW) preload useful low/high + multiplication tables into vector registers when the register file has enough + space. AVX512 GFNI kernels similarly preload Cauchy affine matrices (for RAID) + or constant multiplier bytes (for AES). * The resident set can be partial. When there are more data disks than can be covered by spare registers, later disks fall back to loading their @@ -710,9 +694,9 @@ Memory layout, displacement and stride noise REC1 optimizations ------------------ -The normal REC1 case with P is handled by dedicated XOR-only 1-pass kernels on -AVX512BW and GFNI (AVX2 and AVX512), or via raid_rec1of1() on scalar, SSSE3, -AVX2, AArch64 NEON, and AArch32 NEON. +The normal REC1 case with P is handled via raid_rec1of1() across all backends +(scalar, SSSE3, AVX2, AVX512BW, GFNI, AArch64 NEON, and AArch32 NEON), using +streaming = 0 for temporal stores. For generic REC1, used when P is unavailable: @@ -726,7 +710,7 @@ For generic REC1, used when P is unavailable: REC1 status: * For the current large-block SnapRAID workload and implemented backends, - single-pass XOR streaming achieves optimal memory saturation for rec1of1/P. + raid_rec1of1() achieves optimal memory saturation for rec1of1/P. * Generic non-P REC1 is selected per parity count and backend, so single-pass SIMD and delta-based scalar implementations are benchmarked independently. @@ -1108,8 +1092,7 @@ Short list of ideas already tried or rejected SSSE3, SSSE3ext, AArch64 NEON, and AArch32 NEON. AVX2/AVX2ext and SSSE3/SSSE3ext use dedicated 1-pass Horner kernels for Q-only REC1, and SSSE3/SSSE3ext also use dedicated 1-pass Horner kernels for P+Q REC2. - AVX512BW and GFNI use dedicated XOR-only kernels for rec1of1, while the - other SIMD backends use raid_rec1of1(). Only scalar non-P REC1/scalar REC2 + All backends use raid_rec1of1() for rec1of1. Only scalar non-P REC1/scalar REC2 and AArch32 NEON P+Q retain raid_delta_gen(). * Software prefetch: diff --git a/raid/OTHERS b/raid/OTHERS index d6adc13c..3c19fcc5 100644 --- a/raid/OTHERS +++ b/raid/OTHERS @@ -91,8 +91,13 @@ SnapRAID: A storage-oriented RAID engine specialized for large fixed-size * Current optimized backends include x86 SSSE3, AVX2, AVX512BW and GFNI, plus AArch64 and AArch32 NEON. -* Generation uses non-temporal parity stores where appropriate because parity - normally enters a queue before the disk writer consumes it. +* Parity generation for disk sync uses non-temporal stores (VMOVNTDQ on x86) + because the writer queue is long enough that generated parity is evicted from + cache before being written to disk; temporal stores would pay Read-For-Ownership + (RFO) allocation, writeback, and later reload penalties without any useful cache hit. + Conversely, in-memory verification in scrub/check, delta parity generation, and + data recovery use temporal stores so the data remains cache-hot for immediate + CPU consumption. Linux RAID6: A kernel RAID6 P+Q engine. It is much less general than a Reed-Solomon library and therefore can specialize very @@ -104,8 +109,10 @@ Linux RAID6: A kernel RAID6 P+Q engine. It is much less general than a * Generation keeps P and Q in registers while walking all data disks for the current vector position, then writes both parity vectors once. -* x86 generation contains spatial x1/x2/x4 variants, explicit prefetch, and - non-temporal final stores in optimized paths. +* x86 generation (gen_syndrome) contains spatial x1/x2/x4 variants, explicit + prefetch, and non-temporal final stores (VMOVNTDQ) in optimized paths, + avoiding write-allocate because P/Q are write-only outputs destined for + storage I/O via DMA. * Recovery is specifically designed for one or two failures. Although the public RAID6 recovery interface accepts an arbitrary byte count, it is @@ -460,12 +467,13 @@ SnapRAID. For large blocks, the number of full source and destination passes can dominate small instruction-count savings. SnapRAID: The preferred organization is vector-offset outer, source-disk inner, - with all required parity accumulators live. A source chunk is loaded - once and contributes to all parity outputs before the next source is - loaded. + with all required parity accumulators live in vector registers. A + source chunk is loaded once from memory and contributes to all parity + outputs before the next source is loaded. Parity accumulators incur zero + intermediate memory traffic and are stored to RAM/disk exactly once. for each SIMD chunk: - initialize P/Q/R/... accumulators + initialize P/Q/R/... accumulators in SIMD registers (zero memory traffic) for each source disk: load source chunk once update all parity accumulators @@ -482,16 +490,21 @@ Linux RAID6: Very similar for P+Q generation. P and Q stay live while the OpenZFS RAIDZ: Processes 64 KiB cache-blocked regions. Within a region, each data column updates parity buffers that are loaded and stored - repeatedly. This deliberately trades more parity memory - operations for L2 locality and compatibility with RAIDZ/ABD - geometry. + repeatedly. This approach is dictated by ZFS software design + (handling scattered ABD kernel pages and dynamic variable-width + stripes column-by-column), but is computationally sub-optimal: + for N data disks and P parities, parity buffers are loaded and + stored N*P times. Even with a 100% L1/L2 cache hit rate, these + repeated passes consume CPU load and store execution ports, saturate + store buffers, and waste cache bandwidth compared to keeping + accumulators resident in vector registers. for each 64 KiB region: initialize parity region for each data column: load data - load/update/store P - load/update/store Q/R as required + load/update/store P <-- repeated memory & cache traffic + load/update/store Q/R as required <-- repeated memory & cache traffic ISA-L: Multi-output dot-product kernels load a source vector chunk, split it once, and apply several coefficient tables to several live destination @@ -512,11 +525,13 @@ CM256/gf256: Encoding one recovery block at a time also gives excellent coefficient residency, but producing several recovery blocks requires separate whole-region work for those outputs. -For SnapRAID, copying the Jerasure/CM256 whole-region orientation would -normally be the wrong trade. It could improve coefficient residency while -increasing 256 KiB source rereads. Current GEN2/GEN3 table residency shows that -SnapRAID can obtain part of the coefficient-residency benefit without changing -the single-pass multi-output traversal. +For SnapRAID, copying the Jerasure/CM256 whole-region orientation or the +OpenZFS column-by-column update loops would be strictly the wrong trade. +Rereading and rewriting intermediate parity across multiple passes wastes CPU +memory ports and cache bandwidth that SnapRAID completely avoids by holding +accumulators in vector registers. Current GEN2/GEN3 table residency shows that +SnapRAID can obtain coefficient residency while strictly preserving the optimal +single-pass multi-output traversal. Generation strategy @@ -728,15 +743,27 @@ penalty becomes even less attractive. Memory, cache and store behavior -------------------------------- -SnapRAID: Large source blocks and queued parity favor streaming behavior. - Generation normally writes parity once with non-temporal stores. - Keeping many generated parity blocks resident in cache is not the - objective. +SnapRAID: During sync, large source blocks and a deep writer queue make + parity output non-temporal. Parity blocks generated during sync are + queued and will likely be evicted from cache before the writer thread + consumes them; temporal stores would pay Read-For-Ownership (RFO) + line allocation, cache eviction/writeback, and later reload penalties + without achieving a useful cache hit. Non-temporal stores (VMOVNTDQ) + bypass write-allocate and avoid cache pollution. + Conversely, in-memory parity verification during scrub and check, + delta parity generation in raid_delta_gen(), and data recovery in + raid_rec1of1() use temporal cached stores (streaming = 0) so that + computed parity or reconstructed data remains hot in cache for + immediate CPU consumption. -Linux RAID6: Optimized generation is very similar: source prefetch and - non-temporal final P/Q stores are used in x86 paths. The separate +Linux RAID6: Optimized generation (gen_syndrome) is very similar: source + prefetch and non-temporal final P/Q stores (VMOVNTDQ) are used in + x86 paths because P and Q are write-only outputs destined for + storage I/O, where the ultimate consumer is the block device via + DMA. Keeping them cache-hot for subsequent CPU reads is unnecessary, + and non-temporal stores avoid write-allocate overhead. The separate xor_syndrome update operation reads and modifies existing parity - and therefore uses normal cached stores. + in-place and therefore uses normal cached stores. OpenZFS RAIDZ: Cached stores are integral to the traversal. P/Q/R are intentionally loaded and updated again as later data columns are @@ -941,9 +968,9 @@ What each implementation teaches SnapRAID Linux RAID6: The strongest reference for P+Q streaming generation. It confirms single-pass source traversal, live parity accumulators, - fixed-generator Horner arithmetic and non-temporal final stores as - sensible storage techniques. Its x1/x2/x4 unrolling and prefetch - are ideas to benchmark, not assumptions to copy. + fixed-generator Horner arithmetic and DMA-bound non-temporal final + stores as sensible storage techniques. Its x1/x2/x4 unrolling and + prefetch are ideas to benchmark, not assumptions to copy. OpenZFS RAIDZ: The strongest counterexample for memory policy. Its 64 KiB L2 blocking and cached parity stores are coherent because parity is diff --git a/raid/README b/raid/README index b475a360..bb583caf 100644 --- a/raid/README +++ b/raid/README @@ -38,11 +38,14 @@ library startup before normal RAID operations. The buffer must be aligned to a 64-byte boundary. Required prior to calling raid_rec() or raid_data() for delta parity computation. - void raid_gen(int nd, int np, size_t size, void **v) + void raid_gen(int nd, int np, size_t size, void **v, int streaming) Computes 'np' parity blocks from 'nd' data blocks. Vector 'v' contains 'nd + np' block pointers (data blocks v[0..nd-1], parity blocks v[nd..nd+np-1]). Each block must be aligned to a 64-byte boundary. - Data blocks are read-only; parity blocks are written. + Data blocks are read-only; parity blocks are written. If 'streaming' is + non-zero, non-temporal streaming stores are used where supported to avoid + polluting the CPU cache when parity is written directly to disk. If + 'streaming' is zero, normal cached stores are used. void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v) Recovers 'nr' failed blocks (data or parity) listed in sorted index array diff --git a/raid/avx2.c b/raid/avx2.c index 0e675f30..a2373145 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -11,7 +11,7 @@ * * Uses 64-byte chunks across two 32-byte YMM lanes. */ -void raid_gen1_avx2(int nd, size_t size, void **vv) +static __always_inline void raid_gen1_avx2_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -32,11 +32,24 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (v[d][i + 32])); } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (p[i + 32])); + } } - raid_avx_end(); + raid_avx_end(streaming); +} + +void raid_gen1_avx2(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen1_avx2_gen(nd, size, vv, 1); + else + raid_gen1_avx2_gen(nd, size, vv, 0); } /* @@ -44,7 +57,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) * * Uses Horner's method with 64-byte chunks across two 32-byte YMM lanes. */ -static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -100,13 +113,20 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (q[i + 32])); + } } - raid_avx_end(); + raid_avx_end(streaming); } #ifdef CONFIG_X86_64 @@ -118,7 +138,7 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -215,13 +235,20 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpxor %ymm5,%ymm9,%ymm3"); } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (q[i + 32])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -229,7 +256,7 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_genz_avx2ext_raid(int nd, size_t size, void **vv) +static __always_inline void raid_genz_avx2ext_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -288,15 +315,32 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovdqa %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovdqa %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovdqa %%ymm10,%0" : "=m" (r[i + 32])); + } } - raid_avx_end(); + raid_avx_end(streaming); +} + +void raid_genz_avx2ext_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_genz_avx2ext_gen(nd, size, vv, 1); + else + raid_genz_avx2ext_gen(nd, size, vv, 0); } /* @@ -307,7 +351,7 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -547,15 +591,24 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovdqa %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovdqa %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovdqa %%ymm10,%0" : "=m" (r[i + 32])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -566,7 +619,7 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -721,23 +774,34 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); asm volatile ("vpxor %ymm12,%ymm11,%ymm11"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovdqa %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovdqa %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovdqa %%ymm10,%0" : "=m" (r[i + 32])); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (s[i])); + asm volatile ("vmovdqa %%ymm11,%0" : "=m" (s[i + 32])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* * Generate N parity blocks with Cauchy matrix using AVX2 extended implementation. */ -static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator) +static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -875,69 +939,35 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in if (np >= 6) asm volatile ("vpxor %ymm10,%ymm5,%ymm5"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd + 1][i])); - if (np >= 3) - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 2][i])); - if (np >= 4) - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 3][i])); - if (np >= 5) - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 4][i])); - if (np >= 6) - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 5][i])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 5][i])); + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (v[nd + 5][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } #endif -/* - * Recover one data failure using P with AVX2. - * - * The missing block is: - * - * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn - * - * Use temporal stores because recovered data is consumed immediately by - * validation and parity regeneration. - */ -static __always_inline void raid_rec1of1_avx2(int *id, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *src[RAID_DATA_MAX]; - uint8_t *p; - uint8_t *pa; - size_t i; - int d, s, ns; - - p = v[nd]; - pa = v[id[0]]; - - ns = 0; - for (d = 0; d < nd; ++d) { - if (d != id[0]) - src[ns++] = v[d]; - } - - BUG_ON(ns != nd - 1); - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); - - for (s = 0; s < ns; ++s) { - asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (src[s][i])); - asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (src[s][i + 32])); - } - - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); - } - - raid_avx_end(); -} /* * Recover one data failure using selected parity with AVX2. @@ -1041,7 +1071,7 @@ static __always_inline void raid_rec1_avx2_1(int *id, int *ip, int nd, size_t si asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); } - raid_avx_end(); + raid_avx_end(0); } /* @@ -1161,7 +1191,7 @@ static __always_inline void raid_rec1_avx2_q(int *id, int *ip, int nd, size_t si asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); } - raid_avx_end(); + raid_avx_end(0); } /* @@ -1366,7 +1396,7 @@ static __always_inline void raid_recX_avx2_1234(int nr, int has_p, int *id, int asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[nr - 1][i])); } - raid_avx_end(); + raid_avx_end(0); } /* @@ -1585,7 +1615,7 @@ static __always_inline void raid_recX_avx2(int nr, int has_p, int *id, int *ip, asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[nr - 1][i])); } - raid_avx_end(); + raid_avx_end(0); } #ifdef CONFIG_X86_64 @@ -1767,7 +1797,7 @@ static __always_inline void raid_rec1_avx2ext_q(int *id, int *ip, int nd, size_t asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[i + 32])); } - raid_avx_end(); + raid_avx_end(0); } /* @@ -2037,7 +2067,7 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int has_p, int *id, int } } - raid_avx_end(); + raid_avx_end(0); } /* @@ -2336,7 +2366,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int has_p, int *id, in } } - raid_avx_end(); + raid_avx_end(0); } /* @@ -2672,69 +2702,105 @@ static __always_inline void raid_recX_avx2ext(int nr, int has_p, int *id, int *i asm volatile ("vmovdqa %%ymm15,%0" : "=m" (pa[nr - 1][i])); } - raid_avx_end(); + raid_avx_end(0); } #endif -void raid_gen2_avx2_raid(int nd, size_t size, void **vv) +void raid_gen2_avx2_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen2_avx2_gen(nd, size, vv, 2); + if (streaming) + raid_gen2_avx2_gen(nd, size, vv, 2, 1); + else + raid_gen2_avx2_gen(nd, size, vv, 2, 0); } -void raid_gen2_avx2_aes(int nd, size_t size, void **vv) +void raid_gen2_avx2_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen2_avx2_gen(nd, size, vv, 3); + if (streaming) + raid_gen2_avx2_gen(nd, size, vv, 3, 1); + else + raid_gen2_avx2_gen(nd, size, vv, 3, 0); } #ifdef CONFIG_X86_64 -void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv) +void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen2_avx2ext_gen(nd, size, vv, 2); + if (streaming) + raid_gen2_avx2ext_gen(nd, size, vv, 2, 1); + else + raid_gen2_avx2ext_gen(nd, size, vv, 2, 0); } -void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv) +void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen2_avx2ext_gen(nd, size, vv, 3); + if (streaming) + raid_gen2_avx2ext_gen(nd, size, vv, 3, 1); + else + raid_gen2_avx2ext_gen(nd, size, vv, 3, 0); } -void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv) +void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen3_avx2ext_gen(nd, size, vv, 2); + if (streaming) + raid_gen3_avx2ext_gen(nd, size, vv, 2, 1); + else + raid_gen3_avx2ext_gen(nd, size, vv, 2, 0); } -void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv) +void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen3_avx2ext_gen(nd, size, vv, 3); + if (streaming) + raid_gen3_avx2ext_gen(nd, size, vv, 3, 1); + else + raid_gen3_avx2ext_gen(nd, size, vv, 3, 0); } -void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv) +void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen4_avx2ext_gen(nd, size, vv, 2); + if (streaming) + raid_gen4_avx2ext_gen(nd, size, vv, 2, 1); + else + raid_gen4_avx2ext_gen(nd, size, vv, 2, 0); } -void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv) +void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen4_avx2ext_gen(nd, size, vv, 3); + if (streaming) + raid_gen4_avx2ext_gen(nd, size, vv, 3, 1); + else + raid_gen4_avx2ext_gen(nd, size, vv, 3, 0); } -void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv) +void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2ext(nd, size, vv, 5, 2); + if (streaming) + raid_genX_avx2ext(nd, size, vv, 5, 2, 1); + else + raid_genX_avx2ext(nd, size, vv, 5, 2, 0); } -void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv) +void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2ext(nd, size, vv, 5, 3); + if (streaming) + raid_genX_avx2ext(nd, size, vv, 5, 3, 1); + else + raid_genX_avx2ext(nd, size, vv, 5, 3, 0); } -void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv) +void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2ext(nd, size, vv, 6, 2); + if (streaming) + raid_genX_avx2ext(nd, size, vv, 6, 2, 1); + else + raid_genX_avx2ext(nd, size, vv, 6, 2, 0); } -void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv) +void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2ext(nd, size, vv, 6, 3); + if (streaming) + raid_genX_avx2ext(nd, size, vv, 6, 3, 1); + else + raid_genX_avx2ext(nd, size, vv, 6, 3, 0); } #endif @@ -2745,7 +2811,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx2(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } @@ -2810,7 +2876,7 @@ void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx2(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } diff --git a/raid/avx512.c b/raid/avx512.c index 3202c671..f7934776 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -15,7 +15,7 @@ * specific support for AVX512F because this would be the only function * to benefit from that. */ -void raid_gen1_avx512bw(int nd, size_t size, void **vv) +static __always_inline void raid_gen1_avx512bw_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -43,10 +43,21 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (v[l][i])); } - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + if (streaming) + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + else + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (p[i])); } - raid_avx_end(); + raid_avx_end(streaming); +} + +void raid_gen1_avx512bw(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen1_avx512bw_gen(nd, size, vv, 1); + else + raid_gen1_avx512bw_gen(nd, size, vv, 0); } /* @@ -55,7 +66,7 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) * Preloads 13 pairs of Q coefficient tables in ZMM registers (zmm4..zmm29) * with an unrolled disk loop and early exits. */ -void raid_gen2_avx512bw(int nd, size_t size, void **vv) +static __always_inline void raid_gen2_avx512bw_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -271,11 +282,24 @@ void raid_gen2_avx512bw(int nd, size_t size, void **vv) } store: - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); +} + +void raid_gen2_avx512bw(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen2_avx512bw_gen(nd, size, vv, 1); + else + raid_gen2_avx512bw_gen(nd, size, vv, 0); } /* @@ -284,7 +308,7 @@ store: * Preloads 6 sets of Q and R coefficient tables in ZMM registers (zmm7..zmm30) * with an unrolled disk loop and early exits. */ -void raid_gen3_avx512bw(int nd, size_t size, void **vv) +static __always_inline void raid_gen3_avx512bw_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -444,18 +468,32 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv) } store: - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (v[nd + 2][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); +} + +void raid_gen3_avx512bw(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen3_avx512bw_gen(nd, size, vv, 1); + else + raid_gen3_avx512bw_gen(nd, size, vv, 0); } /* * Generate N parity blocks with Cauchy matrix using AVX512BW implementation. */ -static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, int np) +static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, int np, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -542,19 +580,32 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i asm volatile ("vpternlogq $0x96,%zmm20,%zmm21,%zmm5"); } - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); - if (np >= 3) - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); - if (np >= 4) - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); - if (np >= 5) - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); - if (np >= 6) - asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovdqa64 %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovdqa64 %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovdqa64 %%zmm5,%0" : "=m" (v[nd + 5][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -886,78 +937,31 @@ static __always_inline void raid_recX_avx512bw(int nr, int has_p, int *id, int * asm volatile ("vmovdqa64 %%zmm30,%0" : "=m" (pa[nr - 1][i])); } - raid_avx_end(); + raid_avx_end(0); } -/* - * Recover one data failure using P with AVX512BW. - * - * The missing block is: - * - * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn - * - * Process two surviving data blocks per iteration using vpternlogq. - */ -static __always_inline void raid_rec1of1_avx512bw(int *id, int nd, size_t size, void **vv) +void raid_gen4_avx512bw(int nd, size_t size, void **vv, int streaming) { - uint8_t **v = (uint8_t **)vv; - uint8_t *src[RAID_DATA_MAX]; - uint8_t *p; - uint8_t *pa; - size_t i; - int d, s, ns; - - p = v[nd]; - pa = v[id[0]]; - - /* build the compact list of surviving data blocks */ - ns = 0; - for (d = 0; d < nd; ++d) { - if (d != id[0]) - src[ns++] = v[d]; - } - - BUG_ON(ns != nd - 1); - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - /* start from the stored P block */ - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - - /* XOR two surviving data blocks at a time */ - for (s = 0; s <= ns - 2; s += 2) { - asm volatile ( - "vmovdqa64 %0,%%zmm1\n\t" - "vpternlogq $0x96,%1,%%zmm1,%%zmm0" - : - : "m" (src[s][i]), "m" (src[s + 1][i]) - ); - } - - /* remaining odd survivor */ - if (s < ns) - asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (src[s][i])); - - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - } - - raid_avx_end(); + if (streaming) + raid_genX_avx512bw(nd, size, vv, 4, 1); + else + raid_genX_avx512bw(nd, size, vv, 4, 0); } -void raid_gen4_avx512bw(int nd, size_t size, void **vv) +void raid_gen5_avx512bw(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512bw(nd, size, vv, 4); + if (streaming) + raid_genX_avx512bw(nd, size, vv, 5, 1); + else + raid_genX_avx512bw(nd, size, vv, 5, 0); } -void raid_gen5_avx512bw(int nd, size_t size, void **vv) +void raid_gen6_avx512bw(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512bw(nd, size, vv, 5); -} - -void raid_gen6_avx512bw(int nd, size_t size, void **vv) -{ - raid_genX_avx512bw(nd, size, vv, 6); + if (streaming) + raid_genX_avx512bw(nd, size, vv, 6, 1); + else + raid_genX_avx512bw(nd, size, vv, 6, 0); } void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) @@ -966,7 +970,7 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx512bw(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } diff --git a/raid/gfni.c b/raid/gfni.c index 38d91ba8..4c5f9237 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -9,7 +9,7 @@ /* * Generate N parity blocks for the RAID polynomial using direct Cauchy coefficients with AVX2 GFNI. */ -static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np) +static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -92,29 +92,52 @@ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void ** } } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); - if (np >= 3) { - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); - } - if (np >= 4) { - asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); - asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); - } - if (np >= 5) { - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); - } - if (np >= 6) { - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); - asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (np >= 3) { + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + if (np >= 4) { + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + if (np >= 5) { + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + if (np >= 6) { + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (np >= 3) { + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + if (np >= 4) { + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovdqa %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + if (np >= 5) { + asm volatile ("vmovdqa %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovdqa %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + if (np >= 6) { + asm volatile ("vmovdqa %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovdqa %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -123,7 +146,7 @@ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void ** * Preloads up to 28 Q affine matrix coefficients in ZMM registers (zmm4..zmm31) * with an unrolled disk loop and early exits. */ -static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, void **vv) +static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -382,11 +405,16 @@ static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, v } store: - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -395,7 +423,7 @@ store: * Preloads up to 13 pairs of Q and R affine matrix coefficients in ZMM * registers (zmm6..zmm31) with an unrolled disk loop and early exits. */ -static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, void **vv) +static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -581,18 +609,24 @@ static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, v } store: - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (v[nd + 2][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* * Generate N parity blocks for the RAID polynomial using direct Cauchy coefficients with AVX512 GFNI. */ -static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void **vv, int np) +static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void **vv, int np, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -657,25 +691,38 @@ static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); } - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); - if (np >= 3) - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); - if (np >= 4) - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); - if (np >= 5) - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); - if (np >= 6) - asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovdqa64 %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovdqa64 %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovdqa64 %%zmm5,%0" : "=m" (v[nd + 5][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* * Generate N parity blocks for the AES polynomial using direct Cauchy coefficients with AVX2 GFNI. */ -static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **vv, int np) +static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **vv, int np, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -761,33 +808,60 @@ static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **v } } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); - if (np >= 3) { - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); - } + if (np >= 3) { + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } - if (np >= 4) { - asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); - asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); - } + if (np >= 4) { + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } - if (np >= 5) { - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); - } + if (np >= 5) { + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } - if (np >= 6) { - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); - asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + if (np >= 6) { + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } + } else { + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + + if (np >= 3) { + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + + if (np >= 4) { + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovdqa %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + + if (np >= 5) { + asm volatile ("vmovdqa %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovdqa %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + + if (np >= 6) { + asm volatile ("vmovdqa %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovdqa %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -796,7 +870,7 @@ static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **v * Preloads up to 28 Q multiplier coefficients in ZMM registers (zmm4..zmm31) * with an unrolled disk loop and early exits. */ -static __always_inline void raid_gen2_avx512gfni_aes_gen(int nd, size_t size, void **vv) +static __always_inline void raid_gen2_avx512gfni_aes_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -1055,11 +1129,16 @@ static __always_inline void raid_gen2_avx512gfni_aes_gen(int nd, size_t size, vo } store: - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* @@ -1068,7 +1147,7 @@ store: * Preloads up to 13 pairs of Q and R multiplier coefficients in ZMM * registers (zmm6..zmm31) with an unrolled disk loop and early exits. */ -static __always_inline void raid_gen3_avx512gfni_aes_gen(int nd, size_t size, void **vv) +static __always_inline void raid_gen3_avx512gfni_aes_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -1254,18 +1333,24 @@ static __always_inline void raid_gen3_avx512gfni_aes_gen(int nd, size_t size, vo } store: - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (v[nd + 2][i])); + } } - raid_avx_end(); + raid_avx_end(streaming); } /* * Generate N parity blocks for the AES polynomial using direct Cauchy coefficients with AVX512 GFNI. */ -static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void **vv, int np) +static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void **vv, int np, int streaming) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -1330,123 +1415,34 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void * asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); } - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); - if (np >= 3) - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); - if (np >= 4) - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); - if (np >= 5) - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); - if (np >= 6) - asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); - } - - raid_avx_end(); -} - -/* - * Recover one data failure using P with AVX2 GFNI. - * - * The missing block is: - * - * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn - */ -static __always_inline void raid_rec1of1_avx2gfni(int *id, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *src[RAID_DATA_MAX]; - uint8_t *p; - uint8_t *pa; - size_t i; - int d, s, ns; - - p = v[nd]; - pa = v[id[0]]; - - /* build the compact list of surviving data blocks */ - ns = 0; - for (d = 0; d < nd; ++d) { - if (d != id[0]) - src[ns++] = v[d]; - } - - BUG_ON(ns != nd - 1); - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - /* start from the stored P block */ - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); - - for (s = 0; s < ns; ++s) { - asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (src[s][i])); - asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (src[s][i + 32])); + if (streaming) { + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } else { + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovdqa64 %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovdqa64 %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovdqa64 %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovdqa64 %%zmm5,%0" : "=m" (v[nd + 5][i])); } - - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); } - raid_avx_end(); + raid_avx_end(streaming); } -/* - * Recover one data failure using P with AVX512 GFNI. - * - * The missing block is: - * - * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn - * - * Process two surviving data blocks per iteration using vpternlogq. - */ -static __always_inline void raid_rec1of1_avx512gfni(int *id, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *src[RAID_DATA_MAX]; - uint8_t *p; - uint8_t *pa; - size_t i; - int d, s, ns; - - p = v[nd]; - pa = v[id[0]]; - - /* build the compact list of surviving data blocks */ - ns = 0; - for (d = 0; d < nd; ++d) { - if (d != id[0]) - src[ns++] = v[d]; - } - - BUG_ON(ns != nd - 1); - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - /* start from the stored P block */ - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - - /* XOR two surviving data blocks at a time */ - for (s = 0; s <= ns - 2; s += 2) { - asm volatile ( - "vmovdqa64 %0,%%zmm1\n\t" - "vpternlogq $0x96,%1,%%zmm1,%%zmm0" - : - : "m" (src[s][i]), "m" (src[s + 1][i]) - ); - } - - /* remaining odd survivor */ - if (s < ns) - asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (src[s][i])); - - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - } - - raid_avx_end(); -} /* * Recover multiple data failures using selected parity blocks with AVX2 GFNI. @@ -1660,7 +1656,7 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id, } } - raid_avx_end(); + raid_avx_end(0); } /* @@ -1838,7 +1834,7 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int has_p, int *id asm volatile ("vmovdqa64 %%zmm30,%0" : "=m" (pa[nr - 1][i])); } - raid_avx_end(); + raid_avx_end(0); } /* @@ -2043,7 +2039,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, i } } - raid_avx_end(); + raid_avx_end(0); } /* @@ -2207,107 +2203,167 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int has_p, int *id, asm volatile ("vmovdqa64 %%zmm30,%0" : "=m" (pa[nr - 1][i])); } - raid_avx_end(); + raid_avx_end(0); } -void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv) +void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_raid(nd, size, vv, 2); + if (streaming) + raid_genX_avx2gfni_raid(nd, size, vv, 2, 1); + else + raid_genX_avx2gfni_raid(nd, size, vv, 2, 0); } -void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv) +void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen2_avx512gfni_raid_gen(nd, size, vv); + if (streaming) + raid_gen2_avx512gfni_raid_gen(nd, size, vv, 1); + else + raid_gen2_avx512gfni_raid_gen(nd, size, vv, 0); } -void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv) +void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_raid(nd, size, vv, 3); + if (streaming) + raid_genX_avx2gfni_raid(nd, size, vv, 3, 1); + else + raid_genX_avx2gfni_raid(nd, size, vv, 3, 0); } -void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv) +void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen3_avx512gfni_raid_gen(nd, size, vv); + if (streaming) + raid_gen3_avx512gfni_raid_gen(nd, size, vv, 1); + else + raid_gen3_avx512gfni_raid_gen(nd, size, vv, 0); } -void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv) +void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_raid(nd, size, vv, 4); + if (streaming) + raid_genX_avx2gfni_raid(nd, size, vv, 4, 1); + else + raid_genX_avx2gfni_raid(nd, size, vv, 4, 0); } -void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv) +void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512gfni_raid(nd, size, vv, 4); + if (streaming) + raid_genX_avx512gfni_raid(nd, size, vv, 4, 1); + else + raid_genX_avx512gfni_raid(nd, size, vv, 4, 0); } -void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv) +void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_raid(nd, size, vv, 5); + if (streaming) + raid_genX_avx2gfni_raid(nd, size, vv, 5, 1); + else + raid_genX_avx2gfni_raid(nd, size, vv, 5, 0); } -void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv) +void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512gfni_raid(nd, size, vv, 5); + if (streaming) + raid_genX_avx512gfni_raid(nd, size, vv, 5, 1); + else + raid_genX_avx512gfni_raid(nd, size, vv, 5, 0); } -void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv) +void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_raid(nd, size, vv, 6); + if (streaming) + raid_genX_avx2gfni_raid(nd, size, vv, 6, 1); + else + raid_genX_avx2gfni_raid(nd, size, vv, 6, 0); } -void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv) +void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512gfni_raid(nd, size, vv, 6); + if (streaming) + raid_genX_avx512gfni_raid(nd, size, vv, 6, 1); + else + raid_genX_avx512gfni_raid(nd, size, vv, 6, 0); } -void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv) +void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_aes(nd, size, vv, 2); + if (streaming) + raid_genX_avx2gfni_aes(nd, size, vv, 2, 1); + else + raid_genX_avx2gfni_aes(nd, size, vv, 2, 0); } -void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv) +void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen2_avx512gfni_aes_gen(nd, size, vv); + if (streaming) + raid_gen2_avx512gfni_aes_gen(nd, size, vv, 1); + else + raid_gen2_avx512gfni_aes_gen(nd, size, vv, 0); } -void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv) +void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_aes(nd, size, vv, 3); + if (streaming) + raid_genX_avx2gfni_aes(nd, size, vv, 3, 1); + else + raid_genX_avx2gfni_aes(nd, size, vv, 3, 0); } -void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv) +void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen3_avx512gfni_aes_gen(nd, size, vv); + if (streaming) + raid_gen3_avx512gfni_aes_gen(nd, size, vv, 1); + else + raid_gen3_avx512gfni_aes_gen(nd, size, vv, 0); } -void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv) +void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_aes(nd, size, vv, 4); + if (streaming) + raid_genX_avx2gfni_aes(nd, size, vv, 4, 1); + else + raid_genX_avx2gfni_aes(nd, size, vv, 4, 0); } -void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv) +void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512gfni_aes(nd, size, vv, 4); + if (streaming) + raid_genX_avx512gfni_aes(nd, size, vv, 4, 1); + else + raid_genX_avx512gfni_aes(nd, size, vv, 4, 0); } -void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv) +void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_aes(nd, size, vv, 5); + if (streaming) + raid_genX_avx2gfni_aes(nd, size, vv, 5, 1); + else + raid_genX_avx2gfni_aes(nd, size, vv, 5, 0); } -void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv) +void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512gfni_aes(nd, size, vv, 5); + if (streaming) + raid_genX_avx512gfni_aes(nd, size, vv, 5, 1); + else + raid_genX_avx512gfni_aes(nd, size, vv, 5, 0); } -void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv) +void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx2gfni_aes(nd, size, vv, 6); + if (streaming) + raid_genX_avx2gfni_aes(nd, size, vv, 6, 1); + else + raid_genX_avx2gfni_aes(nd, size, vv, 6, 0); } -void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) +void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_avx512gfni_aes(nd, size, vv, 6); + if (streaming) + raid_genX_avx512gfni_aes(nd, size, vv, 6, 1); + else + raid_genX_avx512gfni_aes(nd, size, vv, 6, 0); } void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) @@ -2316,7 +2372,7 @@ void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx2gfni(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } @@ -2374,7 +2430,7 @@ void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx512gfni(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } @@ -2432,7 +2488,7 @@ void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx2gfni(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } @@ -2490,7 +2546,7 @@ void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_avx512gfni(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } diff --git a/raid/int.c b/raid/int.c index 018bd429..36b06c54 100644 --- a/raid/int.c +++ b/raid/int.c @@ -9,7 +9,7 @@ * * Uses 16-byte chunks across four 32-bit words. */ -void raid_gen1_int32(int nd, size_t size, void **vv) +void raid_gen1_int32(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -21,6 +21,8 @@ void raid_gen1_int32(int nd, size_t size, void **vv) uint32_t p2; uint32_t p3; + (void)streaming; + l = nd - 1; p = v[nd]; @@ -47,7 +49,7 @@ void raid_gen1_int32(int nd, size_t size, void **vv) * * Uses 32-byte chunks across four 64-bit words. */ -void raid_gen1_int64(int nd, size_t size, void **vv) +void raid_gen1_int64(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -59,6 +61,8 @@ void raid_gen1_int64(int nd, size_t size, void **vv) uint64_t p2; uint64_t p3; + (void)streaming; + l = nd - 1; p = v[nd]; @@ -189,7 +193,7 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, * * Uses Horner's method with 8-byte chunks across two 32-bit words. */ -void raid_genz_int32_raid(int nd, size_t size, void **vv) +void raid_genz_int32_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -203,6 +207,8 @@ void raid_genz_int32_raid(int nd, size_t size, void **vv) uint32_t d0, r0, q0, p0; uint32_t d1, r1, q1, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -244,7 +250,7 @@ void raid_genz_int32_raid(int nd, size_t size, void **vv) * * Uses Horner's method with 16-byte chunks across two 64-bit words. */ -void raid_genz_int64_raid(int nd, size_t size, void **vv) +void raid_genz_int64_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -258,6 +264,8 @@ void raid_genz_int64_raid(int nd, size_t size, void **vv) uint64_t d0, r0, q0, p0; uint64_t d1, r1, q1, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -299,7 +307,7 @@ void raid_genz_int64_raid(int nd, size_t size, void **vv) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -void raid_gen2_int8(int nd, size_t size, void **vv) +void raid_gen2_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -310,6 +318,8 @@ void raid_gen2_int8(int nd, size_t size, void **vv) uint8_t d0, d1, q0, q1, p0, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -351,7 +361,7 @@ void raid_gen2_int8(int nd, size_t size, void **vv) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -void raid_gen3_int8(int nd, size_t size, void **vv) +void raid_gen3_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -363,6 +373,8 @@ void raid_gen3_int8(int nd, size_t size, void **vv) uint8_t d0, d1, r0, r1, q0, q1, p0, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -413,7 +425,7 @@ void raid_gen3_int8(int nd, size_t size, void **vv) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -void raid_gen4_int8(int nd, size_t size, void **vv) +void raid_gen4_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -426,6 +438,8 @@ void raid_gen4_int8(int nd, size_t size, void **vv) uint8_t d0, d1, s0, s1, r0, r1, q0, q1, p0, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -484,7 +498,7 @@ void raid_gen4_int8(int nd, size_t size, void **vv) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -void raid_gen5_int8(int nd, size_t size, void **vv) +void raid_gen5_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -498,6 +512,8 @@ void raid_gen5_int8(int nd, size_t size, void **vv) uint8_t d0, d1, t0, t1, s0, s1, r0, r1, q0, q1, p0, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -564,7 +580,7 @@ void raid_gen5_int8(int nd, size_t size, void **vv) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -void raid_gen6_int8(int nd, size_t size, void **vv) +void raid_gen6_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -579,6 +595,8 @@ void raid_gen6_int8(int nd, size_t size, void **vv) uint8_t d0, d1, u0, u1, t0, t1, s0, s1, r0, r1, q0, q1, p0, p1; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -1217,23 +1235,27 @@ static __always_inline void raid_recX_int8(int nr, int has_p, int *id, int *ip, } } -void raid_gen2_int32_raid(int nd, size_t size, void **vv) +void raid_gen2_int32_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_int32_gen(nd, size, vv, 2); } -void raid_gen2_int32_aes(int nd, size_t size, void **vv) +void raid_gen2_int32_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_int32_gen(nd, size, vv, 3); } -void raid_gen2_int64_raid(int nd, size_t size, void **vv) +void raid_gen2_int64_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_int64_gen(nd, size, vv, 2); } -void raid_gen2_int64_aes(int nd, size_t size, void **vv) +void raid_gen2_int64_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_int64_gen(nd, size, vv, 3); } diff --git a/raid/internal.h b/raid/internal.h index 1558aa92..12ff9b0a 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -91,80 +91,80 @@ void raid_gen_ref(int nd, int np, size_t size, void **vv); void raid_invert(uint8_t *M, uint8_t *V, int n); void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v); void raid_rec1of1(int *id, int nd, size_t size, void **v); -void raid_gen1_int32(int nd, size_t size, void **vv); -void raid_gen1_int64(int nd, size_t size, void **vv); -void raid_gen1_sse2(int nd, size_t size, void **vv); -void raid_gen1_avx2(int nd, size_t size, void **vv); -void raid_gen1_avx512bw(int nd, size_t size, void **vv); -void raid_gen2_int8(int nd, size_t size, void **vv); -void raid_gen2_int32_raid(int nd, size_t size, void **vv); -void raid_gen2_int32_aes(int nd, size_t size, void **vv); -void raid_gen2_int64_raid(int nd, size_t size, void **vv); -void raid_gen2_int64_aes(int nd, size_t size, void **vv); -void raid_gen2_sse2_raid(int nd, size_t size, void **vv); -void raid_gen2_sse2_aes(int nd, size_t size, void **vv); -void raid_gen2_avx2_raid(int nd, size_t size, void **vv); -void raid_gen2_avx2_aes(int nd, size_t size, void **vv); -void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv); -void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv); -void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv); -void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv); -void raid_gen2_avx512bw(int nd, size_t size, void **vv); -void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv); -void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv); -void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv); -void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv); -void raid_genz_int32_raid(int nd, size_t size, void **vv); -void raid_genz_int64_raid(int nd, size_t size, void **vv); -void raid_genz_sse2_raid(int nd, size_t size, void **vv); -void raid_genz_sse2ext_raid(int nd, size_t size, void **vv); -void raid_genz_avx2ext_raid(int nd, size_t size, void **vv); -void raid_gen3_int8(int nd, size_t size, void **vv); -void raid_gen3_ssse3_raid(int nd, size_t size, void **vv); -void raid_gen3_ssse3_aes(int nd, size_t size, void **vv); -void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv); -void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv); -void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv); -void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv); -void raid_gen3_avx512bw(int nd, size_t size, void **vv); -void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv); -void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv); -void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv); -void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv); -void raid_gen4_int8(int nd, size_t size, void **vv); -void raid_gen4_ssse3_raid(int nd, size_t size, void **vv); -void raid_gen4_ssse3_aes(int nd, size_t size, void **vv); -void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv); -void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv); -void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv); -void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv); -void raid_gen4_avx512bw(int nd, size_t size, void **vv); -void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv); -void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv); -void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv); -void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv); -void raid_gen5_int8(int nd, size_t size, void **vv); -void raid_gen5_ssse3_raid(int nd, size_t size, void **vv); -void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv); -void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv); -void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv); -void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv); -void raid_gen5_avx512bw(int nd, size_t size, void **vv); -void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv); -void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv); -void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv); -void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv); -void raid_gen6_int8(int nd, size_t size, void **vv); -void raid_gen6_ssse3_raid(int nd, size_t size, void **vv); -void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv); -void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv); -void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv); -void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv); -void raid_gen6_avx512bw(int nd, size_t size, void **vv); -void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv); -void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv); -void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv); -void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv); +void raid_gen1_int32(int nd, size_t size, void **vv, int streaming); +void raid_gen1_int64(int nd, size_t size, void **vv, int streaming); +void raid_gen1_sse2(int nd, size_t size, void **vv, int streaming); +void raid_gen1_avx2(int nd, size_t size, void **vv, int streaming); +void raid_gen1_avx512bw(int nd, size_t size, void **vv, int streaming); +void raid_gen2_int8(int nd, size_t size, void **vv, int streaming); +void raid_gen2_int32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_int32_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_int64_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_int64_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_sse2_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_sse2_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx2_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx2_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx512bw(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_genz_int32_raid(int nd, size_t size, void **vv, int streaming); +void raid_genz_int64_raid(int nd, size_t size, void **vv, int streaming); +void raid_genz_sse2_raid(int nd, size_t size, void **vv, int streaming); +void raid_genz_sse2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_genz_avx2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_int8(int nd, size_t size, void **vv, int streaming); +void raid_gen3_ssse3_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_ssse3_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx512bw(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_int8(int nd, size_t size, void **vv, int streaming); +void raid_gen4_ssse3_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_ssse3_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx512bw(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen5_int8(int nd, size_t size, void **vv, int streaming); +void raid_gen5_ssse3_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx512bw(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen6_int8(int nd, size_t size, void **vv, int streaming); +void raid_gen6_ssse3_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx512bw(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv, int streaming); void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -225,36 +225,36 @@ void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_gen1_neon(int nd, size_t size, void **vv); -void raid_gen2_neon_raid(int nd, size_t size, void **vv); -void raid_gen2_neon_aes(int nd, size_t size, void **vv); -void raid_genz_neon_raid(int nd, size_t size, void **vv); -void raid_gen3_neon_raid(int nd, size_t size, void **vv); -void raid_gen3_neon_aes(int nd, size_t size, void **vv); -void raid_gen4_neon_raid(int nd, size_t size, void **vv); -void raid_gen4_neon_aes(int nd, size_t size, void **vv); -void raid_gen5_neon_raid(int nd, size_t size, void **vv); -void raid_gen5_neon_aes(int nd, size_t size, void **vv); -void raid_gen6_neon_raid(int nd, size_t size, void **vv); -void raid_gen6_neon_aes(int nd, size_t size, void **vv); +void raid_gen1_neon(int nd, size_t size, void **vv, int streaming); +void raid_gen2_neon_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_neon_aes(int nd, size_t size, void **vv, int streaming); +void raid_genz_neon_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_neon_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_neon_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_neon_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_neon_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen5_neon_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_neon_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen6_neon_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_neon_aes(int nd, size_t size, void **vv, int streaming); void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_gen1_neon32(int nd, size_t size, void **vv); -void raid_gen2_neon32_raid(int nd, size_t size, void **vv); -void raid_gen2_neon32_aes(int nd, size_t size, void **vv); -void raid_genz_neon32_raid(int nd, size_t size, void **vv); -void raid_gen3_neon32_raid(int nd, size_t size, void **vv); -void raid_gen3_neon32_aes(int nd, size_t size, void **vv); -void raid_gen4_neon32_raid(int nd, size_t size, void **vv); -void raid_gen4_neon32_aes(int nd, size_t size, void **vv); -void raid_gen5_neon32_raid(int nd, size_t size, void **vv); -void raid_gen5_neon32_aes(int nd, size_t size, void **vv); -void raid_gen6_neon32_raid(int nd, size_t size, void **vv); -void raid_gen6_neon32_aes(int nd, size_t size, void **vv); +void raid_gen1_neon32(int nd, size_t size, void **vv, int streaming); +void raid_gen2_neon32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen2_neon32_aes(int nd, size_t size, void **vv, int streaming); +void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_neon32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen3_neon32_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen4_neon32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen4_neon32_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen5_neon32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen5_neon32_aes(int nd, size_t size, void **vv, int streaming); +void raid_gen6_neon32_raid(int nd, size_t size, void **vv, int streaming); +void raid_gen6_neon32_aes(int nd, size_t size, void **vv, int streaming); void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -283,8 +283,9 @@ void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); * It has (@nd + #parities) elements. The starting elements are the blocks * for data, following with the parity blocks. * Each block has @size bytes. + * @streaming Nonzero for streaming/non-temporal stores where supported, 0 for cached stores. */ -typedef void (raid_gen_fn)(int nd, size_t size, void **vv); +typedef void (raid_gen_fn)(int nd, size_t size, void **vv, int streaming); /* * Functions for data recovery. @@ -444,15 +445,16 @@ static __always_inline void raid_sse_begin(void) { } -static __always_inline void raid_sse_end(void) +static __always_inline void raid_sse_end(int streaming) { /* - * SSE and AVX code uses non-temporal writes, like MOVNTDQ, - * that use a weak memory model. To ensure that other processors - * see correctly the data written, we use a store-store memory - * barrier at the end of the asm code + * Non-temporal streaming stores, like MOVNTDQ, use a weak memory model. + * To ensure that other processors correctly observe data written with + * streaming stores, execute a store-store memory barrier. Cached stores + * maintain normal cache-coherent ordering and do not require sfence. */ - asm volatile ("sfence" : : : "memory"); + if (streaming) + asm volatile ("sfence" : : : "memory"); /* * Clobbers registers used in the asm code. @@ -495,9 +497,9 @@ static __always_inline void raid_avx_begin(void) raid_sse_begin(); } -static __always_inline void raid_avx_end(void) +static __always_inline void raid_avx_end(int streaming) { - raid_sse_end(); + raid_sse_end(streaming); /* * Clear the upper parts of the vector registers before returning. diff --git a/raid/module.c b/raid/module.c index f1759055..23af6320 100644 --- a/raid/module.c +++ b/raid/module.c @@ -73,7 +73,7 @@ struct gfconst16 __aligned(16) gfconst16 = }; #endif -void raid_gen(int nd, int np, size_t size, void **v) +void raid_gen(int nd, int np, size_t size, void **v, int streaming) { /* enforce limit on size */ BUG_ON(size % 64 != 0); @@ -88,8 +88,11 @@ void raid_gen(int nd, int np, size_t size, void **v) /* enforce valid vector */ BUG_ON(v == 0); + /* enforce valid streaming flag */ + BUG_ON(streaming != 0 && streaming != 1); + BUG_ON(raid_gen_ptr[np - 1] == 0); - raid_gen_ptr[np - 1](nd, size, v); + raid_gen_ptr[np - 1](nd, size, v, streaming); } /** @@ -199,7 +202,7 @@ void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v) } } - raid_gen(nd, max_p, size, v_gen); + raid_gen(nd, max_p, size, v_gen, 0); } } @@ -468,27 +471,30 @@ void raid_gen_ref(int nd, int np, size_t size, void **vv) */ static int selftest_par(int nd, int np, size_t size, void **v, void **ref) { - int i; + int i, streaming; void *t[TEST_COUNT + RAID_PARITY_MAX]; /* setup data */ for (i = 0; i < nd; ++i) t[i] = ref[i]; - /* setup parity */ - for (i = 0; i < np; ++i) { - memset(v[nd + i], 0x55, size); - t[nd + i] = v[nd + i]; - } + /* test both cached and streaming generation */ + for (streaming = 0; streaming <= 1; ++streaming) { + /* setup parity */ + for (i = 0; i < np; ++i) { + memset(v[nd + i], 0x55, size); + t[nd + i] = v[nd + i]; + } - raid_gen(nd, np, size, t); + raid_gen(nd, np, size, t, streaming); - /* compare parity */ - for (i = 0; i < np; ++i) { - if (memcmp(t[nd + i], ref[nd + i], size) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ + /* compare parity */ + for (i = 0; i < np; ++i) { + if (memcmp(t[nd + i], ref[nd + i], size) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } } } diff --git a/raid/neon.c b/raid/neon.c index 1f58ed0d..033cdcea 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -11,18 +11,21 @@ * * Uses 64-byte chunks across four 16-byte vectors. */ -void raid_gen1_neon(int nd, size_t size, void **vv) +void raid_gen1_neon(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; int d, l; size_t i; + (void)streaming; + l = nd - 1; p = v[nd]; raid_neon_begin(); + for (i = 0; i < size; i += 64) { asm volatile ( "ldr q0, %0\n" @@ -147,7 +150,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i /* * Generate three parity blocks with powers of 2^-1 using NEON implementation. */ -void raid_genz_neon_raid(int nd, size_t size, void **vv) +void raid_genz_neon_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -156,6 +159,8 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv) int d, l; size_t i; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -1324,53 +1329,63 @@ static __always_inline void raid_recX_neon(int nr, int has_p, int *id, int *ip, raid_neon_end(); } -void raid_gen2_neon_raid(int nd, size_t size, void **vv) +void raid_gen2_neon_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_neon_gen(nd, size, vv, 2); } -void raid_gen2_neon_aes(int nd, size_t size, void **vv) +void raid_gen2_neon_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_neon_gen(nd, size, vv, 3); } -void raid_gen3_neon_raid(int nd, size_t size, void **vv) +void raid_gen3_neon_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 3, 2); } -void raid_gen3_neon_aes(int nd, size_t size, void **vv) +void raid_gen3_neon_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 3, 3); } -void raid_gen4_neon_raid(int nd, size_t size, void **vv) +void raid_gen4_neon_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 4, 2); } -void raid_gen4_neon_aes(int nd, size_t size, void **vv) +void raid_gen4_neon_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 4, 3); } -void raid_gen5_neon_raid(int nd, size_t size, void **vv) +void raid_gen5_neon_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 5, 2); } -void raid_gen5_neon_aes(int nd, size_t size, void **vv) +void raid_gen5_neon_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 5, 3); } -void raid_gen6_neon_raid(int nd, size_t size, void **vv) +void raid_gen6_neon_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 6, 2); } -void raid_gen6_neon_aes(int nd, size_t size, void **vv) +void raid_gen6_neon_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon(nd, size, vv, 6, 3); } diff --git a/raid/neon32.c b/raid/neon32.c index 98257ba3..980bbe81 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -11,18 +11,21 @@ * * Uses 64-byte chunks across four 16-byte vectors. */ -void raid_gen1_neon32(int nd, size_t size, void **vv) +void raid_gen1_neon32(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; int d, l; size_t i; + (void)streaming; + l = nd - 1; p = v[nd]; raid_neon32_begin(); + for (i = 0; i < size; i += 64) { asm volatile ( "vld1.8 {q0}, %0\n" @@ -147,7 +150,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, /* * Generate three parity blocks with powers of 2^-1 using AArch32 NEON implementation. */ -void raid_genz_neon32_raid(int nd, size_t size, void **vv) +void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -156,6 +159,8 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv) int d, l; size_t i; + (void)streaming; + l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -1163,53 +1168,63 @@ static __always_inline void raid_recX_neon32(int nr, int has_p, int *id, int *ip raid_neon32_end(); } -void raid_gen2_neon32_raid(int nd, size_t size, void **vv) +void raid_gen2_neon32_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_neon32_gen(nd, size, vv, 2); } -void raid_gen2_neon32_aes(int nd, size_t size, void **vv) +void raid_gen2_neon32_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_gen2_neon32_gen(nd, size, vv, 3); } -void raid_gen3_neon32_raid(int nd, size_t size, void **vv) +void raid_gen3_neon32_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 3, 2); } -void raid_gen3_neon32_aes(int nd, size_t size, void **vv) +void raid_gen3_neon32_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 3, 3); } -void raid_gen4_neon32_raid(int nd, size_t size, void **vv) +void raid_gen4_neon32_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 4, 2); } -void raid_gen4_neon32_aes(int nd, size_t size, void **vv) +void raid_gen4_neon32_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 4, 3); } -void raid_gen5_neon32_raid(int nd, size_t size, void **vv) +void raid_gen5_neon32_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 5, 2); } -void raid_gen5_neon32_aes(int nd, size_t size, void **vv) +void raid_gen5_neon32_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 5, 3); } -void raid_gen6_neon32_raid(int nd, size_t size, void **vv) +void raid_gen6_neon32_raid(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 6, 2); } -void raid_gen6_neon32_aes(int nd, size_t size, void **vv) +void raid_gen6_neon32_aes(int nd, size_t size, void **vv, int streaming) { + (void)streaming; raid_genX_neon32(nd, size, vv, 6, 3); } diff --git a/raid/raid.c b/raid/raid.c index 0e2b1cc7..827fc516 100644 --- a/raid/raid.c +++ b/raid/raid.c @@ -366,7 +366,7 @@ void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v) * Recompute the parity, note that np may be smaller than the * total number of parities available */ - raid_gen(nd, np, size, v); + raid_gen(nd, np, size, v, 0); /* restore data buffers as before */ for (j = 0; j < nr; ++j) @@ -407,7 +407,7 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v) v[nd] = pa; /* compute */ - raid_gen(nd, 1, size, v); + raid_gen(nd, 1, size, v, 0); /* restore as before */ v[id[0]] = pa; diff --git a/raid/raid.h b/raid/raid.h index 459adeba..36534b65 100644 --- a/raid/raid.h +++ b/raid/raid.h @@ -222,9 +222,12 @@ void raid_zero(void *zero); * data, following with the parity blocks. * Data entries are used as inputs and parity entries as outputs. * Entries may point to the same block buffer. - * Each block has @size bytes and must be aligned to a 64-byte boundary. + * @streaming Store policy hint: nonzero for streaming / non-temporal stores + * (bypassing cache) where supported, or 0 for normal cached / temporal stores. + * Both modes produce bit-identical parity data. Backends without streaming + * stores ignore this hint. */ -void raid_gen(int nd, int np, size_t size, void **v); +void raid_gen(int nd, int np, size_t size, void **v, int streaming); /** * Recovers failures in data and parity blocks. diff --git a/raid/sse2.c b/raid/sse2.c index 0ca2fb3e..bb5809ee 100644 --- a/raid/sse2.c +++ b/raid/sse2.c @@ -104,7 +104,7 @@ * * Uses 64-byte chunks across four 16-byte XMM vectors. */ -void raid_gen1_sse2(int nd, size_t size, void **vv) +static __always_inline void raid_gen1_sse2_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -129,19 +129,34 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) asm volatile ("pxor %0,%%xmm3" : : "m" (v[d][i + 48])); } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (p[i + 48])); + } } - raid_sse_end(); + raid_sse_end(streaming); +} + +void raid_gen1_sse2(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen1_sse2_gen(nd, size, vv, 1); + else + raid_gen1_sse2_gen(nd, size, vv, 0); } /* * Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 implementation. */ -static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -201,23 +216,36 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i asm volatile ("pxor %xmm5,%xmm3"); } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (q[i + 16])); + } } - raid_sse_end(); + raid_sse_end(streaming); } -void raid_gen2_sse2_raid(int nd, size_t size, void **vv) +void raid_gen2_sse2_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen2_sse2_gen(nd, size, vv, 2); + if (streaming) + raid_gen2_sse2_gen(nd, size, vv, 2, 1); + else + raid_gen2_sse2_gen(nd, size, vv, 2, 0); } -void raid_gen2_sse2_aes(int nd, size_t size, void **vv) +void raid_gen2_sse2_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen2_sse2_gen(nd, size, vv, 3); + if (streaming) + raid_gen2_sse2_gen(nd, size, vv, 3, 1); + else + raid_gen2_sse2_gen(nd, size, vv, 3, 0); } #ifdef CONFIG_X86_64 @@ -229,7 +257,7 @@ void raid_gen2_sse2_aes(int nd, size_t size, void **vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -438,34 +466,51 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv asm volatile ("pxor %xmm11,%xmm7"); } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); - asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32])); - asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); + asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32])); + asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (p[i + 48])); + asm volatile ("movdqa %%xmm4,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm5,%0" : "=m" (q[i + 16])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (q[i + 32])); + asm volatile ("movdqa %%xmm7,%0" : "=m" (q[i + 48])); + } } - raid_sse_end(); + raid_sse_end(streaming); } -void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv) +void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen2_sse2ext_gen(nd, size, vv, 2); + if (streaming) + raid_gen2_sse2ext_gen(nd, size, vv, 2, 1); + else + raid_gen2_sse2ext_gen(nd, size, vv, 2, 0); } -void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv) +void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen2_sse2ext_gen(nd, size, vv, 3); + if (streaming) + raid_gen2_sse2ext_gen(nd, size, vv, 3, 1); + else + raid_gen2_sse2ext_gen(nd, size, vv, 3, 0); } #endif /* * Generate three parity blocks with powers of 2^-1 using SSE2 implementation. */ -void raid_genz_sse2_raid(int nd, size_t size, void **vv) +static __always_inline void raid_genz_sse2_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -510,12 +555,26 @@ void raid_genz_sse2_raid(int nd, size_t size, void **vv) asm volatile ("pxor %xmm4,%xmm1"); asm volatile ("pxor %xmm4,%xmm2"); } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + } } - raid_sse_end(); + raid_sse_end(streaming); +} + +void raid_genz_sse2_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_genz_sse2_gen(nd, size, vv, 1); + else + raid_genz_sse2_gen(nd, size, vv, 0); } #ifdef CONFIG_X86_64 @@ -524,7 +583,7 @@ void raid_genz_sse2_raid(int nd, size_t size, void **vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_genz_sse2ext_raid(int nd, size_t size, void **vv) +static __always_inline void raid_genz_sse2ext_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -588,15 +647,32 @@ void raid_genz_sse2ext_raid(int nd, size_t size, void **vv) asm volatile ("pxor %xmm12,%xmm9"); asm volatile ("pxor %xmm12,%xmm10"); } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movdqa %%xmm10,%0" : "=m" (r[i + 16])); + } } - raid_sse_end(); + raid_sse_end(streaming); +} + +void raid_genz_sse2ext_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_genz_sse2ext_gen(nd, size, vv, 1); + else + raid_genz_sse2ext_gen(nd, size, vv, 0); } #endif diff --git a/raid/ssse3.c b/raid/ssse3.c index f792becd..ee3853c0 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -9,7 +9,7 @@ /* * Generate three parity blocks with Cauchy matrix using SSSE3 implementation. */ -static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -102,18 +102,24 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pxor %xmm4,%xmm1"); asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + } } - raid_sse_end(); + raid_sse_end(streaming); } /* * Generate four parity blocks with Cauchy matrix using SSSE3 implementation. */ -static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -224,13 +230,20 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pxor %xmm4,%xmm2"); asm volatile ("pxor %xmm4,%xmm3"); - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (s[i])); + } } - raid_sse_end(); + raid_sse_end(streaming); } /* @@ -238,7 +251,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, * * Uses an aligned stack buffer for parity accumulators under 8-register pressure. */ -void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) +static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -361,14 +374,30 @@ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) asm volatile ("pxor %xmm4,%xmm3"); asm volatile ("pxor %xmm4,%xmm6"); - asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); + if (streaming) { + asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); + } else { + asm volatile ("movdqa %%xmm6,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm0,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (r[i])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (s[i])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (t[i])); + } } - raid_sse_end(); + raid_sse_end(streaming); +} + +void raid_gen5_ssse3_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen5_ssse3_gen(nd, size, vv, 1); + else + raid_gen5_ssse3_gen(nd, size, vv, 0); } /* @@ -376,7 +405,7 @@ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) * * Uses an aligned stack buffer for parity accumulators under 8-register pressure. */ -void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) +static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -519,15 +548,32 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) asm volatile ("pxor %xmm4,%xmm5"); asm volatile ("pxor %xmm4,%xmm6"); - asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); + if (streaming) { + asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); + } else { + asm volatile ("movdqa %%xmm5,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm0,%0" : "=m" (r[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (s[i])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (t[i])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (u[i])); + } } - raid_sse_end(); + raid_sse_end(streaming); +} + +void raid_gen6_ssse3_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen6_ssse3_gen(nd, size, vv, 1); + else + raid_gen6_ssse3_gen(nd, size, vv, 0); } #ifdef CONFIG_X86_64 @@ -539,7 +585,7 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -782,15 +828,24 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm12,%xmm9"); asm volatile ("pxor %xmm12,%xmm10"); - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movdqa %%xmm10,%0" : "=m" (r[i + 16])); + } } - raid_sse_end(); + raid_sse_end(streaming); } /* @@ -801,7 +856,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int generator) +static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -971,23 +1026,34 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm12,%xmm10"); asm volatile ("pxor %xmm12,%xmm11"); - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movdqa %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movdqa %%xmm10,%0" : "=m" (r[i + 16])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (s[i])); + asm volatile ("movdqa %%xmm11,%0" : "=m" (s[i + 16])); + } } - raid_sse_end(); + raid_sse_end(streaming); } /* * Generate N parity blocks with Cauchy matrix using SSSE3 extended implementation. */ -static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator) +static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator, int streaming) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -1149,76 +1215,36 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i if (np >= 6) asm volatile ("pxor %xmm10,%xmm5"); - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - if (np >= 3) - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - if (np >= 4) - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); - if (np >= 5) - asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); - if (np >= 6) - asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); + if (streaming) { + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + if (np >= 3) + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + if (np >= 4) + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + if (np >= 5) + asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); + if (np >= 6) + asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); + } else { + asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i])); + if (np >= 3) + asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i])); + if (np >= 4) + asm volatile ("movdqa %%xmm3,%0" : "=m" (s[i])); + if (np >= 5) + asm volatile ("movdqa %%xmm4,%0" : "=m" (t[i])); + if (np >= 6) + asm volatile ("movdqa %%xmm5,%0" : "=m" (u[i])); + } } - raid_sse_end(); + raid_sse_end(streaming); } #endif -/* - * Recover one data failure using P with SSSE3. - * - * The missing block is: - * - * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn - * - * Use temporal stores because recovered data is consumed immediately by - * validation and parity regeneration. - */ -static __always_inline void raid_rec1of1_ssse3(int *id, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *src[RAID_DATA_MAX]; - uint8_t *p; - uint8_t *pa; - size_t i; - int d, s, ns; - - p = v[nd]; - pa = v[id[0]]; - - ns = 0; - for (d = 0; d < nd; ++d) { - if (d != id[0]) - src[ns++] = v[d]; - } - - BUG_ON(ns != nd - 1); - - raid_sse_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16])); - asm volatile ("movdqa %0,%%xmm2" : : "m" (p[i + 32])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (p[i + 48])); - - for (s = 0; s < ns; ++s) { - asm volatile ("pxor %0,%%xmm0" : : "m" (src[s][i])); - asm volatile ("pxor %0,%%xmm1" : : "m" (src[s][i + 16])); - asm volatile ("pxor %0,%%xmm2" : : "m" (src[s][i + 32])); - asm volatile ("pxor %0,%%xmm3" : : "m" (src[s][i + 48])); - } - - asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i])); - asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16])); - asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32])); - asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48])); - } - - raid_sse_end(); -} /* * Recover failure of one data block using selected parity with SSSE3, optimized for one failure. @@ -1303,7 +1329,7 @@ static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t s asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[i])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -1438,7 +1464,7 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -1626,7 +1652,7 @@ static __always_inline void raid_rec2_ssse3_2(int has_p, int *id, int *ip, int n } } - raid_sse_end(); + raid_sse_end(0); } /* @@ -1856,7 +1882,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -2078,7 +2104,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int has_p, int *id, int asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[nr - 1][i])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -2241,7 +2267,7 @@ static __always_inline void raid_recX_ssse3(int nr, int has_p, int *id, int *ip, asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[nr - 1][i])); } - raid_sse_end(); + raid_sse_end(0); } #ifdef CONFIG_X86_64 @@ -2360,7 +2386,7 @@ static __always_inline void raid_rec1_ssse3ext_1(int *id, int *ip, int nd, size_ asm volatile ("movdqa %%xmm9,%0" : "=m" (pa[i + 16])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -2559,7 +2585,7 @@ static __always_inline void raid_rec1_ssse3ext_q(int *id, int *ip, int nd, size_ asm volatile ("movdqa %%xmm11,%0" : "=m" (pa[i + 48])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -2876,7 +2902,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in } } - raid_sse_end(); + raid_sse_end(0); } /* @@ -3218,7 +3244,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly[0])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -3471,7 +3497,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int has_p, int *id, i } } - raid_sse_end(); + raid_sse_end(0); } /* @@ -3767,7 +3793,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id, asm volatile ("movdqa %%xmm10,%0" : "=m" (pa[nr - 1][i])); } - raid_sse_end(); + raid_sse_end(0); } /* @@ -4034,70 +4060,106 @@ static __always_inline void raid_recX_ssse3ext(int nr, int has_p, int *id, int * } } - raid_sse_end(); + raid_sse_end(0); } #endif -void raid_gen3_ssse3_raid(int nd, size_t size, void **vv) +void raid_gen3_ssse3_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen3_ssse3_gen(nd, size, vv, 2); + if (streaming) + raid_gen3_ssse3_gen(nd, size, vv, 2, 1); + else + raid_gen3_ssse3_gen(nd, size, vv, 2, 0); } -void raid_gen3_ssse3_aes(int nd, size_t size, void **vv) +void raid_gen3_ssse3_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen3_ssse3_gen(nd, size, vv, 3); + if (streaming) + raid_gen3_ssse3_gen(nd, size, vv, 3, 1); + else + raid_gen3_ssse3_gen(nd, size, vv, 3, 0); } -void raid_gen4_ssse3_raid(int nd, size_t size, void **vv) +void raid_gen4_ssse3_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen4_ssse3_gen(nd, size, vv, 2); + if (streaming) + raid_gen4_ssse3_gen(nd, size, vv, 2, 1); + else + raid_gen4_ssse3_gen(nd, size, vv, 2, 0); } -void raid_gen4_ssse3_aes(int nd, size_t size, void **vv) +void raid_gen4_ssse3_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen4_ssse3_gen(nd, size, vv, 3); + if (streaming) + raid_gen4_ssse3_gen(nd, size, vv, 3, 1); + else + raid_gen4_ssse3_gen(nd, size, vv, 3, 0); } #ifdef CONFIG_X86_64 -void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv) +void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen3_ssse3ext_gen(nd, size, vv, 2); + if (streaming) + raid_gen3_ssse3ext_gen(nd, size, vv, 2, 1); + else + raid_gen3_ssse3ext_gen(nd, size, vv, 2, 0); } -void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv) +void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen3_ssse3ext_gen(nd, size, vv, 3); + if (streaming) + raid_gen3_ssse3ext_gen(nd, size, vv, 3, 1); + else + raid_gen3_ssse3ext_gen(nd, size, vv, 3, 0); } -void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv) +void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_gen4_ssse3ext_gen(nd, size, vv, 2); + if (streaming) + raid_gen4_ssse3ext_gen(nd, size, vv, 2, 1); + else + raid_gen4_ssse3ext_gen(nd, size, vv, 2, 0); } -void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv) +void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_gen4_ssse3ext_gen(nd, size, vv, 3); + if (streaming) + raid_gen4_ssse3ext_gen(nd, size, vv, 3, 1); + else + raid_gen4_ssse3ext_gen(nd, size, vv, 3, 0); } -void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv) +void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_ssse3ext(nd, size, vv, 5, 2); + if (streaming) + raid_genX_ssse3ext(nd, size, vv, 5, 2, 1); + else + raid_genX_ssse3ext(nd, size, vv, 5, 2, 0); } -void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv) +void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_ssse3ext(nd, size, vv, 5, 3); + if (streaming) + raid_genX_ssse3ext(nd, size, vv, 5, 3, 1); + else + raid_genX_ssse3ext(nd, size, vv, 5, 3, 0); } -void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv) +void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv, int streaming) { - raid_genX_ssse3ext(nd, size, vv, 6, 2); + if (streaming) + raid_genX_ssse3ext(nd, size, vv, 6, 2, 1); + else + raid_genX_ssse3ext(nd, size, vv, 6, 2, 0); } -void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv) +void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv, int streaming) { - raid_genX_ssse3ext(nd, size, vv, 6, 3); + if (streaming) + raid_genX_ssse3ext(nd, size, vv, 6, 3, 1); + else + raid_genX_ssse3ext(nd, size, vv, 6, 3, 0); } #endif @@ -4108,7 +4170,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_ssse3(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } @@ -4180,7 +4242,7 @@ void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1_ssse3(id, nd, size, vv); + raid_rec1of1(id, nd, size, vv); return; } diff --git a/raid/test.c b/raid/test.c index 2bfffe18..6b906ea4 100644 --- a/raid/test.c +++ b/raid/test.c @@ -1115,19 +1115,23 @@ int raid_test_par(int mode, int nd, size_t size) /* check all the functions */ for (k = 0; k < np; ++k) { for (j = 0; j < nf[k]; ++j) { - /* change the buffers incrementing all values */ - for (i = 0; i <= k; ++i) - meminc(v[nd + i], size); + int streaming; - /* compute parity */ - f[k][j](nd, size, v); + for (streaming = 0; streaming <= 1; ++streaming) { + /* change the buffers incrementing all values */ + for (i = 0; i <= k; ++i) + meminc(v[nd + i], size); - /* check it */ - for (i = 0; i < np; ++i) { - if (memcmp(v[nd + np + i], v[nd + i], size) != 0) { - /* LCOV_EXCL_START */ - goto bail; - /* LCOV_EXCL_STOP */ + /* compute parity */ + f[k][j](nd, size, v, streaming); + + /* check it */ + for (i = 0; i < np; ++i) { + if (memcmp(v[nd + np + i], v[nd + i], size) != 0) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } } } } diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 93dda9fd..cea6227b 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -151,14 +151,14 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8s", ""); SPEED_START { - raid_gen1_int32(nd, size, v); + raid_gen1_int32(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_gen1_int64(nd, size, v); + raid_gen1_int64(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -166,7 +166,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON SPEED_START { - raid_gen1_neon(nd, size, v); + raid_gen1_neon(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -174,7 +174,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen1_neon32(nd, size, v); + raid_gen1_neon32(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -184,7 +184,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { - raid_gen1_sse2(nd, size, v); + raid_gen1_sse2(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -201,7 +201,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms } if (raid_cpu_has_avx2()) { SPEED_START { - raid_gen1_avx2(nd, size, v); + raid_gen1_avx2(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -213,7 +213,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen1_avx512bw(nd, size, v); + raid_gen1_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -233,7 +233,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms fflush(stdout); SPEED_START { - raid_gen2_int8(nd, size, v); + raid_gen2_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -241,28 +241,28 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_int32_aes(nd, size, v); + raid_gen2_int32_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_gen2_int64_aes(nd, size, v); + raid_gen2_int64_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } else { SPEED_START { - raid_gen2_int32_raid(nd, size, v); + raid_gen2_int32_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_gen2_int64_raid(nd, size, v); + raid_gen2_int64_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -272,11 +272,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_neon_aes(nd, size, v); + raid_gen2_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_neon_raid(nd, size, v); + raid_gen2_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -286,11 +286,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_neon32_aes(nd, size, v); + raid_gen2_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_neon32_raid(nd, size, v); + raid_gen2_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -302,11 +302,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_sse2()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_sse2_aes(nd, size, v); + raid_gen2_sse2_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_sse2_raid(nd, size, v); + raid_gen2_sse2_raid(nd, size, v, 1); } SPEED_STOP } @@ -316,11 +316,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_sse2ext_aes(nd, size, v); + raid_gen2_sse2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_sse2ext_raid(nd, size, v); + raid_gen2_sse2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -337,11 +337,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx2()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx2_aes(nd, size, v); + raid_gen2_avx2_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_avx2_raid(nd, size, v); + raid_gen2_avx2_raid(nd, size, v, 1); } SPEED_STOP } @@ -351,11 +351,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx2ext_aes(nd, size, v); + raid_gen2_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen2_avx2ext_raid(nd, size, v); + raid_gen2_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -366,7 +366,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen2_avx512bw(nd, size, v); + raid_gen2_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -375,13 +375,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx2gfni_aes(nd, size, v); + raid_gen2_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen2_avx2gfni_raid(nd, size, v); + raid_gen2_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -391,13 +391,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen2_avx512gfni_aes(nd, size, v); + raid_gen2_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen2_avx512gfni_raid(nd, size, v); + raid_gen2_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -414,7 +414,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms fflush(stdout); SPEED_START { - raid_gen3_int8(nd, size, v); + raid_gen3_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -426,11 +426,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_neon_aes(nd, size, v); + raid_gen3_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_neon_raid(nd, size, v); + raid_gen3_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -440,11 +440,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_neon32_aes(nd, size, v); + raid_gen3_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_neon32_raid(nd, size, v); + raid_gen3_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -462,11 +462,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_ssse3_aes(nd, size, v); + raid_gen3_ssse3_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_ssse3_raid(nd, size, v); + raid_gen3_ssse3_raid(nd, size, v, 1); } SPEED_STOP } @@ -475,11 +475,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_ssse3ext_aes(nd, size, v); + raid_gen3_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_ssse3ext_raid(nd, size, v); + raid_gen3_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -492,11 +492,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_avx2ext_aes(nd, size, v); + raid_gen3_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen3_avx2ext_raid(nd, size, v); + raid_gen3_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -507,7 +507,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen3_avx512bw(nd, size, v); + raid_gen3_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -516,13 +516,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_avx2gfni_aes(nd, size, v); + raid_gen3_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen3_avx2gfni_raid(nd, size, v); + raid_gen3_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -532,13 +532,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen3_avx512gfni_aes(nd, size, v); + raid_gen3_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen3_avx512gfni_raid(nd, size, v); + raid_gen3_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -555,7 +555,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms fflush(stdout); SPEED_START { - raid_gen4_int8(nd, size, v); + raid_gen4_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -567,11 +567,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_neon_aes(nd, size, v); + raid_gen4_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_neon_raid(nd, size, v); + raid_gen4_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -581,11 +581,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_neon32_aes(nd, size, v); + raid_gen4_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_neon32_raid(nd, size, v); + raid_gen4_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -603,11 +603,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_ssse3_aes(nd, size, v); + raid_gen4_ssse3_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_ssse3_raid(nd, size, v); + raid_gen4_ssse3_raid(nd, size, v, 1); } SPEED_STOP } @@ -616,11 +616,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_ssse3ext_aes(nd, size, v); + raid_gen4_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_ssse3ext_raid(nd, size, v); + raid_gen4_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -633,11 +633,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_avx2ext_aes(nd, size, v); + raid_gen4_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen4_avx2ext_raid(nd, size, v); + raid_gen4_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -648,7 +648,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen4_avx512bw(nd, size, v); + raid_gen4_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -657,13 +657,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_avx2gfni_aes(nd, size, v); + raid_gen4_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen4_avx2gfni_raid(nd, size, v); + raid_gen4_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -673,13 +673,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen4_avx512gfni_aes(nd, size, v); + raid_gen4_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen4_avx512gfni_raid(nd, size, v); + raid_gen4_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -696,7 +696,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms fflush(stdout); SPEED_START { - raid_gen5_int8(nd, size, v); + raid_gen5_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -708,11 +708,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_neon_aes(nd, size, v); + raid_gen5_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_neon_raid(nd, size, v); + raid_gen5_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -722,11 +722,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_neon32_aes(nd, size, v); + raid_gen5_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_neon32_raid(nd, size, v); + raid_gen5_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -744,7 +744,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_RAID) { SPEED_START { - raid_gen5_ssse3_raid(nd, size, v); + raid_gen5_ssse3_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -755,11 +755,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_ssse3ext_aes(nd, size, v); + raid_gen5_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_ssse3ext_raid(nd, size, v); + raid_gen5_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -772,11 +772,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_avx2ext_aes(nd, size, v); + raid_gen5_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen5_avx2ext_raid(nd, size, v); + raid_gen5_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -787,7 +787,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen5_avx512bw(nd, size, v); + raid_gen5_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -796,13 +796,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_avx2gfni_aes(nd, size, v); + raid_gen5_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen5_avx2gfni_raid(nd, size, v); + raid_gen5_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -812,13 +812,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen5_avx512gfni_aes(nd, size, v); + raid_gen5_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen5_avx512gfni_raid(nd, size, v); + raid_gen5_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -835,7 +835,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms fflush(stdout); SPEED_START { - raid_gen6_int8(nd, size, v); + raid_gen6_int8(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -847,11 +847,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_neon_aes(nd, size, v); + raid_gen6_neon_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_neon_raid(nd, size, v); + raid_gen6_neon_raid(nd, size, v, 1); } SPEED_STOP } @@ -861,11 +861,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON32 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_neon32_aes(nd, size, v); + raid_gen6_neon32_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_neon32_raid(nd, size, v); + raid_gen6_neon32_raid(nd, size, v, 1); } SPEED_STOP } @@ -883,7 +883,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_ssse3()) { if (mode == RAID_MODE_CAUCHY_RAID) { SPEED_START { - raid_gen6_ssse3_raid(nd, size, v); + raid_gen6_ssse3_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -894,11 +894,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_ssse3ext_aes(nd, size, v); + raid_gen6_ssse3ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_ssse3ext_raid(nd, size, v); + raid_gen6_ssse3ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -911,11 +911,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_avx2ext_aes(nd, size, v); + raid_gen6_avx2ext_aes(nd, size, v, 1); } SPEED_STOP } else { SPEED_START { - raid_gen6_avx2ext_raid(nd, size, v); + raid_gen6_avx2ext_raid(nd, size, v, 1); } SPEED_STOP } @@ -926,7 +926,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen6_avx512bw(nd, size, v); + raid_gen6_avx512bw(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -935,13 +935,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx2gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_avx2gfni_aes(nd, size, v); + raid_gen6_avx2gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen6_avx2gfni_raid(nd, size, v); + raid_gen6_avx2gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -951,13 +951,13 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms if (raid_cpu_has_avx512gfni()) { if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { - raid_gen6_avx512gfni_aes(nd, size, v); + raid_gen6_avx512gfni_aes(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); } else { SPEED_START { - raid_gen6_avx512gfni_raid(nd, size, v); + raid_gen6_avx512gfni_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1022,14 +1022,14 @@ void speed_genz(int nd, void **v, int size, int delta, int period) printf("%8s", ""); SPEED_START { - raid_genz_int32_raid(nd, size, v); + raid_genz_int32_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); SPEED_START { - raid_genz_int64_raid(nd, size, v); + raid_genz_int64_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1037,7 +1037,7 @@ void speed_genz(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_genz_neon_raid(nd, size, v); + raid_genz_neon_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1045,7 +1045,7 @@ void speed_genz(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_genz_neon32_raid(nd, size, v); + raid_genz_neon32_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1055,7 +1055,7 @@ void speed_genz(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { - raid_genz_sse2_raid(nd, size, v); + raid_genz_sse2_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1063,7 +1063,7 @@ void speed_genz(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { - raid_genz_sse2ext_raid(nd, size, v); + raid_genz_sse2ext_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1080,7 +1080,7 @@ void speed_genz(int nd, void **v, int size, int delta, int period) printf("%8s", ""); #ifdef CONFIG_X86_64 SPEED_START { - raid_genz_avx2ext_raid(nd, size, v); + raid_genz_avx2ext_raid(nd, size, v, 1); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1183,6 +1183,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3(1, id, ip, nd, size, v); } SPEED_STOP @@ -1191,6 +1193,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3ext(1, id, ip, nd, size, v); } SPEED_STOP @@ -1200,6 +1204,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2(1, id, ip, nd, size, v); } SPEED_STOP @@ -1209,6 +1215,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx2()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2ext(1, id, ip, nd, size, v); } SPEED_STOP @@ -1217,6 +1225,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512bw()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512bw(1, id, ip, nd, size, v); } SPEED_STOP @@ -1225,6 +1235,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP @@ -1233,6 +1245,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { + /* force the parity generator used by raid_rec1of1() */ + raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP