diff --git a/Makefile.am b/Makefile.am index 2d1023f1..5fb01150 100644 --- a/Makefile.am +++ b/Makefile.am @@ -31,7 +31,6 @@ snapraid_SOURCES = \ raid/helper.c \ raid/memory.c \ raid/test.c \ - raid/tag.c \ tommyds/tommy.c \ cmdline/main.c \ cmdline/snapraid.c \ diff --git a/cmdline/speed.c b/cmdline/speed.c index 4b408afa..10ae8c8b 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -290,7 +290,7 @@ void speed(int period, int nd, int size) /* GEN1 */ printf("%8s", "gen1"); - printf("%8s", raid_gen1_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR1)); fflush(stdout); printf("%8s", ""); @@ -343,7 +343,7 @@ void speed(int period, int nd, int size) /* GEN2 */ printf("%8s", "gen2"); - printf("%8s", raid_gen2_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2)); fflush(stdout); printf("%8s", ""); @@ -402,7 +402,7 @@ void speed(int period, int nd, int size) /* GENz */ printf("%8s", "genz"); - printf("%8s", raid_genz_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_VANDERMONDE_PAR3)); fflush(stdout); printf("%8s", ""); @@ -465,7 +465,7 @@ void speed(int period, int nd, int size) /* GEN3 */ printf("%8s", "gen3"); - printf("%8s", raid_gen3_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR3)); fflush(stdout); SPEED_START { @@ -528,7 +528,7 @@ void speed(int period, int nd, int size) /* GEN4 */ printf("%8s", "gen4"); - printf("%8s", raid_gen4_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR4)); fflush(stdout); SPEED_START { @@ -593,7 +593,7 @@ void speed(int period, int nd, int size) /* GEN5 */ printf("%8s", "gen5"); - printf("%8s", raid_gen5_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR5)); fflush(stdout); SPEED_START { @@ -658,7 +658,7 @@ void speed(int period, int nd, int size) /* GEN6 */ printf("%8s", "gen6"); - printf("%8s", raid_gen6_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR5)); fflush(stdout); SPEED_START { @@ -734,12 +734,12 @@ void speed(int period, int nd, int size) printf("\n"); printf("%8s", "rec1"); - printf("%8s", raid_rec1_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1)); fflush(stdout); SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[0] = sizeof(void*) == 8 ? raid_gen1_int64 : raid_gen1_int32; + raid_gen_force(1, sizeof(void*) == 8 ? raid_gen1_int64 : raid_gen1_int32); /* +1 to avoid GEN1 optimized case */ raid_rec1_int8(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -752,7 +752,7 @@ void speed(int period, int nd, int size) if (raid_cpu_has_ssse3()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[0] = raid_gen1_sse2; + raid_gen_force(1, raid_gen1_sse2); /* +1 to avoid GEN1 optimized case */ raid_rec1_ssse3(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -765,7 +765,7 @@ void speed(int period, int nd, int size) if (raid_cpu_has_avx2()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[0] = raid_gen1_avx2; + raid_gen_force(1, raid_gen1_avx2); /* +1 to avoid GEN1 optimized case */ raid_rec1_avx2(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -778,12 +778,12 @@ void speed(int period, int nd, int size) printf("\n"); printf("%8s", "rec2"); - printf("%8s", raid_rec2_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2)); fflush(stdout); SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[1] = sizeof(void*) == 8 ? raid_gen2_int64 : raid_gen2_int32; + raid_gen_force(2, sizeof(void*) == 8 ? raid_gen2_int64 : raid_gen2_int32); /* +1 to avoid GEN2 optimized case */ raid_rec2_int8(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -797,9 +797,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[1] = raid_gen2_sse2ext; + raid_gen_force(2, raid_gen2_sse2ext); #else - raid_gen_ptr[1] = raid_gen2_sse2; + raid_gen_force(2, raid_gen2_sse2); #endif /* +1 to avoid GEN2 optimized case */ raid_rec2_ssse3(2, id, ip + 1, nd, size, v); @@ -813,7 +813,7 @@ void speed(int period, int nd, int size) if (raid_cpu_has_avx2()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[1] = raid_gen2_avx2; + raid_gen_force(2, raid_gen2_avx2); /* +1 to avoid GEN1 optimized case */ raid_rec2_avx2(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -826,12 +826,12 @@ void speed(int period, int nd, int size) printf("\n"); printf("%8s", "rec3"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3)); fflush(stdout); SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[2] = raid_gen3_int8; + raid_gen_force(3, raid_gen3_int8); raid_recX_int8(3, id, ip, nd, size, v); } SPEED_STOP @@ -844,9 +844,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[2] = raid_gen3_ssse3ext; + raid_gen_force(3, raid_gen3_ssse3ext); #else - raid_gen_ptr[2] = raid_gen3_ssse3; + raid_gen_force(3, raid_gen3_ssse3); #endif raid_recX_ssse3(3, id, ip, nd, size, v); } SPEED_STOP @@ -860,9 +860,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[2] = raid_gen3_avx2ext; + raid_gen_force(3, raid_gen3_avx2ext); #else - raid_gen_ptr[2] = raid_gen3_ssse3; + raid_gen_force(3, raid_gen3_ssse3); #endif raid_recX_avx2(3, id, ip, nd, size, v); } SPEED_STOP @@ -875,12 +875,12 @@ void speed(int period, int nd, int size) printf("\n"); printf("%8s", "rec4"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR4)); fflush(stdout); SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[3] = raid_gen4_int8; + raid_gen_force(4, raid_gen4_int8); raid_recX_int8(4, id, ip, nd, size, v); } SPEED_STOP @@ -893,9 +893,10 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[3] = raid_gen4_ssse3ext; + raid_gen_force(4, raid_gen4_ssse3ext); #else - raid_gen_ptr[3] = raid_gen4_ssse3; + + raid_gen_force(4, raid_gen4_ssse3); #endif raid_recX_ssse3(4, id, ip, nd, size, v); } SPEED_STOP @@ -909,9 +910,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[3] = raid_gen4_avx2ext; + raid_gen_force(4, raid_gen4_avx2ext); #else - raid_gen_ptr[3] = raid_gen4_ssse3; + raid_gen_force(4, raid_gen4_ssse3); #endif raid_recX_avx2(4, id, ip, nd, size, v); } SPEED_STOP @@ -924,12 +925,12 @@ void speed(int period, int nd, int size) printf("\n"); printf("%8s", "rec5"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR5)); fflush(stdout); SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[4] = raid_gen5_int8; + raid_gen_force(5, raid_gen5_int8); raid_recX_int8(5, id, ip, nd, size, v); } SPEED_STOP @@ -942,9 +943,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[4] = raid_gen5_ssse3ext; + raid_gen_force(5, raid_gen5_ssse3ext); #else - raid_gen_ptr[4] = raid_gen5_ssse3; + raid_gen_force(5, raid_gen5_ssse3); #endif raid_recX_ssse3(5, id, ip, nd, size, v); } SPEED_STOP @@ -958,9 +959,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[4] = raid_gen5_avx2ext; + raid_gen_force(5, raid_gen5_avx2ext); #else - raid_gen_ptr[4] = raid_gen5_ssse3; + raid_gen_force(5, raid_gen5_ssse3); #endif raid_recX_avx2(5, id, ip, nd, size, v); } SPEED_STOP @@ -973,12 +974,12 @@ void speed(int period, int nd, int size) printf("\n"); printf("%8s", "rec6"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR6)); fflush(stdout); SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_ptr[5] = raid_gen6_int8; + raid_gen_force(6, raid_gen6_int8); raid_recX_int8(6, id, ip, nd, size, v); } SPEED_STOP @@ -991,9 +992,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[5] = raid_gen6_ssse3ext; + raid_gen_force(6, raid_gen6_ssse3ext); #else - raid_gen_ptr[5] = raid_gen6_ssse3; + raid_gen_force(6, raid_gen6_ssse3); #endif raid_recX_ssse3(6, id, ip, nd, size, v); } SPEED_STOP @@ -1007,9 +1008,9 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ #ifdef CONFIG_X86_64 - raid_gen_ptr[5] = raid_gen6_avx2ext; + raid_gen_force(6, raid_gen6_avx2ext); #else - raid_gen_ptr[5] = raid_gen6_ssse3; + raid_gen_force(6, raid_gen6_ssse3); #endif raid_recX_avx2(6, id, ip, nd, size, v); } SPEED_STOP diff --git a/raid/check.c b/raid/check.c index 8c0e290f..304c8e1c 100644 --- a/raid/check.c +++ b/raid/check.c @@ -29,7 +29,7 @@ * @v Vector of pointers to the blocks of data and parity. * It has (@nd + @ip[@nv - 1] + 1) elements. The starting elements are the * blocks for data, following with the parity blocks. - * Each block has @size bytes. + * Each block has @size bytes. * @return 0 if the check is satisfied. -1 otherwise. */ static int raid_validate(int nr, int *id, int nv, int *ip, int nd, size_t size, void **vv) @@ -125,7 +125,7 @@ int raid_check(int nr, int *ir, int nd, int np, size_t size, void **v) BUG_ON(nr >= 6 && ir[4] >= ir[5]); /* enforce limit on index vector */ - BUG_ON(nr > 0 && ir[nr-1] >= nd + np); + BUG_ON(nr > 0 && ir[nr - 1] >= nd + np); /* count failed data disk */ rd = 0; diff --git a/raid/int.c b/raid/int.c index 73ef9166..5993a227 100644 --- a/raid/int.c +++ b/raid/int.c @@ -543,3 +543,26 @@ void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } +void raid_register_int(void) +{ + if (sizeof(void *) == 4) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int32", raid_gen1_int32); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32); + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int32", raid_genz_int32); + } else { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int64", raid_gen1_int64); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64); + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64); + } + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_gen6_int8); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "int8", raid_rec1_int8); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_rec2_int8); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_recX_int8); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_recX_int8); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_recX_int8); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_recX_int8); +} diff --git a/raid/internal.h b/raid/internal.h index 3825b7d0..36d61528 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -9,7 +9,7 @@ * * It may happen that the assembler is too old to support * all instructions, even if the architecture supports them. - * These defines allow to exclude from the build the not supported ones. + * These defines allow to exclude from the build the unsupported ones. * * If in your project you use a predefined assembler, you can define them * using fixed values, instead of using the HAVE_* defines. @@ -112,9 +112,8 @@ static __always_inline void *__align_ptr(void *ptr, uintptr_t size) /* * Internal functions. * - * These are intended to provide access for testing. + * These are intended to provide external access for testing. */ -int raid_selftest(void); void raid_gen_ref(int nd, int np, size_t size, void **vv); void raid_invert(uint8_t *M, uint8_t *V, int n); void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v); @@ -161,30 +160,118 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); /* - * Internal naming. + * Functions for parity computation. * - * These are intended to provide access for testing. + * These functions compute the parity blocks from the provided data. + * + * The number of parities to compute is implicit in the position in the + * forwarder vector. Position at index #i, computes (#i+1) parities. + * + * All these functions give the guarantee that parities are written + * in order. First parity P, then parity Q, and so on. + * This allows to specify the same memory buffer for multiple parities + * knowing that you'll get the latest written one. + * This characteristic is used by the raid_delta_gen() function to + * avoid to damage unused parities in recovering. + * + * @nd Number of data blocks + * @size Size of the blocks pointed by @vv. It must be a multiple of 64. + * @vv Vector of pointers to the blocks of data and parity. + * It has (@nd + #parities) elements. The starting elements are the blocks + * for data, following with the parity blocks. + * Each block has @size bytes. */ -const char *raid_gen1_tag(void); -const char *raid_gen2_tag(void); -const char *raid_genz_tag(void); -const char *raid_gen3_tag(void); -const char *raid_gen4_tag(void); -const char *raid_gen5_tag(void); -const char *raid_gen6_tag(void); -const char *raid_rec1_tag(void); -const char *raid_rec2_tag(void); -const char *raid_recX_tag(void); +typedef void (raid_gen_fn)(int nd, size_t size, void **vv); /* - * Internal forwarders. + * Functions for data recovery. + * + * These functions recover data blocks using the specified parity + * to recompute the missing data. + * + * Note that the format of vectors @id/@ip is different than raid_rec(). + * For example, in the vector @ip the first parity is represented with the + * value 0 and not @nd. + * + * @nr Number of failed data blocks to recover. + * @id[] Vector of @nr indexes of the data blocks to recover. + * The indexes start from 0. They must be in order. + * @ip[] Vector of @nr indexes of the parity blocks to use in the recovering. + * The indexes start from 0. They must be in order. + * @nd Number of data blocks. + * @np Number of parity blocks. + * @size Size of the blocks pointed by @vv. It must be a multiple of 64. + * @vv Vector of pointers to the blocks of data and parity. + * It has (@nd + @np) elements. The starting elements are the blocks + * for data, following with the parity blocks. + * Each block has @size bytes. */ -extern void (*raid_gen3_ptr)(int nd, size_t size, void **vv); -extern void (*raid_genz_ptr)(int nd, size_t size, void **vv); -extern void (*raid_gen_ptr[RAID_PARITY_MAX])( - int nd, size_t size, void **vv); -extern void (*raid_rec_ptr[RAID_PARITY_MAX])( - int nr, int *id, int *ip, int nd, size_t size, void **vv); +typedef void (raid_rec_fn)(int nr, int *id, int *ip, int nd, size_t size, void **vv); + +/** + * Algorithm indexes + * + * To be used with the register and tag functions. + */ +#define RAID_ALGO_CAUCHY_PAR1 0 +#define RAID_ALGO_CAUCHY_PAR2 1 +#define RAID_ALGO_CAUCHY_PAR3 2 +#define RAID_ALGO_CAUCHY_PAR4 3 +#define RAID_ALGO_CAUCHY_PAR5 4 +#define RAID_ALGO_CAUCHY_PAR6 5 +#define RAID_ALGO_VANDERMONDE_PAR3 6 +#define RAID_ALGO_MAX 7 + +/** + * Register functions for parity computation and data recovery. + * + * Each call overwrites the previous setting. Thus, call it from the + * slowest to the fastest. + * + * @na Algo code of the function. One of RAID_ALGO_*. + * @tag Descriptive short tag of the implementation, like "sse2", "avx2",... + * @fn Function to register. + */ +void raid_gen_register(int na, const char *tag, raid_gen_fn *fn); +void raid_rec_register(int na, const char *tag, raid_rec_fn *fn); + +/** + * Set functions for data recovery. + * + * Intended only for testing the recovery function forcing a specific + * parity generation for the delta step. + * + * Each call overwrites the previous setting. + * + * @np Number of parities. + * @fn Function to register. + */ +void raid_gen_force(int np, raid_gen_fn *fn); + +/** + * Register all the functions based on integer variables. + */ +void raid_register_int(void); + +/** + * Register all the functions based on x86 intructions. + */ +void raid_register_x86(void); + +/* + * Tag functions. + * + * Given the specified algo code, return the tag of the registered function. + */ +const char *raid_gen_tag(int na); +const char *raid_rec_tag(int na); + +/** + * Basic functionality self test. + * + * Returns 0 on success. + */ +int raid_selftest(void); /* * Tables. diff --git a/raid/module.c b/raid/module.c index 18cc7f2c..36cb4d44 100644 --- a/raid/module.c +++ b/raid/module.c @@ -5,101 +5,246 @@ #include "memory.h" #include "cpu.h" +/** + * Forwarders for parity generation. + * + * Set by the raid_mode() call. + * + * Index 0 is for one parity, 1 for the two parities, and so on. + */ +static raid_gen_fn *raid_gen_ptr[RAID_PARITY_MAX]; + +struct raid_gen_algo { + raid_gen_fn *gen; + const char *tag; /**< Hardware descriptive tag. */ +}; + +/** + * Registered algorithms for parity generation. + * + * Set by the raid_gen_register() calls. + * + * Indexes are the RAID_ALGO_* constants + */ +static struct raid_gen_algo raid_gen_algo[RAID_ALGO_MAX]; + +void raid_gen(int nd, int np, size_t size, void **v) +{ + /* enforce limit on size */ + BUG_ON(size % 64 != 0); + + /* enforce limit on number of failures */ + BUG_ON(np < 1); + BUG_ON(np > RAID_PARITY_MAX); + + raid_gen_ptr[np - 1](nd, size, v); +} + +/** + * Forwarders for data recovery. + * + * Set by the raid_mode() call. + * + * Index 0 is for one parity, 1 for the two parities, and so on. + */ +static raid_rec_fn *raid_rec_ptr[RAID_PARITY_MAX]; + +struct raid_rec_algo { + raid_rec_fn *rec; + const char *tag; /**< Hardware descriptive tag. */ +}; + +/** + * Registered algorithms for data recovery. + * + * Set by the raid_rec_register() calls. + * + * Indexes are the RAID_ALGO_* constants + */ +static struct raid_rec_algo raid_rec_algo[RAID_PARITY_MAX]; + +void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v) +{ + int nrd; /* number of data blocks to recover */ + int nrp; /* number of parity blocks to recover */ + + /* enforce limit on size */ + BUG_ON(size % 64 != 0); + + /* enforce limit on number of failures */ + BUG_ON(nr > np); + BUG_ON(np > RAID_PARITY_MAX); + + /* enforce order in index vector */ + BUG_ON(nr >= 2 && ir[0] >= ir[1]); + BUG_ON(nr >= 3 && ir[1] >= ir[2]); + BUG_ON(nr >= 4 && ir[2] >= ir[3]); + BUG_ON(nr >= 5 && ir[3] >= ir[4]); + BUG_ON(nr >= 6 && ir[4] >= ir[5]); + + /* enforce limit on index vector */ + BUG_ON(nr > 0 && ir[nr - 1] >= nd + np); + + /* count the number of data blocks to recover */ + nrd = 0; + while (nrd < nr && ir[nrd] < nd) + ++nrd; + + /* all the remaining are parity */ + nrp = nr - nrd; + + /* enforce limit on number of failures */ + BUG_ON(nrd > nd); + BUG_ON(nrp > np); + + /* if failed data is present */ + if (nrd != 0) { + int ip[RAID_PARITY_MAX]; + int i, j, k; + + /* setup the vector of parities to use */ + for (i = 0, j = 0, k = 0; i < np; ++i) { + if (j < nrp && ir[nrd + j] == nd + i) { + /* this parity has to be recovered */ + ++j; + } else { + /* this parity is used for recovering */ + ip[k] = i; + ++k; + } + } + + /* recover the nrd data blocks specified in ir[], */ + /* using the first nrd parity in ip[] for recovering */ + raid_rec_ptr[nrd - 1](nrd, ir, ip, nd, size, v); + } + + /* recompute all the parities up to the last bad one */ + if (nrp != 0) + raid_gen(nd, ir[nr - 1] - nd + 1, size, v); +} + +void raid_data(int nr, int *id, int *ip, int nd, size_t size, void **v) +{ + /* enforce limit on size */ + BUG_ON(size % 64 != 0); + + /* enforce limit on number of failures */ + BUG_ON(nr > nd); + BUG_ON(nr > RAID_PARITY_MAX); + + /* enforce order in index vector for data */ + BUG_ON(nr >= 2 && id[0] >= id[1]); + BUG_ON(nr >= 3 && id[1] >= id[2]); + BUG_ON(nr >= 4 && id[2] >= id[3]); + BUG_ON(nr >= 5 && id[3] >= id[4]); + BUG_ON(nr >= 6 && id[4] >= id[5]); + + /* enforce limit on index vector for data */ + BUG_ON(nr > 0 && id[nr - 1] >= nd); + + /* enforce order in index vector for parity */ + BUG_ON(nr >= 2 && ip[0] >= ip[1]); + BUG_ON(nr >= 3 && ip[1] >= ip[2]); + BUG_ON(nr >= 4 && ip[2] >= ip[3]); + BUG_ON(nr >= 5 && ip[3] >= ip[4]); + BUG_ON(nr >= 6 && ip[4] >= ip[5]); + + /* if failed data is present */ + if (nr != 0) + raid_rec_algo[nr - 1].rec(nr, id, ip, nd, size, v); +} + +const char *raid_gen_tag(int na) +{ + BUG_ON(na < 0 || na >= RAID_ALGO_MAX); + + return raid_gen_algo[na].tag; +} + +const char *raid_rec_tag(int na) +{ + /* there is no custom recover for vandermonde */ + if (na == RAID_ALGO_VANDERMONDE_PAR3) + na = RAID_ALGO_CAUCHY_PAR3; + + BUG_ON(na < 0 || na >= RAID_PARITY_MAX); + + return raid_rec_algo[na].tag; +} + +/** + * Generator matrix currently used. + */ +const uint8_t(*raid_gfgen)[256]; + +void raid_mode(int mode) +{ + BUG_ON(mode != RAID_MODE_VANDERMONDE && mode != RAID_MODE_CAUCHY); + + if (mode == RAID_MODE_VANDERMONDE) { + raid_gen_ptr[0] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR1].gen; + raid_gen_ptr[1] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR2].gen; + raid_gen_ptr[2] = raid_gen_algo[RAID_ALGO_VANDERMONDE_PAR3].gen; + raid_gen_ptr[3] = 0; + raid_gen_ptr[4] = 0; + raid_gen_ptr[5] = 0; + raid_rec_ptr[0] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR1].rec; + raid_rec_ptr[1] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR2].rec; + raid_rec_ptr[2] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR3].rec; + raid_rec_ptr[3] = 0; + raid_rec_ptr[4] = 0; + raid_rec_ptr[5] = 0; + raid_gfgen = gfvandermonde; + } else { + raid_gen_ptr[0] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR1].gen; + raid_gen_ptr[1] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR2].gen; + raid_gen_ptr[2] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR3].gen; + raid_gen_ptr[3] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR4].gen; + raid_gen_ptr[4] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR5].gen; + raid_gen_ptr[5] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR6].gen; + raid_rec_ptr[0] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR1].rec; + raid_rec_ptr[1] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR2].rec; + raid_rec_ptr[2] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR3].rec; + raid_rec_ptr[3] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR4].rec; + raid_rec_ptr[4] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR5].rec; + raid_rec_ptr[5] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR6].rec; + raid_gfgen = gfcauchy; + } +} + +void raid_gen_force(int np, raid_gen_fn *fn) +{ + BUG_ON(np < 1 || np > RAID_PARITY_MAX); + + raid_gen_ptr[np - 1] = fn; +} + +void raid_gen_register(int na, const char *tag, raid_gen_fn *gen) +{ + BUG_ON(na < 0 || na >= RAID_ALGO_MAX); + + raid_gen_algo[na].tag = tag; + raid_gen_algo[na].gen = gen; +} + +void raid_rec_register(int na, const char *tag, raid_rec_fn *rec) +{ + BUG_ON(na < 0 || na >= RAID_PARITY_MAX); + + raid_rec_algo[na].tag = tag; + raid_rec_algo[na].rec = rec; +} + /* * Initializes and selects the best algorithm. */ void raid_init(void) { - raid_gen3_ptr = raid_gen3_int8; - raid_gen_ptr[3] = raid_gen4_int8; - raid_gen_ptr[4] = raid_gen5_int8; - raid_gen_ptr[5] = raid_gen6_int8; - - if (sizeof(void *) == 4) { - raid_gen_ptr[0] = raid_gen1_int32; - raid_gen_ptr[1] = raid_gen2_int32; - raid_genz_ptr = raid_genz_int32; - } else { - raid_gen_ptr[0] = raid_gen1_int64; - raid_gen_ptr[1] = raid_gen2_int64; - raid_genz_ptr = raid_genz_int64; - } - - raid_rec_ptr[0] = raid_rec1_int8; - raid_rec_ptr[1] = raid_rec2_int8; - raid_rec_ptr[2] = raid_recX_int8; - raid_rec_ptr[3] = raid_recX_int8; - raid_rec_ptr[4] = raid_recX_int8; - raid_rec_ptr[5] = raid_recX_int8; - -#ifdef CONFIG_X86 -#ifdef CONFIG_SSE2 - if (raid_cpu_has_sse2()) { - raid_gen_ptr[0] = raid_gen1_sse2; -#ifdef CONFIG_X86_64 - if (raid_cpu_has_slowextendedreg()) { - raid_gen_ptr[1] = raid_gen2_sse2; - } else { - raid_gen_ptr[1] = raid_gen2_sse2ext; - } - /* note that raid_cpu_has_slowextendedreg() doesn't affect parz */ - raid_genz_ptr = raid_genz_sse2ext; -#else - raid_gen_ptr[1] = raid_gen2_sse2; - raid_genz_ptr = raid_genz_sse2; + raid_register_int(); +#if defined(CONFIG_X86) && defined(CONFIG_SSE2) + raid_register_x86(); #endif - } -#endif - -#ifdef CONFIG_SSSE3 - if (raid_cpu_has_ssse3()) { -#ifdef CONFIG_X86_64 - if (raid_cpu_has_slowextendedreg()) { - raid_gen3_ptr = raid_gen3_ssse3; - raid_gen_ptr[3] = raid_gen4_ssse3; - raid_gen_ptr[4] = raid_gen5_ssse3; - raid_gen_ptr[5] = raid_gen6_ssse3; - } else { - raid_gen3_ptr = raid_gen3_ssse3ext; - raid_gen_ptr[3] = raid_gen4_ssse3ext; - raid_gen_ptr[4] = raid_gen5_ssse3ext; - raid_gen_ptr[5] = raid_gen6_ssse3ext; - } -#else - raid_gen3_ptr = raid_gen3_ssse3; - raid_gen_ptr[3] = raid_gen4_ssse3; - raid_gen_ptr[4] = raid_gen5_ssse3; - raid_gen_ptr[5] = raid_gen6_ssse3; -#endif - raid_rec_ptr[0] = raid_rec1_ssse3; - raid_rec_ptr[1] = raid_rec2_ssse3; - raid_rec_ptr[2] = raid_recX_ssse3; - raid_rec_ptr[3] = raid_recX_ssse3; - raid_rec_ptr[4] = raid_recX_ssse3; - raid_rec_ptr[5] = raid_recX_ssse3; - } -#endif - -#ifdef CONFIG_AVX2 - if (raid_cpu_has_avx2()) { - raid_gen_ptr[0] = raid_gen1_avx2; - raid_gen_ptr[1] = raid_gen2_avx2; -#ifdef CONFIG_X86_64 - raid_gen3_ptr = raid_gen3_avx2ext; - raid_genz_ptr = raid_genz_avx2ext; - raid_gen_ptr[3] = raid_gen4_avx2ext; - raid_gen_ptr[4] = raid_gen5_avx2ext; - raid_gen_ptr[5] = raid_gen6_avx2ext; -#endif - raid_rec_ptr[0] = raid_rec1_avx2; - raid_rec_ptr[1] = raid_rec2_avx2; - raid_rec_ptr[2] = raid_recX_avx2; - raid_rec_ptr[3] = raid_recX_avx2; - raid_rec_ptr[4] = raid_recX_avx2; - raid_rec_ptr[5] = raid_recX_avx2; - } -#endif -#endif /* CONFIG_X86 */ /* set the default mode */ raid_mode(RAID_MODE_CAUCHY); @@ -292,9 +437,6 @@ static int raid_test_scan(int nr, int *ir, int nd, int np, size_t size, void **v return 0; } -/* - * Basic functionality self test. - */ int raid_selftest(void) { const int nd = TEST_COUNT; diff --git a/raid/raid.c b/raid/raid.c index 6faae922..003f3c6b 100644 --- a/raid/raid.c +++ b/raid/raid.c @@ -86,7 +86,7 @@ * on top, and it remains MDS. This forms the code for m=4, with RAID-6 as * a subset: * - * 1 1 1 1 1 1 + * 1 1 1 1 1 1 * 1 2 4 8 16 32 * 1 245 210 196 154 113 * 1 187 166 215 199 7 @@ -196,22 +196,6 @@ * [5] Plank, "Screaming Fast Galois Field Arithmetic Using Intel SIMD Instructions", 2013 */ -/** - * Generator matrix currently used. - */ -const uint8_t (*raid_gfgen)[256]; - -void raid_mode(int mode) -{ - if (mode == RAID_MODE_VANDERMONDE) { - raid_gen_ptr[2] = raid_genz_ptr; - raid_gfgen = gfvandermonde; - } else { - raid_gen_ptr[2] = raid_gen3_ptr; - raid_gfgen = gfcauchy; - } -} - /** * Buffer filled with 0 used in recovering. */ @@ -222,44 +206,6 @@ void raid_zero(void *zero) raid_zero_block = zero; } -/* - * Forwarders for parity computation. - * - * These functions compute the parity blocks from the provided data. - * - * The number of parities to compute is implicit in the position in the - * forwarder vector. Position at index #i, computes (#i+1) parities. - * - * All these functions give the guarantee that parities are written - * in order. First parity P, then parity Q, and so on. - * This allows to specify the same memory buffer for multiple parities - * knowing that you'll get the latest written one. - * This characteristic is used by the raid_delta_gen() function to - * avoid to damage unused parities in recovering. - * - * @nd Number of data blocks - * @size Size of the blocks pointed by @v. It must be a multiplier of 64. - * @v Vector of pointers to the blocks of data and parity. - * It has (@nd + #parities) elements. The starting elements are the blocks - * for data, following with the parity blocks. - * Each block has @size bytes. - */ -void (*raid_gen_ptr[RAID_PARITY_MAX])(int nd, size_t size, void **vv); -void (*raid_gen3_ptr)(int nd, size_t size, void **vv); -void (*raid_genz_ptr)(int nd, size_t size, void **vv); - -void raid_gen(int nd, int np, size_t size, void **v) -{ - /* enforce limit on size */ - BUG_ON(size % 64 != 0); - - /* enforce limit on number of failures */ - BUG_ON(np < 1); - BUG_ON(np > RAID_PARITY_MAX); - - raid_gen_ptr[np - 1](nd, size, v); -} - /** * Inverts the square matrix M of size nxn into V. * @@ -493,122 +439,3 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv) qa[i] = Dy; } } - -/* - * Forwarders for data recovery. - * - * These functions recover data blocks using the specified parity - * to recompute the missing data. - * - * Note that the format of vectors @id/@ip is different than raid_rec(). - * For example, in the vector @ip the first parity is represented with the - * value 0 and not @nd. - * - * @nr Number of failed data blocks to recover. - * @id[] Vector of @nr indexes of the data blocks to recover. - * The indexes start from 0. They must be in order. - * @ip[] Vector of @nr indexes of the parity blocks to use in the recovering. - * The indexes start from 0. They must be in order. - * @nd Number of data blocks. - * @np Number of parity blocks. - * @size Size of the blocks pointed by @v. It must be a multiplier of 64. - * @v Vector of pointers to the blocks of data and parity. - * It has (@nd + @np) elements. The starting elements are the blocks - * for data, following with the parity blocks. - * Each block has @size bytes. - */ -void (*raid_rec_ptr[RAID_PARITY_MAX])( - int nr, int *id, int *ip, int nd, size_t size, void **vv); - -void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v) -{ - int nrd; /* number of data blocks to recover */ - int nrp; /* number of parity blocks to recover */ - - /* enforce limit on size */ - BUG_ON(size % 64 != 0); - - /* enforce limit on number of failures */ - BUG_ON(nr > np); - BUG_ON(np > RAID_PARITY_MAX); - - /* enforce order in index vector */ - BUG_ON(nr >= 2 && ir[0] >= ir[1]); - BUG_ON(nr >= 3 && ir[1] >= ir[2]); - BUG_ON(nr >= 4 && ir[2] >= ir[3]); - BUG_ON(nr >= 5 && ir[3] >= ir[4]); - BUG_ON(nr >= 6 && ir[4] >= ir[5]); - - /* enforce limit on index vector */ - BUG_ON(nr > 0 && ir[nr-1] >= nd + np); - - /* count the number of data blocks to recover */ - nrd = 0; - while (nrd < nr && ir[nrd] < nd) - ++nrd; - - /* all the remaining are parity */ - nrp = nr - nrd; - - /* enforce limit on number of failures */ - BUG_ON(nrd > nd); - BUG_ON(nrp > np); - - /* if failed data is present */ - if (nrd != 0) { - int ip[RAID_PARITY_MAX]; - int i, j, k; - - /* setup the vector of parities to use */ - for (i = 0, j = 0, k = 0; i < np; ++i) { - if (j < nrp && ir[nrd + j] == nd + i) { - /* this parity has to be recovered */ - ++j; - } else { - /* this parity is used for recovering */ - ip[k] = i; - ++k; - } - } - - /* recover the nrd data blocks specified in ir[], */ - /* using the first nrd parity in ip[] for recovering */ - raid_rec_ptr[nrd - 1](nrd, ir, ip, nd, size, v); - } - - /* recompute all the parities up to the last bad one */ - if (nrp != 0) - raid_gen(nd, ir[nr - 1] - nd + 1, size, v); -} - -void raid_data(int nr, int *id, int *ip, int nd, size_t size, void **v) -{ - /* enforce limit on size */ - BUG_ON(size % 64 != 0); - - /* enforce limit on number of failures */ - BUG_ON(nr > nd); - BUG_ON(nr > RAID_PARITY_MAX); - - /* enforce order in index vector for data */ - BUG_ON(nr >= 2 && id[0] >= id[1]); - BUG_ON(nr >= 3 && id[1] >= id[2]); - BUG_ON(nr >= 4 && id[2] >= id[3]); - BUG_ON(nr >= 5 && id[3] >= id[4]); - BUG_ON(nr >= 6 && id[4] >= id[5]); - - /* enforce limit on index vector for data */ - BUG_ON(nr > 0 && id[nr-1] >= nd); - - /* enforce order in index vector for parity */ - BUG_ON(nr >= 2 && ip[0] >= ip[1]); - BUG_ON(nr >= 3 && ip[1] >= ip[2]); - BUG_ON(nr >= 4 && ip[2] >= ip[3]); - BUG_ON(nr >= 5 && ip[3] >= ip[4]); - BUG_ON(nr >= 6 && ip[4] >= ip[5]); - - /* if failed data is present */ - if (nr != 0) - raid_rec_ptr[nr - 1](nr, id, ip, nd, size, v); -} - diff --git a/raid/tag.c b/raid/tag.c deleted file mode 100644 index 82dbfdba..00000000 --- a/raid/tag.c +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -// Copyright (C) 2013 Andrea Mazzoleni - -#include "internal.h" - -typedef void (void_f)(void); - -static struct raid_func { - const char *name; - void_f* func; -} RAID_FUNC[] = { - { "int8", (void_f*)raid_gen3_int8 }, - { "int8", (void_f*)raid_gen4_int8 }, - { "int8", (void_f*)raid_gen5_int8 }, - { "int8", (void_f*)raid_gen6_int8 }, - { "int32", (void_f*)raid_gen1_int32 }, - { "int64", (void_f*)raid_gen1_int64 }, - { "int32", (void_f*)raid_gen2_int32 }, - { "int64", (void_f*)raid_gen2_int64 }, - { "int32", (void_f*)raid_genz_int32 }, - { "int64", (void_f*)raid_genz_int64 }, - { "int8", (void_f*)raid_rec1_int8 }, - { "int8", (void_f*)raid_rec2_int8 }, - { "int8", (void_f*)raid_recX_int8 }, - -#ifdef CONFIG_X86 -#ifdef CONFIG_SSE2 - { "sse2", (void_f*)raid_gen1_sse2 }, - { "sse2", (void_f*)raid_gen2_sse2 }, - { "sse2", (void_f*)raid_genz_sse2 }, -#endif -#ifdef CONFIG_SSSE3 - { "ssse3", (void_f*)raid_gen3_ssse3 }, - { "ssse3", (void_f*)raid_gen4_ssse3 }, - { "ssse3", (void_f*)raid_gen5_ssse3 }, - { "ssse3", (void_f*)raid_gen6_ssse3 }, - { "ssse3", (void_f*)raid_rec1_ssse3 }, - { "ssse3", (void_f*)raid_rec2_ssse3 }, - { "ssse3", (void_f*)raid_recX_ssse3 }, -#endif -#ifdef CONFIG_AVX2 - { "avx2", (void_f*)raid_gen1_avx2 }, - { "avx2", (void_f*)raid_gen2_avx2 }, - { "avx2", (void_f*)raid_rec1_avx2 }, - { "avx2", (void_f*)raid_rec2_avx2 }, - { "avx2", (void_f*)raid_recX_avx2 }, -#endif -#endif - -#ifdef CONFIG_X86_64 -#ifdef CONFIG_SSE2 - { "sse2e", (void_f*)raid_gen2_sse2ext }, - { "sse2e", (void_f*)raid_genz_sse2ext }, -#endif -#ifdef CONFIG_SSSE3 - { "ssse3e", (void_f*)raid_gen3_ssse3ext }, - { "ssse3e", (void_f*)raid_gen4_ssse3ext }, - { "ssse3e", (void_f*)raid_gen5_ssse3ext }, - { "ssse3e", (void_f*)raid_gen6_ssse3ext }, -#endif -#ifdef CONFIG_AVX2 - { "avx2e", (void_f*)raid_gen3_avx2ext }, - { "avx2e", (void_f*)raid_genz_avx2ext }, - { "avx2e", (void_f*)raid_gen4_avx2ext }, - { "avx2e", (void_f*)raid_gen5_avx2ext }, - { "avx2e", (void_f*)raid_gen6_avx2ext }, -#endif -#endif - { 0, 0 } -}; - -static const char *raid_tag(void_f* func) -{ - struct raid_func *i = RAID_FUNC; - - while (i->name != 0) { - if (i->func == func) - return i->name; - ++i; - } - - /* LCOV_EXCL_START */ - return "unknown"; - /* LCOV_EXCL_STOP */ -} - -const char *raid_gen1_tag(void) -{ - return raid_tag((void_f*)raid_gen_ptr[0]); -} - -const char *raid_gen2_tag(void) -{ - return raid_tag((void_f*)raid_gen_ptr[1]); -} - -const char *raid_genz_tag(void) -{ - return raid_tag((void_f*)raid_genz_ptr); -} - -const char *raid_gen3_tag(void) -{ - return raid_tag((void_f*)raid_gen_ptr[2]); -} - -const char *raid_gen4_tag(void) -{ - return raid_tag((void_f*)raid_gen_ptr[3]); -} - -const char *raid_gen5_tag(void) -{ - return raid_tag((void_f*)raid_gen_ptr[4]); -} - -const char *raid_gen6_tag(void) -{ - return raid_tag((void_f*)raid_gen_ptr[5]); -} - -const char *raid_rec1_tag(void) -{ - return raid_tag((void_f*)raid_rec_ptr[0]); -} - -const char *raid_rec2_tag(void) -{ - return raid_tag((void_f*)raid_rec_ptr[1]); -} - -const char *raid_recX_tag(void) -{ - return raid_tag((void_f*)raid_rec_ptr[2]); -} - diff --git a/raid/test/Makefile b/raid/test/Makefile index 6423fd15..5d77fd90 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -24,7 +24,7 @@ CFLAGS += -O2 else CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 endif -OBJS = raid.o check.o int.o intz.o x86.o x86z.o tables.o memory.o test.o helper.o module.o tag.o +OBJS = raid.o check.o int.o intz.o x86.o x86z.o tables.o memory.o test.o helper.o module.o %.o: ../%.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/raid/test/invtest.c b/raid/test/invtest.c index c1bd561a..539ccaad 100644 --- a/raid/test/invtest.c +++ b/raid/test/invtest.c @@ -68,15 +68,15 @@ long long EXPECTED[RAID_PARITY_MAX] = { 327012476050LL }; -static __always_inline int test_sub_matrix(int nr, long long *total) +static __always_inline int test_sub_matrix(int nr, int64_t *total) { uint8_t M[RAID_PARITY_MAX * RAID_PARITY_MAX]; int np = RAID_PARITY_MAX; int nd = RAID_DATA_MAX; int ip[RAID_PARITY_MAX]; int id[RAID_DATA_MAX]; - long long count; - long long expected; + int64_t count; + int64_t expected; printf("\n%ux%u\n", nr, nr); @@ -119,7 +119,7 @@ static __always_inline int test_sub_matrix(int nr, long long *total) int test_all_sub_matrix(void) { - long long total; + int64_t total; printf("Invert all square submatrices of the %dx%d Cauchy matrix\n", RAID_PARITY_MAX, RAID_DATA_MAX); diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index add52d21..927c7ebe 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -143,7 +143,7 @@ void speed(void) /* GEN1 */ printf("%8s", "gen1"); - printf("%8s", raid_gen1_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR1)); fflush(stdout); printf("%8s", ""); @@ -196,7 +196,7 @@ void speed(void) /* GEN2 */ printf("%8s", "gen2"); - printf("%8s", raid_gen2_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2)); fflush(stdout); printf("%8s", ""); @@ -255,7 +255,7 @@ void speed(void) /* GENz */ printf("%8s", "genz"); - printf("%8s", raid_genz_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_VANDERMONDE_PAR3)); fflush(stdout); printf("%8s", ""); @@ -318,7 +318,7 @@ void speed(void) /* GEN3 */ printf("%8s", "gen3"); - printf("%8s", raid_gen3_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR3)); fflush(stdout); SPEED_START { @@ -381,7 +381,7 @@ void speed(void) /* GEN4 */ printf("%8s", "gen4"); - printf("%8s", raid_gen4_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR4)); fflush(stdout); SPEED_START { @@ -446,7 +446,7 @@ void speed(void) /* GEN5 */ printf("%8s", "gen5"); - printf("%8s", raid_gen5_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR5)); fflush(stdout); SPEED_START { @@ -511,7 +511,7 @@ void speed(void) /* GEN6 */ printf("%8s", "gen6"); - printf("%8s", raid_gen6_tag()); + printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR6)); fflush(stdout); SPEED_START { @@ -587,13 +587,13 @@ void speed(void) printf("\n"); printf("%8s", "rec1"); - printf("%8s", raid_rec1_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1)); fflush(stdout); SPEED_START { - for (j = 0; j < nd; ++j) - /* +1 to avoid GEN1 optimized case */ - raid_rec1_int8(1, id, ip + 1, nd, size, v); + raid_gen_force(1, sizeof(void *) == 8 ? raid_gen1_int64 : raid_gen1_int32); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_int8(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -603,9 +603,9 @@ void speed(void) #ifdef CONFIG_SSSE3 if (raid_cpu_has_ssse3()) { SPEED_START { - for (j = 0; j < nd; ++j) - /* +1 to avoid GEN1 optimized case */ - raid_rec1_ssse3(1, id, ip + 1, nd, size, v); + raid_gen_force(1, raid_gen1_sse2); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_ssse3(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -614,9 +614,9 @@ void speed(void) #ifdef CONFIG_AVX2 if (raid_cpu_has_avx2()) { SPEED_START { - for (j = 0; j < nd; ++j) - /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx2(1, id, ip + 1, nd, size, v); + raid_gen_force(1, raid_gen1_avx2); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_avx2(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -626,13 +626,13 @@ void speed(void) printf("\n"); printf("%8s", "rec2"); - printf("%8s", raid_rec2_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2)); fflush(stdout); SPEED_START { - for (j = 0; j < nd; ++j) - /* +1 to avoid GEN2 optimized case */ - raid_rec2_int8(2, id, ip + 1, nd, size, v); + raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64 : raid_gen2_int32); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_int8(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -642,9 +642,13 @@ void speed(void) #ifdef CONFIG_SSSE3 if (raid_cpu_has_ssse3()) { SPEED_START { - for (j = 0; j < nd; ++j) - /* +1 to avoid GEN2 optimized case */ - raid_rec2_ssse3(2, id, ip + 1, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(2, raid_gen2_sse2ext); +#else + raid_gen_force(2, raid_gen2_sse2); +#endif + /* +1 to avoid GEN2 optimized case */ + raid_rec2_ssse3(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -653,9 +657,9 @@ void speed(void) #ifdef CONFIG_AVX2 if (raid_cpu_has_avx2()) { SPEED_START { - for (j = 0; j < nd; ++j) - /* +1 to avoid GEN1 optimized case */ - raid_rec2_avx2(2, id, ip + 1, nd, size, v); + raid_gen_force(2, raid_gen2_avx2); + /* +1 to avoid GEN1 optimized case */ + raid_rec2_avx2(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -665,12 +669,12 @@ void speed(void) printf("\n"); printf("%8s", "rec3"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3)); fflush(stdout); SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_int8(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_int8); + raid_recX_int8(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -680,8 +684,12 @@ void speed(void) #ifdef CONFIG_SSSE3 if (raid_cpu_has_ssse3()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_ssse3(3, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(3, raid_gen3_ssse3ext); +#else + raid_gen_force(3, raid_gen3_ssse3); +#endif + raid_recX_ssse3(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -690,8 +698,8 @@ void speed(void) #ifdef CONFIG_AVX2 if (raid_cpu_has_avx2()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_avx2(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx2ext); + raid_recX_avx2(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -701,12 +709,12 @@ void speed(void) printf("\n"); printf("%8s", "rec4"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR4)); fflush(stdout); SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_int8(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_int8); + raid_recX_int8(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -716,8 +724,12 @@ void speed(void) #ifdef CONFIG_SSSE3 if (raid_cpu_has_ssse3()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_ssse3(4, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(4, raid_gen4_ssse3ext); +#else + raid_gen_force(4, raid_gen4_ssse3); +#endif + raid_recX_ssse3(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -726,8 +738,12 @@ void speed(void) #ifdef CONFIG_AVX2 if (raid_cpu_has_avx2()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_avx2(4, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(4, raid_gen4_avx2ext); +#else + raid_gen_force(4, raid_gen4_ssse3); +#endif + raid_recX_avx2(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -737,12 +753,12 @@ void speed(void) printf("\n"); printf("%8s", "rec5"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR5)); fflush(stdout); SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_int8(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_int8); + raid_recX_int8(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -752,8 +768,12 @@ void speed(void) #ifdef CONFIG_SSSE3 if (raid_cpu_has_ssse3()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_ssse3(5, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(5, raid_gen5_ssse3ext); +#else + raid_gen_force(5, raid_gen5_ssse3); +#endif + raid_recX_ssse3(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -762,8 +782,12 @@ void speed(void) #ifdef CONFIG_AVX2 if (raid_cpu_has_avx2()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_avx2(5, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(5, raid_gen5_avx2ext); +#else + raid_gen_force(5, raid_gen5_ssse3); +#endif + raid_recX_avx2(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -773,12 +797,12 @@ void speed(void) printf("\n"); printf("%8s", "rec6"); - printf("%8s", raid_recX_tag()); + printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR6)); fflush(stdout); SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_int8(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_int8); + raid_recX_int8(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -788,8 +812,12 @@ void speed(void) #ifdef CONFIG_SSSE3 if (raid_cpu_has_ssse3()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_ssse3(6, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(6, raid_gen6_ssse3ext); +#else + raid_gen_force(6, raid_gen6_ssse3); +#endif + raid_recX_ssse3(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -798,8 +826,12 @@ void speed(void) #ifdef CONFIG_AVX2 if (raid_cpu_has_avx2()) { SPEED_START { - for (j = 0; j < nd; ++j) - raid_recX_avx2(6, id, ip, nd, size, v); +#ifdef CONFIG_X86_64 + raid_gen_force(6, raid_gen6_avx2ext); +#else + raid_gen_force(6, raid_gen6_ssse3); +#endif + raid_recX_avx2(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); diff --git a/raid/x86.c b/raid/x86.c index 6749adc5..5bce4f4c 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* - */ +// Copyright (C) 2013 Andrea Mazzoleni #include "internal.h" #include "gf.h" +#include "cpu.h" /* * For x86 optimizations you can see: @@ -2440,3 +2440,67 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) } #endif +#if defined(CONFIG_X86) && defined(CONFIG_SSE2) +void raid_register_x86(void) +{ +#ifdef CONFIG_SSE2 + if (raid_cpu_has_sse2()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2); +#ifdef CONFIG_X86_64 + if (!raid_cpu_has_slowextendedreg()) + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext); + /* note that raid_cpu_has_slowextendedreg() doesn't affect vandermonde */ + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext); +#else + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2); +#endif + } +#endif + +#ifdef CONFIG_SSSE3 + if (raid_cpu_has_ssse3()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3); +#ifdef CONFIG_X86_64 + if (!raid_cpu_has_slowextendedreg()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext); + } +#endif + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3); + } +#endif + +#ifdef CONFIG_AVX2 + if (raid_cpu_has_avx2()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2); +#ifdef CONFIG_X86_64 + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext); + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext); +#endif + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2); + } +#endif +} +#endif