rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support dynamic runtime selection of RAID AES mode

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-07-29 12:47:13 +02:00
parent 1b1ae7cce9
commit 47b321d97c
24 changed files with 16502 additions and 16479 deletions
Download patch file Download diff file Expand all files Collapse all files

114
raid/avx2.c
View file

@ -100,7 +100,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
*/
void raid_genz_avx2ext(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t**)vv;
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
uint8_t *r;
@ -219,8 +219,8 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
@ -256,8 +256,8 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
@ -352,8 +352,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm5,%ymm5");
asm volatile ("vpand %ymm15,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");
@ -361,8 +361,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm11,%ymm3");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm11,%ymm11");
@ -400,8 +400,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm5,%ymm5");
asm volatile ("vpand %ymm15,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
@ -411,8 +411,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
@ -501,32 +501,32 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
if (np >= 3) {
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vpshufb %ymm10,%ymm2,%ymm2");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
}
if (np >= 4) {
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm10,%ymm3,%ymm3");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
}
if (np >= 5) {
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("vpshufb %ymm10,%ymm4,%ymm4");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
}
if (np >= 6) {
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
asm volatile ("vpshufb %ymm10,%ymm5,%ymm5");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
@ -549,8 +549,8 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
if (np >= 3) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
@ -558,8 +558,8 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
}
if (np >= 4) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm3,%ymm3");
@ -567,8 +567,8 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
}
if (np >= 5) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
@ -576,8 +576,8 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
}
if (np >= 6) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm5,%ymm5");
@ -679,8 +679,8 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
raid_avx_begin();
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfmulpshufb[V][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfmulpshufb[V][1][0]));
for (i = 0; i < size; i += 32) {
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
@ -744,8 +744,8 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[0]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[0]][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
asm volatile ("vpsrlw $4,%ymm0,%ymm5");
asm volatile ("vpand %ymm7,%ymm0,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
@ -754,8 +754,8 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm2,%ymm6,%ymm6");
asm volatile ("vpxor %ymm3,%ymm6,%ymm6");
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[1]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[1]][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
asm volatile ("vpsrlw $4,%ymm1,%ymm5");
asm volatile ("vpand %ymm7,%ymm1,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
@ -768,8 +768,8 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[2]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[2]][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
asm volatile ("vpsrlw $4,%ymm0,%ymm5");
asm volatile ("vpand %ymm7,%ymm0,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
@ -778,8 +778,8 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm2,%ymm6,%ymm6");
asm volatile ("vpxor %ymm3,%ymm6,%ymm6");
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[3]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[3]][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
asm volatile ("vpsrlw $4,%ymm1,%ymm5");
asm volatile ("vpand %ymm7,%ymm1,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
@ -805,7 +805,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t buffer[RAID_PARITY_MAX*32+32];
uint8_t buffer[RAID_PARITY_MAX * 32 + 32];
uint8_t *pd = __align_ptr(buffer, 32);
size_t i;
int j, k;
@ -836,7 +836,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i]));
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[j][i]));
asm volatile ("vpxor %ymm1,%ymm0,%ymm0");
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j*32]));
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 32]));
}
/* reconstruct */
@ -847,9 +847,9 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
for (k = 0; k < N; ++k) {
uint8_t m = V[j * N + k];
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[m][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[m][1][0]));
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k*32]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[m][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[m][1][0]));
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 32]));
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
@ -870,22 +870,22 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_avx2(void)
{
if (raid_cpu_has_avx2()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2);
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2, RAID_POLY_ANY);
#ifdef CONFIG_X86_64
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext, RAID_POLY_ANY);
#endif
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2, RAID_POLY_ANY);
}
}
#endif

78
raid/avx512.c
View file

@ -16,7 +16,7 @@
*/
void raid_gen1_avx512bw(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t**)vv;
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
int d, l;
size_t i;
@ -34,7 +34,7 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv)
"vmovdqa64 %0,%%zmm1\n\t"
"vpternlogq $0x96,%1,%%zmm1,%%zmm0"
:
: "m" (v[d][i]), "m" (v[d+1][i])
: "m" (v[d][i]), "m" (v[d + 1][i])
);
}
@ -88,23 +88,23 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
asm volatile ("vpandq %zmm31,%zmm10,%zmm10");
asm volatile ("vpandq %zmm31,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[d][0][1][0]));
if (np >= 3) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
}
if (np >= 4) {
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
}
if (np >= 5) {
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
}
if (np >= 6) {
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
}
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
@ -220,8 +220,8 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
raid_avx_begin();
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfmulpshufb[V][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
@ -279,10 +279,10 @@ void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
asm volatile ("vpxord %zmm6,%zmm6,%zmm6");
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[0]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[0]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[1]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[1]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
asm volatile ("vpsrlw $4,%zmm0,%zmm5");
asm volatile ("vpsrlw $4,%zmm1,%zmm13");
asm volatile ("vpandd %zmm7,%zmm0,%zmm4");
@ -302,10 +302,10 @@ void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
asm volatile ("vpxord %zmm6,%zmm6,%zmm6");
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[2]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[2]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[3]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[3]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
asm volatile ("vpsrlw $4,%zmm0,%zmm5");
asm volatile ("vpsrlw $4,%zmm1,%zmm13");
asm volatile ("vpandd %zmm7,%zmm0,%zmm4");
@ -338,7 +338,7 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t buffer[RAID_PARITY_MAX*64+64];
uint8_t buffer[RAID_PARITY_MAX * 64 + 64];
uint8_t *pd = __align_ptr(buffer, 64);
size_t i;
int j, k;
@ -369,7 +369,7 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i]));
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i]));
asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64]));
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64]));
}
/* reconstruct */
for (j = 0; j < N; ++j) {
@ -377,9 +377,9 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
asm volatile ("vpxorq %zmm1,%zmm1,%zmm1");
for (k = 0; k < N; ++k) {
uint8_t m = V[j * N + k];
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[m][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[m][1][0]));
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (raid_gfmulpshufb[m][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfmulpshufb[m][1][0]));
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64]));
asm volatile ("vpsrlw $4,%zmm4,%zmm5");
asm volatile ("vpandq %zmm7,%zmm4,%zmm4");
asm volatile ("vpandq %zmm7,%zmm5,%zmm5");
@ -400,19 +400,19 @@ void raid_register_avx512(void)
{
if (raid_cpu_has_avx512bw()) {
if (!raid_cpu_has_slow_avx512()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_gen1_avx512bw);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_gen2_avx512bw);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw);
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_gen1_avx512bw, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_gen2_avx512bw, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
}
}
}

4
raid/check.c
View file

@ -76,7 +76,7 @@ static int raid_validate(int nr, int *id, int nv, int *ip, int nd, size_t size,
b = v[j][i];
for (l = 0; l < nv; ++l)
p[l] ^= gfmul[b][gfgen[ip[l]][j]];
p[l] ^= raid_gfmul[b][raid_gfgen[ip[l]][j]];
}
/* reconstruct data */
@ -90,7 +90,7 @@ static int raid_validate(int nr, int *id, int nv, int *ip, int nd, size_t size,
/* add the parity contribution of the reconstructed data */
for (l = nr; l < nv; ++l)
p[l] ^= gfmul[b][gfgen[ip[l]][idj]];
p[l] ^= raid_gfmul[b][raid_gfgen[ip[l]][idj]];
}
/* check that the final parity is 0 */

3
raid/combo.h
View file

@ -51,7 +51,6 @@ recurse:
/* if the position has reached the max */
if (c[i] >= n) {
/* if we are at the first level, we have finished */
if (i == 0)
return 0;
@ -118,7 +117,6 @@ recurse:
/* if the position has reached the max */
if (c[i] >= h) {
/* if we are at the first level, we have finished */
if (i == 0)
return 0;
@ -141,4 +139,3 @@ recurse:
return 1;
}
#endif

110
raid/cpu.h
View file

@ -74,25 +74,25 @@ static inline void raid_cpu_info(char *vendor, unsigned *family, unsigned *model
}
}
static inline int raid_cpu_match_sse(uint32_t cpuid_1_ecx, uint32_t cpuid_1_edx)
static inline int raid_cpu_match_sse(uint32_t leaf_1_ecx, uint32_t leaf_1_edx)
{
uint32_t reg[4];
raid_cpuid(1, 0, reg);
if ((reg[2] & cpuid_1_ecx) != cpuid_1_ecx)
if ((reg[2] & leaf_1_ecx) != leaf_1_ecx)
return 0;
if ((reg[3] & cpuid_1_edx) != cpuid_1_edx)
if ((reg[3] & leaf_1_edx) != leaf_1_edx)
return 0;
return 1;
}
static inline int raid_cpu_match_avx(uint32_t cpuid_1_ecx, uint32_t cpuid_7_ebx, uint32_t cpuid_7_ecx, uint32_t xcr0)
static inline int raid_cpu_match_avx(uint32_t leaf_1_ecx, uint32_t leaf_7_ebx, uint32_t leaf_7_ecx, uint32_t xcr0)
{
uint32_t reg[4];
raid_cpuid(1, 0, reg);
if ((reg[2] & cpuid_1_ecx) != cpuid_1_ecx)
if ((reg[2] & leaf_1_ecx) != leaf_1_ecx)
return 0;
raid_xgetbv(reg);
@ -100,10 +100,10 @@ static inline int raid_cpu_match_avx(uint32_t cpuid_1_ecx, uint32_t cpuid_7_ebx,
return 0;
raid_cpuid(7, 0, reg);
if ((reg[1] & cpuid_7_ebx) != cpuid_7_ebx)
if ((reg[1] & leaf_7_ebx) != leaf_7_ebx)
return 0;
if ((reg[2] & cpuid_7_ecx) != cpuid_7_ecx)
if ((reg[2] & leaf_7_ecx) != leaf_7_ecx)
return 0;
return 1;
@ -123,9 +123,9 @@ static inline int raid_cpu_has_sse2(void)
* 2. Check that the processor supports the SSE and/or SSE2 extensions (true if
* CPUID.01H:EDX.SSE[bit 25] = 1 and/or CPUID.01H:EDX.SSE2[bit 26] = 1).
*/
return raid_cpu_match_sse(
0,
1 << 26); /* SSE2 */
uint32_t leaf_1_edx = 1 << 26; /* SSE2 */
return raid_cpu_match_sse(0, leaf_1_edx);
}
static inline int raid_cpu_has_ssse3(void)
@ -140,9 +140,10 @@ static inline int raid_cpu_has_ssse3(void)
* Next, use the additional step provided below:
* Check that the processor supports SSSE3 (if CPUID.01H:ECX.SSSE3[bit 9] = 1).
*/
return raid_cpu_match_sse(
1 << 9, /* SSSE3 */
1 << 26); /* SSE2 */
uint32_t leaf_1_ecx = 1 << 9; /* SSSE3 */
uint32_t leaf_1_edx = 1 << 26; /* SSE2 */
return raid_cpu_match_sse(leaf_1_ecx, leaf_1_edx);
}
static inline int raid_cpu_has_crc32(void)
@ -156,9 +157,9 @@ static inline int raid_cpu_has_crc32(void)
* Before an application attempts to use the CRC32 instruction, it must check
* that the processor supports SSE4.2 (if CPUID.01H:ECX.SSE4_2[bit 20] = 1).
*/
return raid_cpu_match_sse(
1 << 20, /* CRC32 */
0);
uint32_t leaf_1_ecx = 1 << 20; /* CRC32 */
return raid_cpu_match_sse(leaf_1_ecx, 0);
}
static inline int raid_cpu_has_avx2(void)
@ -178,11 +179,11 @@ static inline int raid_cpu_has_avx2(void)
* Application Software must identify that hardware supports AVX, after that it must
* also detect support for AVX2 by checking CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5].
*/
return raid_cpu_match_avx(
(1 << 27) | (1 << 28), /* Leaf 1, ECX: XSAVE and AVX */
1 << 5, /* Leaf 7, EBX: AVX2 */
0, /* Leaf 7, ECX: */
3 << 1); /* XCR0: OS saves XMM and YMM registers */
uint32_t leaf_1_ecx = (1 << 27) | (1 << 28); /* XSAVE and AVX */
uint32_t leaf_7_ebx = 1 << 5; /* AVX2 */
uint32_t xcr0 = 3 << 1; /* OS saves XMM and YMM registers */
return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, 0, xcr0);
}
static inline int raid_cpu_has_avx2gfni(void)
@ -194,11 +195,12 @@ static inline int raid_cpu_has_avx2gfni(void)
* 3) Verify AVX2 support (CPUID.7.0:EBX[5])
* 4) Verify GFNI support (CPUID.7.0:ECX[8])
*/
return raid_cpu_match_avx(
(1 << 27) | (1 << 28), /* Leaf 1, ECX: XSAVE and AVX */
1 << 5, /* Leaf 7, EBX: AVX2 */
1 << 8, /* Leaf 7, ECX: GFNI */
3 << 1); /* XCR0: OS saves XMM and YMM registers */
uint32_t leaf_1_ecx = (1 << 27) | (1 << 28); /* XSAVE and AVX */
uint32_t leaf_7_ebx = 1 << 5; /* AVX2 */
uint32_t leaf_7_ecx = 1 << 8; /* GFNI */
uint32_t xcr0 = 3 << 1; /* OS saves XMM and YMM registers */
return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, leaf_7_ecx, xcr0);
}
static inline int raid_cpu_has_avx512bw(void)
@ -219,21 +221,21 @@ static inline int raid_cpu_has_avx512bw(void)
* Note that intentionally we don't check for AVX and AVX2
* because the documentation doesn't require that
*/
return raid_cpu_match_avx(
1 << 27, /* Leaf 1, ECX: XSAVE/XGETBV */
(1 << 16) | (1 << 30), /* Leaf 7, EBX: AVX512F and AVX512BW */
0, /* Leaf 7, ECX: */
(3 << 1) | (7 << 5)); /* XCR0: OS saves XMM, YMM and ZMM registers */
uint32_t leaf_1_ecx = 1 << 27; /* XSAVE/XGETBV */
uint32_t leaf_7_ebx = (1 << 16) | (1 << 30); /* AVX512F and AVX512BW */
uint32_t xcr0 = (3 << 1) | (7 << 5); /* OS saves XMM, YMM and ZMM registers */
return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, 0, xcr0);
}
static inline int raid_cpu_has_avx512gfni(void)
{
return raid_cpu_match_avx(
1 << 27, /* Leaf 1, ECX: XSAVE/XGETBV */
1 << 16, /* Leaf 7, EBX: AVX512F (Foundation) */
1 << 8, /* Leaf 7, ECX: GFNI */
(3 << 1) | (7 << 5) /* XCR0: OS saves XMM, YMM and ZMM registers */
);
uint32_t leaf_1_ecx = 1 << 27; /* XSAVE/XGETBV */
uint32_t leaf_7_ebx = 1 << 16; /* AVX512F (Foundation) */
uint32_t leaf_7_ecx = 1 << 8; /* GFNI */
uint32_t xcr0 = (3 << 1) | (7 << 5); /* OS saves XMM, YMM and ZMM registers */
return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, leaf_7_ecx, xcr0);
}
/**
@ -247,25 +249,23 @@ static inline int raid_cpu_is_atom(unsigned family, unsigned model)
/*
* x86 Architecture CPUID
* http://www.sandpile.org/x86/cpuid.htm
*
* Intel Atom
* 1C (28) Atom (45 nm) with 512 KB on-die L2
* 26 (38) Atom (45 nm) with 512 KB on-die L2
* 36 (54) Atom (32 nm) with 512 KB on-die L2
* 27 (39) Atom (32 nm) with 512 KB on-die L2
* 35 (53) Atom (?? nm) with ??? KB on-die L2
* 4A (74) Atom 2C (22 nm) 1 MB L2 + PowerVR (TGR)
* 5A (90) Atom 4C (22 nm) 2 MB L2 + PowerVR (ANN)
* 37 (55) Atom 4C (22 nm) 2 MB L2 + Intel Gen7 (BYT)
* 4C (76) Atom 4C (14 nm) 2 MB L2 + Intel Gen8 (BSW)
* 5D (93) Atom 4C (28 nm TSMC) 1 MB L2 + Mali (SoFIA)
* 4D (77) Atom 8C (22 nm) 4 MB L2 (AVN)
* ?? Atom ?C (14 nm) ? MB L2 (DVN)
*/
return model == 28 || model == 38 || model == 54
|| model == 39 || model == 53 || model == 74
|| model == 90 || model == 55 || model == 76
|| model == 93 || model == 77;
switch (model) {
case 28: /* 1C Atom (45 nm) */
case 38: /* 26 Atom (45 nm) */
case 39: /* 27 Atom (32 nm) */
case 53: /* 35 Atom */
case 54: /* 36 Atom (32 nm) */
case 55: /* 37 Atom 4C (22 nm BYT) */
case 74: /* 4A Atom 2C (22 nm TGR) */
case 76: /* 4C Atom 4C (14 nm BSW) */
case 77: /* 4D Atom 8C (22 nm AVN) */
case 90: /* 5A Atom 4C (22 nm ANN) */
case 93: /* 5D Atom 4C (28 nm SoFIA) */
return 1;
default:
return 0;
}
}
/**

58
raid/gf.h
View file

@ -15,7 +15,7 @@
*/
static __always_inline uint8_t mul(uint8_t a, uint8_t b)
{
return gfmul[a][b];
return raid_gfmul[a][b];
}
/*
@ -26,7 +26,7 @@ static __always_inline uint8_t inv(uint8_t v)
{
BUG_ON(v == 0); /* division by zero */
return gfinv[v];
return raid_gfinv[v];
}
/*
@ -36,7 +36,7 @@ static __always_inline uint8_t pow2(int v)
{
BUG_ON(v < 0 || v > 254); /* invalid exponent */
return gfexp[v];
return raid_gfexp[v];
}
/*
@ -44,7 +44,7 @@ static __always_inline uint8_t pow2(int v)
*/
static __always_inline const uint8_t *table(uint8_t v)
{
return gfmul[v];
return raid_gfmul[v];
}
/*
@ -52,7 +52,7 @@ static __always_inline const uint8_t *table(uint8_t v)
*/
static __always_inline uint8_t A(int p, int d)
{
return gfgen[p][d];
return raid_gfgen[p][d];
}
/*
@ -70,23 +70,34 @@ static __always_inline uint8_t A(int p, int d)
*/
#define v_64(p) (*(uint64_t *)&(p))
/*
* Galois field reduction polynomials.
*
* RAID_POLY_ANY - Wildcard matching algorithms compatible with any polynomial.
* RAID_POLY_RAID - Standard RAID polynomial 0x1d (x^8 + x^4 + x^3 + x^2 + 1).
* RAID_POLY_AES - AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1) for GFNI acceleration.
*/
#define RAID_POLY_ANY 0
#define RAID_POLY_RAID 0x1d
#define RAID_POLY_AES 0x1b
/*
* Polynomial-dependent XOR masks for GF(2^8) multiply and divide by 2.
*
* RAID_POLY_32 and RAID_POLY_64 are RAID_POLY repeated in every byte
* raid_poly_32 and raid_poly_64 are RAID_POLY repeated in every byte
* of a uint32/uint64, used as the conditional XOR in the multiply-by-2
* path: when the MSB of a byte is set before the left shift, the
* reducing polynomial is XORed in to reduce the result back into the
* field.
*
* RAID_INV2_32 and RAID_INV2_64 are the GF(2^8) inverse of 2 (i.e.
* raid_inv2_32 and raid_inv2_64 are the GF(2^8) inverse of 2 (i.e.
* 2^{-1} mod RAID_POLY) repeated in every byte, used in the
* divide-by-2 path. The inverse depends on the polynomial:
*
* RAID_POLY = 0x1d -> 2^{-1} = 0x8e (standard RAID)
* RAID_POLY = 0x1b -> 2^{-1} = 0x8d (AES polynomial)
*
* Both are computed from RAID_POLY via the macro RAID_INV2_BYTE:
* Both RAID_INV2_RAID and RAID_INV2_AES are computed from their respective RAID_POLY:
* in GF(2^8), 2^{-1} is the value x such that 2*x = 1, which for
* any polynomial of the form x^8 + ... + 1 resolves to:
*
@ -95,62 +106,67 @@ static __always_inline uint8_t A(int p, int d)
* since right-shifting the polynomial by 1 and setting the MSB gives
* the unique solution in both cases.
*/
#define RAID_INV2_BYTE (((RAID_POLY) >> 1) | 0x80)
#define RAID_INV2_RAID (((RAID_POLY_RAID) >> 1) | 0x80)
#define RAID_POLY_32_RAID ((uint32_t)RAID_POLY_RAID * 0x01010101U)
#define RAID_POLY_64_RAID ((uint64_t)RAID_POLY_RAID * 0x0101010101010101ULL)
#define RAID_INV2_32_RAID ((uint32_t)RAID_INV2_RAID * 0x01010101U)
#define RAID_INV2_64_RAID ((uint64_t)RAID_INV2_RAID * 0x0101010101010101ULL)
#define RAID_POLY_32 ((uint32_t)RAID_POLY * 0x01010101U)
#define RAID_POLY_64 ((uint64_t)RAID_POLY * 0x0101010101010101ULL)
#define RAID_INV2_32 ((uint32_t)RAID_INV2_BYTE * 0x01010101U)
#define RAID_INV2_64 ((uint64_t)RAID_INV2_BYTE * 0x0101010101010101ULL)
#define RAID_INV2_AES (((RAID_POLY_AES) >> 1) | 0x80)
#define RAID_POLY_32_AES ((uint32_t)RAID_POLY_AES * 0x01010101U)
#define RAID_POLY_64_AES ((uint64_t)RAID_POLY_AES * 0x0101010101010101ULL)
#define RAID_INV2_32_AES ((uint32_t)RAID_INV2_AES * 0x01010101U)
#define RAID_INV2_64_AES ((uint64_t)RAID_INV2_AES * 0x0101010101010101ULL)
/*
* Multiply each byte of a uint32 by 2 in GF(2^8).
*/
static __always_inline uint32_t x2_32(uint32_t v)
static __always_inline uint32_t x2_32(uint32_t v, uint32_t poly_32)
{
uint32_t mask = v & 0x80808080U;
mask = (mask << 1) - (mask >> 7);
v = (v << 1) & 0xfefefefeU;
v ^= mask & RAID_POLY_32;
v ^= mask & poly_32;
return v;
}
/*
* Multiply each byte of a uint64 by 2 in GF(2^8).
*/
static __always_inline uint64_t x2_64(uint64_t v)
static __always_inline uint64_t x2_64(uint64_t v, uint64_t poly_64)
{
uint64_t mask = v & 0x8080808080808080ULL;
mask = (mask << 1) - (mask >> 7);
v = (v << 1) & 0xfefefefefefefefeULL;
v ^= mask & RAID_POLY_64;
v ^= mask & poly_64;
return v;
}
/*
* Divide each byte of a uint32 by 2 in GF(2^8).
*/
static __always_inline uint32_t d2_32(uint32_t v)
static __always_inline uint32_t d2_32(uint32_t v, uint32_t inv2_32)
{
uint32_t mask = v & 0x01010101U;
mask = (mask << 8) - mask;
v = (v >> 1) & 0x7f7f7f7fU;
v ^= mask & RAID_INV2_32;
v ^= mask & inv2_32;
return v;
}
/*
* Divide each byte of a uint64 by 2 in GF(2^8).
*/
static __always_inline uint64_t d2_64(uint64_t v)
static __always_inline uint64_t d2_64(uint64_t v, uint64_t inv2_64)
{
uint64_t mask = v & 0x0101010101010101ULL;
mask = (mask << 8) - mask;
v = (v >> 1) & 0x7f7f7f7f7f7f7f7fULL;
v ^= mask & RAID_INV2_64;
v ^= mask & inv2_64;
return v;
}

72
raid/gfni.c
View file

@ -56,34 +56,34 @@ static __always_inline void raid_genX_avx2gfni(int nd, size_t size, void **vv, i
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[1][d]));
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (raid_gfcauchy[1][d]));
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
if (np >= 3) {
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][d]));
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (raid_gfcauchy[2][d]));
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
}
if (np >= 4) {
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][d]));
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (raid_gfcauchy[3][d]));
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
}
if (np >= 5) {
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][d]));
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (raid_gfcauchy[4][d]));
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
}
if (np >= 6) {
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][d]));
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (raid_gfcauchy[5][d]));
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
@ -151,15 +151,15 @@ static __always_inline void raid_genX_avx512gfni(int nd, size_t size, void **vv,
for (d = 1; d < nd; ++d) {
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d]));
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (raid_gfcauchy[1][d]));
if (np >= 3)
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d]));
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (raid_gfcauchy[2][d]));
if (np >= 4)
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d]));
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (raid_gfcauchy[3][d]));
if (np >= 5)
asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d]));
asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (raid_gfcauchy[4][d]));
if (np >= 6)
asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][d]));
asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (raid_gfcauchy[5][d]));
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12");
if (np >= 3)
@ -519,7 +519,7 @@ void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t buffer[RAID_PARITY_MAX*64+64];
uint8_t buffer[RAID_PARITY_MAX * 64 + 64];
uint8_t *pd = __align_ptr(buffer, 64);
size_t i;
int j, k;
@ -589,7 +589,7 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t buffer[RAID_PARITY_MAX*64+64];
uint8_t buffer[RAID_PARITY_MAX * 64 + 64];
uint8_t *pd = __align_ptr(buffer, 64);
size_t i;
int j, k;
@ -635,43 +635,39 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
void raid_register_avx2gfni(void)
{
#ifdef USE_RAID_AES
if (raid_cpu_has_avx2gfni()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
}
#endif
}
void raid_register_avx512gfni(void)
{
#ifdef USE_RAID_AES
if (raid_cpu_has_avx512gfni()) {
if (!raid_cpu_has_slow_avx512()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
}
}
#endif
}
#endif

1
raid/helper.h
View file

@ -29,4 +29,3 @@ void raid_insert(int n, int *v, int i);
void raid_sort(int n, int *v);
#endif

94
raid/int.c
View file

@ -70,6 +70,7 @@ void raid_gen2_int32(int nd, size_t size, void **vv)
uint8_t *q;
int d, l;
size_t i;
const uint32_t poly_32 = raid_poly_32;
uint32_t d0, q0, p0;
uint32_t d1, q1, p1;
@ -88,8 +89,8 @@ void raid_gen2_int32(int nd, size_t size, void **vv)
p0 ^= d0;
p1 ^= d1;
q0 = x2_32(q0);
q1 = x2_32(q1);
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
q0 ^= d0;
q1 ^= d1;
@ -111,6 +112,7 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
uint8_t *q;
int d, l;
size_t i;
const uint64_t poly_64 = raid_poly_64;
uint64_t d0, q0, p0;
uint64_t d1, q1, p1;
@ -129,8 +131,8 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
p0 ^= d0;
p1 ^= d1;
q0 = x2_64(q0);
q1 = x2_64(q1);
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
q0 ^= d0;
q1 ^= d1;
@ -147,12 +149,14 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
*/
void raid_genz_int32(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t**)vv;
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
uint8_t *r;
int d, l;
size_t i;
const uint32_t poly_32 = raid_poly_32;
const uint32_t inv2_32 = raid_inv2_32;
uint32_t d0, r0, q0, p0;
uint32_t d1, r1, q1, p1;
@ -172,14 +176,14 @@ void raid_genz_int32(int nd, size_t size, void **vv)
p0 ^= d0;
p1 ^= d1;
q0 = x2_32(q0);
q1 = x2_32(q1);
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
q0 ^= d0;
q1 ^= d1;
r0 = d2_32(r0);
r1 = d2_32(r1);
r0 = d2_32(r0, inv2_32);
r1 = d2_32(r1, inv2_32);
r0 ^= d0;
r1 ^= d1;
@ -198,12 +202,14 @@ void raid_genz_int32(int nd, size_t size, void **vv)
*/
void raid_genz_int64(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t**)vv;
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
uint8_t *r;
int d, l;
size_t i;
const uint64_t poly_64 = raid_poly_64;
const uint64_t inv2_64 = raid_inv2_64;
uint64_t d0, r0, q0, p0;
uint64_t d1, r1, q1, p1;
@ -223,14 +229,14 @@ void raid_genz_int64(int nd, size_t size, void **vv)
p0 ^= d0;
p1 ^= d1;
q0 = x2_64(q0);
q1 = x2_64(q1);
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
q0 ^= d0;
q1 ^= d1;
r0 = d2_64(r0);
r1 = d2_64(r1);
r0 = d2_64(r0, inv2_64);
r1 = d2_64(r1, inv2_64);
r0 ^= d0;
r1 ^= d1;
@ -274,8 +280,8 @@ void raid_gen3_int8(int nd, size_t size, void **vv)
d0 = v_8(v[d][i]);
p0 ^= d0;
q0 ^= gfmul[d0][gfgen[1][d]];
r0 ^= gfmul[d0][gfgen[2][d]];
q0 ^= raid_gfmul[d0][raid_gfgen[1][d]];
r0 ^= raid_gfmul[d0][raid_gfgen[2][d]];
}
/* first disk with all coefficients at 1 */
@ -323,9 +329,9 @@ void raid_gen4_int8(int nd, size_t size, void **vv)
d0 = v_8(v[d][i]);
p0 ^= d0;
q0 ^= gfmul[d0][gfgen[1][d]];
r0 ^= gfmul[d0][gfgen[2][d]];
s0 ^= gfmul[d0][gfgen[3][d]];
q0 ^= raid_gfmul[d0][raid_gfgen[1][d]];
r0 ^= raid_gfmul[d0][raid_gfgen[2][d]];
s0 ^= raid_gfmul[d0][raid_gfgen[3][d]];
}
/* first disk with all coefficients at 1 */
@ -377,10 +383,10 @@ void raid_gen5_int8(int nd, size_t size, void **vv)
d0 = v_8(v[d][i]);
p0 ^= d0;
q0 ^= gfmul[d0][gfgen[1][d]];
r0 ^= gfmul[d0][gfgen[2][d]];
s0 ^= gfmul[d0][gfgen[3][d]];
t0 ^= gfmul[d0][gfgen[4][d]];
q0 ^= raid_gfmul[d0][raid_gfgen[1][d]];
r0 ^= raid_gfmul[d0][raid_gfgen[2][d]];
s0 ^= raid_gfmul[d0][raid_gfgen[3][d]];
t0 ^= raid_gfmul[d0][raid_gfgen[4][d]];
}
/* first disk with all coefficients at 1 */
@ -436,11 +442,11 @@ void raid_gen6_int8(int nd, size_t size, void **vv)
d0 = v_8(v[d][i]);
p0 ^= d0;
q0 ^= gfmul[d0][gfgen[1][d]];
r0 ^= gfmul[d0][gfgen[2][d]];
s0 ^= gfmul[d0][gfgen[3][d]];
t0 ^= gfmul[d0][gfgen[4][d]];
u0 ^= gfmul[d0][gfgen[5][d]];
q0 ^= raid_gfmul[d0][raid_gfgen[1][d]];
r0 ^= raid_gfmul[d0][raid_gfgen[2][d]];
s0 ^= raid_gfmul[d0][raid_gfgen[3][d]];
t0 ^= raid_gfmul[d0][raid_gfgen[4][d]];
u0 ^= raid_gfmul[d0][raid_gfgen[5][d]];
}
/* first disk with all coefficients at 1 */
@ -648,23 +654,23 @@ void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_int(void)
{
if (sizeof(void *) == 4) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int32", raid_gen1_int32);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int32", raid_genz_int32);
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int32", raid_gen1_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int32", raid_genz_int32, RAID_POLY_ANY);
} else {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int64", raid_gen1_int64);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64);
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int64", raid_gen1_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64, RAID_POLY_ANY);
}
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_gen6_int8);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_gen6_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "int8", raid_rec1_int8);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_rec2_int8);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_recX_int8);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_recX_int8);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_recX_int8);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_recX_int8);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "int8", raid_rec1_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_rec2_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_recX_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_recX_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_recX_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_recX_int8, RAID_POLY_ANY);
}

78
raid/internal.h
View file

@ -30,7 +30,6 @@
#endif
#endif
/*
* Includes anything required for compatibility.
*/
@ -61,7 +60,7 @@
/*
* Align a pointer at the specified size.
*/
static __always_inline void *__align_ptr(void *ptr, uintptr_t size)
static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
{
uintptr_t offset = (uintptr_t)ptr;
@ -174,10 +173,10 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
* This allows to specify the same memory buffer for multiple parities
* knowing that you'll get the latest written one.
* This characteristic is used by the raid_delta_gen() function to
* avoid to damage unused parities in recovering.
* avoid damaging unused parities during recovery.
*
* @nd Number of data blocks
* @size Size of the blocks pointed by @vv. It must be a multiple of 64.
* @size Size of the blocks pointed to by @vv. It must be a multiple of 64.
* @vv Vector of pointers to the blocks of data and parity.
* It has (@nd + #parities) elements. The starting elements are the blocks
* for data, following with the parity blocks.
@ -233,9 +232,10 @@ typedef void (raid_rec_fn)(int nr, int *id, int *ip, int nd, size_t size, void *
* @na Algo code of the function. One of RAID_ALGO_*.
* @tag Descriptive short tag of the implementation, like "sse2", "avx2",...
* @fn Function to register.
* @poly Polynomial choice (RAID_POLY_ANY, RAID_POLY_RAID, RAID_POLY_AES).
*/
void raid_gen_register(int na, const char *tag, raid_gen_fn *fn);
void raid_rec_register(int na, const char *tag, raid_rec_fn *fn);
void raid_gen_register(int na, const char *tag, raid_gen_fn *fn, uint8_t poly);
void raid_rec_register(int na, const char *tag, raid_rec_fn *fn, uint8_t poly);
/**
* Set functions for data recovery.
@ -256,7 +256,7 @@ void raid_gen_force(int np, raid_gen_fn *fn);
void raid_register_int(void);
/**
* Register all the functions based on x86 intructions.
* Register all the functions based on x86 instructions.
*/
void raid_register_sse2(void);
void raid_register_ssse3(void);
@ -264,7 +264,7 @@ void raid_register_avx2(void);
void raid_register_avx512(void);
/**
* Register all the functions based on GFNI intructions.
* Register all the functions based on GFNI instructions.
*/
void raid_register_avx2gfni(void);
void raid_register_avx512gfni(void);
@ -274,8 +274,8 @@ void raid_register_avx512gfni(void);
*
* Given the specified algo code, return the tag of the registered function.
*/
const char *raid_gen_tag(int na);
const char *raid_rec_tag(int na);
const char * raid_gen_tag(int na);
const char * raid_rec_tag(int na);
/**
* Basic functionality self test.
@ -291,30 +291,50 @@ struct gfconst16 {
uint8_t half[16];
uint8_t low7[16];
};
extern const struct gfconst16 gfconst16;
extern struct gfconst16 gfconst16 __aligned(16);
#endif
/*
* Tables.
*/
extern const uint8_t raid_gfmul[256][256] __aligned(256);
extern const uint8_t raid_gfexp[256] __aligned(256);
extern const uint8_t raid_gfinv[256] __aligned(256);
extern const uint8_t raid_gfvandermonde[3][256] __aligned(256);
extern const uint8_t raid_gfcauchy[6][256] __aligned(256);
extern const uint8_t raid_gfcauchypshufb[251][5][2][16] __aligned(256);
extern const uint8_t raid_gfmulpshufb[256][2][16] __aligned(256);
extern const uint8_t raid_gfcauchycoeff[251][4][16] __aligned(256);
extern const uint8_t (*raid_gfgen)[256];
#define gfmul raid_gfmul
#define gfexp raid_gfexp
#define gfinv raid_gfinv
#define gfvandermonde raid_gfvandermonde
#define gfcauchy raid_gfcauchy
#define gfgenpshufb raid_gfcauchypshufb
#define gfmulpshufb raid_gfmulpshufb
#define gfgencoeff raid_gfcauchycoeff
#define gfgen raid_gfgen
extern const uint8_t raid_gfmul_raid[256][256] __aligned(256);
extern const uint8_t raid_gfexp_raid[256] __aligned(256);
extern const uint8_t raid_gfinv_raid[256] __aligned(256);
extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256);
extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256);
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256);
extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256);
#endif
extern const uint8_t raid_gfmul_aes[256][256] __aligned(256);
extern const uint8_t raid_gfexp_aes[256] __aligned(256);
extern const uint8_t raid_gfinv_aes[256] __aligned(256);
extern const uint8_t raid_gfcauchy_aes[6][256] __aligned(256);
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
extern const uint8_t raid_gfcauchypshufb_aes[251][5][2][16] __aligned(256);
extern const uint8_t raid_gfmulpshufb_aes[256][2][16] __aligned(256);
#endif
extern const uint8_t(*raid_gfmul)[256];
extern const uint8_t *raid_gfexp;
extern const uint8_t *raid_gfinv;
extern const uint8_t(*raid_gfvandermonde)[256];
extern const uint8_t(*raid_gfcauchy)[256];
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
extern const uint8_t(*raid_gfcauchypshufb)[5][2][16];
extern const uint8_t(*raid_gfmulpshufb)[2][16];
#endif
extern const uint8_t(*raid_gfgen)[256];
extern uint8_t raid_poly_byte;
extern uint8_t raid_inv2_byte;
extern uint32_t raid_poly_32;
extern uint64_t raid_poly_64;
extern uint32_t raid_inv2_32;
extern uint64_t raid_inv2_64;
#include "gf.h"
/*
* Assembler blocks.

145
raid/mktables.c
View file

@ -4,10 +4,19 @@
#include <stdio.h>
#include <stdint.h>
/*
* Galois field reduction polynomials.
*
* RAID_POLY_RAID - Standard RAID polynomial 0x1d (x^8 + x^4 + x^3 + x^2 + 1).
* RAID_POLY_AES - AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1) for GFNI acceleration.
*/
#define RAID_POLY_RAID 0x1d
#define RAID_POLY_AES 0x1b
/**
* Multiplication a*b in GF(2^8).
*/
static uint8_t gfmul(uint8_t poly, uint8_t a, uint8_t b)
static uint8_t raid_gfmul(uint8_t poly, uint8_t a, uint8_t b)
{
uint8_t v;
@ -32,7 +41,7 @@ static uint8_t gfmul(uint8_t poly, uint8_t a, uint8_t b)
/**
* Inversion (1/a) in GF(2^8).
*/
uint8_t gfinv[256];
uint8_t raid_gfinv[256];
/**
* Number of parities.
@ -107,7 +116,7 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
inv_x = 1;
for (i = 0; i < DISK; ++i) {
matrix[1 * DISK + i] = inv_x;
inv_x = gfmul(poly, 2, inv_x);
inv_x = raid_gfmul(poly, 2, inv_x);
}
/*
@ -135,13 +144,13 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
for (j = 0; j < PARITY - 2; ++j) {
inv_x = 1;
for (i = 0; i < DISK; ++i) {
uint8_t x = gfinv[inv_x];
uint8_t x = raid_gfinv[inv_x];
matrix[(j + 2) * DISK + i] = gfinv[y ^ x];
inv_x = gfmul(poly, 2, inv_x);
matrix[(j + 2) * DISK + i] = raid_gfinv[y ^ x];
inv_x = raid_gfmul(poly, 2, inv_x);
}
y = gfmul(poly, 2, y);
y = raid_gfmul(poly, 2, y);
}
/*
@ -158,10 +167,10 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
* 1 187 166 215 7 106
*/
for (j = 0; j < PARITY - 2; ++j) {
uint8_t f = gfinv[matrix[(j + 2) * DISK]];
uint8_t f = raid_gfinv[matrix[(j + 2) * DISK]];
for (i = 0; i < DISK; ++i)
matrix[(j + 2) * DISK + i] = gfmul(poly, matrix[(j + 2) * DISK + i], f);
matrix[(j + 2) * DISK + i] = raid_gfmul(poly, matrix[(j + 2) * DISK + i], f);
}
}
@ -181,33 +190,33 @@ static void set_power(uint8_t poly, uint8_t *matrix)
v = 1;
for (i = 0; i < DISK; ++i) {
matrix[1 * DISK + i] = v;
v = gfmul(poly, 2, v);
v = raid_gfmul(poly, 2, v);
}
v = 1;
for (i = 0; i < DISK; ++i) {
matrix[2 * DISK + i] = v;
v = gfmul(poly, div_by_2, v);
v = raid_gfmul(poly, div_by_2, v);
}
}
void tables(uint8_t poly)
void tables(uint8_t poly, const char *tag)
{
uint8_t v;
int i, j, k, p;
uint8_t matrix[PARITY * 256];
/* a*b */
printf("const uint8_t __aligned(256) raid_gfmul[256][256] =\n");
printf("const uint8_t __aligned(256) raid_gfmul_%s[256][256] =\n", tag);
printf("{\n");
for (i = 0; i < 256; ++i) {
printf("\t{\n");
for (j = 0; j < 256; ++j) {
if (j % 8 == 0)
printf("\t\t");
v = gfmul(poly, i, j);
v = raid_gfmul(poly, i, j);
if (v == 1)
gfinv[i] = j;
raid_gfinv[i] = j;
printf("0x%02x,", (unsigned)v);
if (j % 8 == 7)
printf("\n");
@ -219,14 +228,14 @@ void tables(uint8_t poly)
printf("};\n\n");
/* 2^a */
printf("const uint8_t __aligned(256) raid_gfexp[256] =\n");
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
printf("{\n");
v = 1;
for (i = 0; i < 256; ++i) {
if (i % 8 == 0)
printf("\t");
printf("0x%02x,", v);
v = gfmul(poly, v, 2);
v = raid_gfmul(poly, v, 2);
if (i % 8 == 7)
printf("\n");
else
@ -235,7 +244,7 @@ void tables(uint8_t poly)
printf("};\n\n");
/* 1/a */
printf("const uint8_t __aligned(256) raid_gfinv[256] =\n");
printf("const uint8_t __aligned(256) raid_gfinv_%s[256] =\n", tag);
printf("{\n");
printf("\t/* note that the first element is not significant */\n");
for (i = 0; i < 256; ++i) {
@ -244,7 +253,7 @@ void tables(uint8_t poly)
if (i == 0)
v = 0;
else
v = gfinv[i];
v = raid_gfinv[i];
printf("0x%02x,", v);
if (i % 8 == 7)
printf("\n");
@ -253,39 +262,6 @@ void tables(uint8_t poly)
}
printf("};\n\n");
/* power matrix */
set_power(poly, matrix);
printf("/**\n");
printf(" * Power matrix used to generate parity.\n");
printf(" * This matrix is valid for up to %u parities with %u data disks.\n", 3, DISK);
printf(" *\n");
for (p = 0; p < 3; ++p) {
printf(" *");
for (i = 0; i < DISK; ++i)
printf(" %02x", matrix[p * DISK + i]);
printf("\n");
}
printf(" */\n");
printf("const uint8_t __aligned(256) raid_gfvandermonde[%u][256] =\n", 3);
printf("{\n");
for (p = 0; p < 3; ++p) {
printf("\t{\n");
for (i = 0; i < DISK; ++i) {
if (i % 8 == 0)
printf("\t\t");
printf("0x%02x,", matrix[p * DISK + i]);
if (i != DISK - 1) {
if (i % 8 == 7)
printf("\n");
else
printf(" ");
}
}
printf("\n\t},\n");
}
printf("};\n\n");
/* cauchy matrix */
set_cauchy(poly, matrix);
@ -300,7 +276,7 @@ void tables(uint8_t poly)
printf("\n");
}
printf(" */\n");
printf("const uint8_t __aligned(256) raid_gfcauchy[%u][256] =\n", PARITY);
printf("const uint8_t __aligned(256) raid_gfcauchy_%s[%u][256] =\n", tag, PARITY);
printf("{\n");
for (p = 0; p < PARITY; ++p) {
printf("\t{\n");
@ -319,14 +295,14 @@ void tables(uint8_t poly)
}
printf("};\n\n");
printf("#ifdef CONFIG_X86\n");
printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON)\n");
printf("/**\n");
printf(" * PSHUFB tables for the Cauchy matrix.\n");
printf(" *\n");
printf(" * Indexes are [DISK][PARITY - 2][LH].\n");
printf(" * Where DISK is from 0 to %u, PARITY from 1 to %u, LH from 0 to 1.\n", DISK - 1, PARITY);
printf(" */\n");
printf("const uint8_t __aligned(256) raid_gfcauchypshufb[%u][%u][2][16] =\n", DISK, PARITY - 1);
printf("const uint8_t __aligned(256) raid_gfcauchypshufb_%s[%u][%u][2][16] =\n", tag, DISK, PARITY - 1);
printf("{\n");
for (i = 0; i < DISK; ++i) {
printf("\t{\n");
@ -335,9 +311,9 @@ void tables(uint8_t poly)
for (j = 0; j < 2; ++j) {
printf("\t\t\t{ ");
for (k = 0; k < 16; ++k) {
v = gfmul(poly, matrix[p * DISK + i], k);
v = raid_gfmul(poly, matrix[p * DISK + i], k);
if (j == 1)
v = gfmul(poly, v, 16);
v = raid_gfmul(poly, v, 16);
printf("0x%02x", (unsigned)v);
if (k != 15)
printf(", ");
@ -351,23 +327,23 @@ void tables(uint8_t poly)
printf("};\n");
printf("#endif\n\n");
printf("#ifdef CONFIG_X86\n");
printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON)\n");
printf("/**\n");
printf(" * PSHUFB tables for generic multiplication.\n");
printf(" *\n");
printf(" * Indexes are [MULTIPLIER][LH].\n");
printf(" * Where MULTIPLIER is from 0 to 255, LH from 0 to 1.\n");
printf(" */\n");
printf("const uint8_t __aligned(256) raid_gfmulpshufb[256][2][16] =\n");
printf("const uint8_t __aligned(256) raid_gfmulpshufb_%s[256][2][16] =\n", tag);
printf("{\n");
for (i = 0; i < 256; ++i) {
printf("\t{\n");
for (j = 0; j < 2; ++j) {
printf("\t\t{ ");
for (k = 0; k < 16; ++k) {
v = gfmul(poly, i, k);
v = raid_gfmul(poly, i, k);
if (j == 1)
v = gfmul(poly, v, 16);
v = raid_gfmul(poly, v, 16);
printf("0x%02x", (unsigned)v);
if (k != 15)
printf(", ");
@ -378,6 +354,42 @@ void tables(uint8_t poly)
}
printf("};\n");
printf("#endif\n\n");
if (poly == RAID_POLY_AES)
return;
/* power matrix */
set_power(poly, matrix);
printf("/**\n");
printf(" * Power matrix used to generate parity.\n");
printf(" * This matrix is valid for up to %u parities with %u data disks.\n", 3, DISK);
printf(" *\n");
for (p = 0; p < 3; ++p) {
printf(" *");
for (i = 0; i < DISK; ++i)
printf(" %02x", matrix[p * DISK + i]);
printf("\n");
}
printf(" */\n");
printf("const uint8_t __aligned(256) raid_gfvandermonde_%s[%u][256] =\n", tag, 3);
printf("{\n");
for (p = 0; p < 3; ++p) {
printf("\t{\n");
for (i = 0; i < DISK; ++i) {
if (i % 8 == 0)
printf("\t\t");
printf("0x%02x,", matrix[p * DISK + i]);
if (i != DISK - 1) {
if (i % 8 == 7)
printf("\n");
else
printf(" ");
}
}
printf("\n\t},\n");
}
printf("};\n\n");
}
int main(void)
@ -387,15 +399,14 @@ int main(void)
printf("\n");
printf("#include \"internal.h\"\n");
printf("#include \"gf.h\"\n");
printf("\n");
printf("#ifdef USE_RAID_AES\n");
printf("/* Tables with the AES Polynomial 0x1b */\n");
tables(0x1b);
printf("#else\n");
printf("/* Tables with the RAID Polynomial 0x1d */\n");
tables(0x1d);
printf("#endif\n");
tables(RAID_POLY_RAID, "raid");
printf("/* Tables with the AES Polynomial 0x1b */\n");
tables(RAID_POLY_AES, "aes");
return 0;
}

178
raid/module.c
View file

@ -19,14 +19,59 @@ struct raid_gen_algo {
const char *tag; /**< Hardware descriptive tag. */
};
struct raid_rec_algo {
raid_rec_fn *rec;
const char *tag; /**< Hardware descriptive tag. */
};
/**
* Registered algorithms for parity generation.
* Registered algorithms for parity generation and recovery.
*
* Set by the raid_gen_register() calls.
*
* Indexes are the RAID_ALGO_* constants
* Set by the raid_gen_register() / raid_rec_register() calls.
*/
static struct raid_gen_algo raid_gen_algo[RAID_ALGO_MAX];
static struct raid_gen_algo raid_gen_algo_raid[RAID_ALGO_MAX];
static struct raid_rec_algo raid_rec_algo_raid[RAID_PARITY_MAX];
static struct raid_gen_algo raid_gen_algo_aes[RAID_ALGO_MAX];
static struct raid_rec_algo raid_rec_algo_aes[RAID_PARITY_MAX];
static struct raid_gen_algo *raid_gen_algo;
static struct raid_rec_algo *raid_rec_algo;
static int raid_mode_active = RAID_MODE_GET;
const uint8_t(*raid_gfmul)[256];
const uint8_t *raid_gfexp;
const uint8_t *raid_gfinv;
const uint8_t(*raid_gfvandermonde)[256];
const uint8_t(*raid_gfcauchy)[256];
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
const uint8_t(*raid_gfcauchypshufb)[5][2][16];
const uint8_t(*raid_gfmulpshufb)[2][16];
#endif
const uint8_t(*raid_gfgen)[256];
uint8_t raid_poly_byte;
uint8_t raid_inv2_byte;
uint32_t raid_poly_32;
uint64_t raid_poly_64;
uint32_t raid_inv2_32;
uint64_t raid_inv2_64;
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
struct gfconst16 __aligned(16) gfconst16 =
{
{ 0 },
{
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f
},
{ 0 },
{
0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f
}
};
#endif
void raid_gen(int nd, int np, size_t size, void **v)
{
@ -49,20 +94,6 @@ void raid_gen(int nd, int np, size_t size, void **v)
*/
static raid_rec_fn *raid_rec_ptr[RAID_PARITY_MAX];
struct raid_rec_algo {
raid_rec_fn *rec;
const char *tag; /**< Hardware descriptive tag. */
};
/**
* Registered algorithms for data recovery.
*
* Set by the raid_rec_register() calls.
*
* Indexes are the RAID_ALGO_* constants
*/
static struct raid_rec_algo raid_rec_algo[RAID_PARITY_MAX];
void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v)
{
int nrd; /* number of data blocks to recover */
@ -157,14 +188,14 @@ void raid_data(int nr, int *id, int *ip, int nd, size_t size, void **v)
raid_rec_algo[nr - 1].rec(nr, id, ip, nd, size, v);
}
const char *raid_gen_tag(int na)
const char * raid_gen_tag(int na)
{
BUG_ON(na < 0 || na >= RAID_ALGO_MAX);
return raid_gen_algo[na].tag;
}
const char *raid_rec_tag(int na)
const char * raid_rec_tag(int na)
{
/* there is no custom recover for vandermonde */
if (na == RAID_ALGO_VANDERMONDE_PAR3)
@ -175,29 +206,80 @@ const char *raid_rec_tag(int na)
return raid_rec_algo[na].tag;
}
/**
* Generator matrix currently used.
*/
const uint8_t(*raid_gfgen)[256];
void raid_mode(int mode)
int raid_mode(int mode)
{
BUG_ON(mode != RAID_MODE_VANDERMONDE && mode != RAID_MODE_CAUCHY);
int prev = raid_mode_active;
if (mode == RAID_MODE_VANDERMONDE) {
if (mode == RAID_MODE_GET)
return prev;
BUG_ON(mode != RAID_MODE_CAUCHY_RAID && mode != RAID_MODE_VANDERMONDE_RAID && mode != RAID_MODE_CAUCHY_AES);
raid_mode_active = mode;
if (mode == RAID_MODE_CAUCHY_AES) {
raid_gfmul = raid_gfmul_aes;
raid_gfexp = raid_gfexp_aes;
raid_gfinv = raid_gfinv_aes;
raid_gfvandermonde = 0;
raid_gfcauchy = raid_gfcauchy_aes;
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
raid_gfcauchypshufb = raid_gfcauchypshufb_aes;
raid_gfmulpshufb = raid_gfmulpshufb_aes;
#endif
raid_poly_byte = RAID_POLY_AES;
raid_inv2_byte = RAID_INV2_AES;
raid_poly_32 = RAID_POLY_32_AES;
raid_poly_64 = RAID_POLY_64_AES;
raid_inv2_32 = RAID_INV2_32_AES;
raid_inv2_64 = RAID_INV2_64_AES;
raid_gen_algo = raid_gen_algo_aes;
raid_rec_algo = raid_rec_algo_aes;
} else {
raid_gfmul = raid_gfmul_raid;
raid_gfexp = raid_gfexp_raid;
raid_gfinv = raid_gfinv_raid;
raid_gfvandermonde = raid_gfvandermonde_raid;
raid_gfcauchy = raid_gfcauchy_raid;
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
raid_gfcauchypshufb = raid_gfcauchypshufb_raid;
raid_gfmulpshufb = raid_gfmulpshufb_raid;
#endif
raid_poly_byte = RAID_POLY_RAID;
raid_inv2_byte = RAID_INV2_RAID;
raid_poly_32 = RAID_POLY_32_RAID;
raid_poly_64 = RAID_POLY_64_RAID;
raid_inv2_32 = RAID_INV2_32_RAID;
raid_inv2_64 = RAID_INV2_64_RAID;
raid_gen_algo = raid_gen_algo_raid;
raid_rec_algo = raid_rec_algo_raid;
}
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
memset(gfconst16.poly, raid_poly_byte, 16);
memset(gfconst16.half, raid_inv2_byte, 16);
#endif
if (mode == RAID_MODE_VANDERMONDE_RAID) {
raid_gen_ptr[0] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR1].gen;
raid_gen_ptr[1] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR2].gen;
raid_gen_ptr[2] = raid_gen_algo[RAID_ALGO_VANDERMONDE_PAR3].gen;
raid_gen_ptr[3] = 0;
raid_gen_ptr[4] = 0;
raid_gen_ptr[5] = 0;
raid_rec_ptr[0] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR1].rec;
raid_rec_ptr[1] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR2].rec;
raid_rec_ptr[2] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR3].rec;
raid_rec_ptr[3] = 0;
raid_rec_ptr[4] = 0;
raid_rec_ptr[5] = 0;
raid_gfgen = gfvandermonde;
raid_gfgen = raid_gfvandermonde;
} else {
raid_gen_ptr[0] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR1].gen;
raid_gen_ptr[1] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR2].gen;
@ -205,14 +287,18 @@ void raid_mode(int mode)
raid_gen_ptr[3] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR4].gen;
raid_gen_ptr[4] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR5].gen;
raid_gen_ptr[5] = raid_gen_algo[RAID_ALGO_CAUCHY_PAR6].gen;
raid_rec_ptr[0] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR1].rec;
raid_rec_ptr[1] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR2].rec;
raid_rec_ptr[2] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR3].rec;
raid_rec_ptr[3] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR4].rec;
raid_rec_ptr[4] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR5].rec;
raid_rec_ptr[5] = raid_rec_algo[RAID_ALGO_CAUCHY_PAR6].rec;
raid_gfgen = gfcauchy;
raid_gfgen = raid_gfcauchy;
}
return prev;
}
void raid_gen_force(int np, raid_gen_fn *fn)
@ -222,20 +308,32 @@ void raid_gen_force(int np, raid_gen_fn *fn)
raid_gen_ptr[np - 1] = fn;
}
void raid_gen_register(int na, const char *tag, raid_gen_fn *gen)
void raid_gen_register(int na, const char *tag, raid_gen_fn *gen, uint8_t poly)
{
BUG_ON(na < 0 || na >= RAID_ALGO_MAX);
raid_gen_algo[na].tag = tag;
raid_gen_algo[na].gen = gen;
if (poly == RAID_POLY_ANY || poly == RAID_POLY_RAID) {
raid_gen_algo_raid[na].tag = tag;
raid_gen_algo_raid[na].gen = gen;
}
if (poly == RAID_POLY_ANY || poly == RAID_POLY_AES) {
raid_gen_algo_aes[na].tag = tag;
raid_gen_algo_aes[na].gen = gen;
}
}
void raid_rec_register(int na, const char *tag, raid_rec_fn *rec)
void raid_rec_register(int na, const char *tag, raid_rec_fn *rec, uint8_t poly)
{
BUG_ON(na < 0 || na >= RAID_PARITY_MAX);
raid_rec_algo[na].tag = tag;
raid_rec_algo[na].rec = rec;
if (poly == RAID_POLY_ANY || poly == RAID_POLY_RAID) {
raid_rec_algo_raid[na].tag = tag;
raid_rec_algo_raid[na].rec = rec;
}
if (poly == RAID_POLY_ANY || poly == RAID_POLY_AES) {
raid_rec_algo_aes[na].tag = tag;
raid_rec_algo_aes[na].rec = rec;
}
}
/*
@ -259,7 +357,7 @@ void raid_init(void)
#endif
/* set the default mode */
raid_mode(RAID_MODE_CAUCHY);
raid_mode(RAID_MODE_CAUCHY_RAID);
}
/*
@ -281,7 +379,7 @@ void raid_gen_ref(int nd, int np, size_t size, void **vv)
uint8_t b = v[d][i];
for (j = 0; j < np; ++j)
p[j] ^= gfmul[b][gfgen[j][d]];
p[j] ^= raid_gfmul[b][raid_gfgen[j][d]];
}
for (j = 0; j < np; ++j)
@ -477,7 +575,7 @@ int raid_selftest(void)
/* use the multiplication table as data */
for (i = 0; i < nd; ++i)
ref[i] = ((uint8_t *)gfmul) + size * i;
ref[i] = ((uint8_t *)raid_gfmul) + size * i;
/* setup reference parity */
for (i = 0; i < RAID_PARITY_MAX; ++i)

72
raid/neon.c
View file

@ -123,7 +123,7 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
"str q2, %2\n"
"str q3, %3\n"
: "=m" (p[i]), "=m" (p[i + 16]),
"=m" (q[i]), "=m" (q[i + 16])
"=m" (q[i]), "=m" (q[i + 16])
);
}
@ -216,8 +216,8 @@ void raid_genz_neon(int nd, size_t size, void **vv)
"str q4, %4\n"
"str q5, %5\n"
: "=m" (p[i]), "=m" (p[i + 16]),
"=m" (q[i]), "=m" (q[i + 16]),
"=m" (r[i]), "=m" (r[i + 16])
"=m" (q[i]), "=m" (q[i + 16]),
"=m" (r[i]), "=m" (r[i + 16])
);
}
@ -285,7 +285,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0])
: "m" (raid_gfcauchypshufb[l][1][0][0]), "m" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
@ -300,7 +300,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0])
: "m" (raid_gfcauchypshufb[l][2][0][0]), "m" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
@ -315,7 +315,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (gfgenpshufb[l][3][0][0]), "m" (gfgenpshufb[l][3][1][0])
: "m" (raid_gfcauchypshufb[l][3][0][0]), "m" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
@ -330,7 +330,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (gfgenpshufb[l][4][0][0]), "m" (gfgenpshufb[l][4][1][0])
: "m" (raid_gfcauchypshufb[l][4][0][0]), "m" (raid_gfcauchypshufb[l][4][1][0])
);
}
@ -383,7 +383,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0])
: "m" (raid_gfcauchypshufb[d][1][0][0]), "m" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
@ -400,7 +400,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0])
: "m" (raid_gfcauchypshufb[d][2][0][0]), "m" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
@ -417,7 +417,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (gfgenpshufb[d][3][0][0]), "m" (gfgenpshufb[d][3][1][0])
: "m" (raid_gfcauchypshufb[d][3][0][0]), "m" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
@ -434,7 +434,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (gfgenpshufb[d][4][0][0]), "m" (gfgenpshufb[d][4][1][0])
: "m" (raid_gfcauchypshufb[d][4][0][0]), "m" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
@ -494,7 +494,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"str q2, %2\n"
"str q3, %3\n"
: "=m" (v[nd][i]), "=m" (v[nd][i + 16]),
"=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16])
"=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16])
);
if (np >= 3) {
@ -613,7 +613,7 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
"str q2, %0\n"
: "=m" (pa[i])
: "m" (p[i]), "m" (pa[i]), "m" (gfconst16.low4[0]),
"m" (gfmulpshufb[V][0][0]), "m" (gfmulpshufb[V][1][0])
"m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0])
);
}
@ -713,12 +713,12 @@ void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
"str q4, %1\n"
: "=m" (pa[0][i]), "=m" (pa[1][i])
: "m" (gfconst16.low4[0]),
"m" (p[0][i]), "m" (pa[0][i]),
"m" (p[1][i]), "m" (pa[1][i]),
"m" (gfmulpshufb[V[0]][0][0]), "m" (gfmulpshufb[V[0]][1][0]),
"m" (gfmulpshufb[V[1]][0][0]), "m" (gfmulpshufb[V[1]][1][0]),
"m" (gfmulpshufb[V[2]][0][0]), "m" (gfmulpshufb[V[2]][1][0]),
"m" (gfmulpshufb[V[3]][0][0]), "m" (gfmulpshufb[V[3]][1][0])
"m" (p[0][i]), "m" (pa[0][i]),
"m" (p[1][i]), "m" (pa[1][i]),
"m" (raid_gfmulpshufb[V[0]][0][0]), "m" (raid_gfmulpshufb[V[0]][1][0]),
"m" (raid_gfmulpshufb[V[1]][0][0]), "m" (raid_gfmulpshufb[V[1]][1][0]),
"m" (raid_gfmulpshufb[V[2]][0][0]), "m" (raid_gfmulpshufb[V[2]][1][0]),
"m" (raid_gfmulpshufb[V[3]][0][0]), "m" (raid_gfmulpshufb[V[3]][1][0])
);
}
@ -736,7 +736,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t buffer[RAID_PARITY_MAX*16+16];
uint8_t buffer[RAID_PARITY_MAX * 16 + 16];
uint8_t *pd = __align_ptr(buffer, 16);
size_t i;
int j, k;
@ -767,7 +767,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
"ldr q1, %2\n"
"eor v0.16b, v0.16b, v1.16b\n"
"str q0, %0\n"
: "=m" (pd[j*16])
: "=m" (pd[j * 16])
: "m" (p[j][i]), "m" (pa[j][i])
);
}
@ -793,8 +793,8 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
"eor v2.16b, v2.16b, v3.16b\n"
"eor v0.16b, v0.16b, v2.16b\n"
:
: "m" (pd[k*16]), "m" (gfconst16.low4[0]),
"m" (gfmulpshufb[m][0][0]), "m" (gfmulpshufb[m][1][0])
: "m" (pd[k * 16]), "m" (gfconst16.low4[0]),
"m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0])
);
}
@ -810,19 +810,19 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_neon(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_gen1_neon);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon", raid_genz_neon);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon);
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_gen1_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon", raid_genz_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon, RAID_POLY_ANY);
}
#endif

78
raid/raid.h
View file

@ -23,7 +23,7 @@
* other RAID implementations. Use this when reading or writing data
* that may be accessed by other software or hardware RAID controllers.
*
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial (USE_RAID_AES)
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial
*
* The polynomial used by the AES encryption standard. Choosing this
* polynomial enables the use of the Intel GFNI (Galois Field New
@ -39,30 +39,40 @@
* the standard 0x1d polynomial, so this option requires that all
* data be generated and recovered using the same polynomial choice.
*/
#ifdef USE_RAID_AES
#define RAID_POLY 0x1b
#else
#define RAID_POLY 0x1d
#endif
/**
* RAID mode supporting up to 6 parities.
* Special mode code used to query the active mode.
*/
#define RAID_MODE_GET -1
/**
* Default RAID mode supporting up to 6 parities using the standard polynomial 0x1d.
*
* It requires SSSE3 to get good performance with triple or more parities.
* Provides high performance on modern CPUs with SSSE3, AVX2, AVX-512, or NEON support.
* Ensures maximum compatibility with standard Linux RAID arrays.
*
* This is the default mode set after calling raid_init().
*/
#define RAID_MODE_CAUCHY 0
#define RAID_MODE_CAUCHY_RAID 0
/**
* RAID mode supporting up to 3 parities,
* Legacy RAID mode supporting up to 3 parities using standard polynomial 0x1d.
*
* It has a fast triple parity implementation without SSSE3, but it cannot
* go beyond triple parity.
* Uses Vandermonde matrix coefficients for triple parity (Z parity), which allows
* efficient 3-parity generation on older/low-end CPUs without SSSE3 or NEON.
*
* This is mostly intended for low end CPUs like ARM and AMD Athlon.
* It cannot be used for arrays with more than 3 parities.
*/
#define RAID_MODE_VANDERMONDE 1
#define RAID_MODE_VANDERMONDE_RAID 1
/**
* RAID mode supporting up to 6 parities using AES polynomial 0x1b.
*
* It has the same performance as RAID_MODE_CAUCHY_RAID on CPUs without GFNI,
* but achieves significantly higher throughput on CPUs supporting GFNI hardware
* acceleration.
*/
#define RAID_MODE_CAUCHY_AES 2
/**
* Maximum number of parity disks supported.
@ -79,7 +89,7 @@
*
* You must call this function before any other.
*
* The RAID system is initialized in the RAID_MODE_CAUCHY mode.
* The RAID system is initialized in the RAID_MODE_CAUCHY_RAID mode.
*/
void raid_init(void);
@ -94,21 +104,21 @@ void raid_init(void);
int raid_selftest(void);
/**
* Sets the mode to use. One of RAID_MODE_*.
* Sets or queries the mode to use. One of RAID_MODE_*.
*
* You can change mode at any time, and it will affect next calls to raid_gen(),
* raid_rec() and raid_data().
* Passing RAID_MODE_GET (-1) queries the current active mode without changing it.
* Passing a valid mode (RAID_MODE_CAUCHY_RAID, RAID_MODE_VANDERMONDE_RAID,
* RAID_MODE_CAUCHY_AES) sets the mode.
*
* The two modes are compatible for the first two levels of parity.
* The third one is different.
* Returns the mode that was active prior to this call.
*/
void raid_mode(int mode);
int raid_mode(int mode);
/**
* Sets the zero buffer to use in recovering.
* Sets the zero buffer to use during recovery.
*
* Before calling raid_rec() and raid_data() you must provide a memory
* buffer filled with zero with the same size of the blocks to recover.
* buffer filled with zeros of the same size as the blocks to recover.
*
* This buffer is only read and never written.
*/
@ -123,8 +133,8 @@ void raid_zero(void *zero);
* Each parity block allows to recover one data block.
*
* @nd Number of data blocks.
* @np Number of parities blocks to compute.
* @size Size of the blocks pointed by @v. It must be a multiple of 64.
* @np Number of parity blocks to compute.
* @size Size of the blocks pointed to by @v. It must be a multiple of 64.
* @v Vector of pointers to the blocks of data and parity.
* It has (@nd + @np) elements. The starting elements are the blocks for
* data, following with the parity blocks.
@ -139,18 +149,18 @@ void raid_gen(int nd, int np, size_t size, void **v);
* This function recovers all the data and parity blocks marked as bad
* in the @ir vector.
*
* Ensure to have @nr <= @np, otherwise recovering is not possible.
* Ensure @nr <= @np, otherwise recovery is not possible.
*
* The parities blocks used for recovering are automatically selected from
* The parity blocks used for recovery are automatically selected from
* the ones NOT present in the @ir vector.
*
* In case there are more parity blocks than needed, the parities at lower
* indexes are used in the recovering, and the others are ignored.
* indexes are used for recovery, and the others are ignored.
*
* Note that no internal integrity check is done when recovering. If the
* provided parities are correct, the resulting data will be correct.
* If parities are wrong, the resulting recovered data will be wrong.
* This happens even in the case you have more parities blocks than needed,
* This happens even in the case you have more parity blocks than needed,
* and some form of integrity verification would be possible.
*
* @nr Number of failed data and parity blocks to recover.
@ -177,14 +187,14 @@ void raid_rec(int nr, int *ir, int nd, int np, size_t size, void **v);
* @nr Number of failed data blocks to recover.
* @id[] Vector of @nr indexes of the data blocks to recover.
* The indexes start from 0. They must be in order.
* @ip[] Vector of @nr indexes of the parity blocks to use for recovering.
* @ip[] Vector of @nr indexes of the parity blocks to use for recovery.
* The indexes start from 0. They must be in order.
* @nd Number of data blocks.
* @size Size of the blocks pointed by @v. It must be a multiple of 64.
* @size Size of the blocks pointed to by @v. It must be a multiple of 64.
* @v Vector of pointers to the blocks of data and parity.
* It has (@nd + @ip[@nr - 1] + 1) elements. The starting elements are the
* blocks for data, following with the parity blocks.
* Each blocks has @size bytes.
* Each block has @size bytes.
*/
void raid_data(int nr, int *id, int *ip, int nd, size_t size, void **v);
@ -193,10 +203,10 @@ void raid_data(int nr, int *id, int *ip, int nd, size_t size, void **v);
*
* This function checks if the specified failed blocks combination satisfies
* the redundancy information. A combination is assumed matching, if the
* remaining valid parity is matching the expected value after recovering.
* remaining valid parity matches the expected value after recovery.
*
* The number of failed blocks @nr must be strictly less than the number of
* parities @np, because you need one more parity to validate the recovering.
* parities @np, because you need one more parity to validate recovery.
*
* No data or parity blocks are modified.
*

10
raid/sse2.c
View file

@ -414,15 +414,15 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv)
void raid_register_sse2(void)
{
if (raid_cpu_has_sse2()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2);
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2, RAID_POLY_ANY);
#ifdef CONFIG_X86_64
if (!raid_cpu_has_slow_extendedreg())
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext, RAID_POLY_ANY);
/* note that raid_cpu_has_slow_extendedreg() doesn't affect vandermonde */
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext, RAID_POLY_ANY);
#else
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2, RAID_POLY_ANY);
#endif
}
}

198
raid/ssse3.c
View file

@ -48,8 +48,8 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
@ -72,10 +72,10 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("pshufb %xmm5,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
}
@ -153,8 +153,8 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %xmm2,%xmm10");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm2");
@ -194,8 +194,8 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -284,14 +284,14 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -317,15 +317,15 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -408,8 +408,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm5");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %xmm2,%xmm10");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm2");
@ -419,8 +419,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("movdqa %xmm3,%xmm11");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm3");
@ -462,8 +462,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm5");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -475,8 +475,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -542,7 +542,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
uint8_t *t;
int d, l;
size_t i;
uint8_t buffer[16+16];
uint8_t buffer[16 + 16];
uint8_t *pd = __align_ptr(buffer, 16);
l = nd - 1;
@ -575,20 +575,20 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm1");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -615,22 +615,22 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -715,32 +715,32 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
asm volatile ("pand %xmm15,%xmm11");
if (np >= 3) {
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("pshufb %xmm10,%xmm2");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm2");
}
if (np >= 4) {
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("pshufb %xmm10,%xmm3");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm3");
}
if (np >= 5) {
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("pshufb %xmm10,%xmm4");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm4");
}
if (np >= 6) {
asm volatile ("movdqa %0,%%xmm5" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
asm volatile ("pshufb %xmm10,%xmm5");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm5");
@ -765,8 +765,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
asm volatile ("pand %xmm15,%xmm11");
if (np >= 3) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm2");
@ -774,8 +774,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
}
if (np >= 4) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm3");
@ -783,8 +783,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
}
if (np >= 5) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm4");
@ -792,8 +792,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
}
if (np >= 6) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm5");
@ -861,7 +861,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
uint8_t *u;
int d, l;
size_t i;
uint8_t buffer[2*16+16];
uint8_t buffer[2 * 16 + 16];
uint8_t *pd = __align_ptr(buffer, 16);
l = nd - 1;
@ -895,26 +895,26 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm0" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm0");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm0");
asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm1");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -944,29 +944,29 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm0");
asm volatile ("pxor %xmm7,%xmm0");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -1050,8 +1050,8 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (gfmulpshufb[V][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
for (i = 0; i < size; i += 16) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
@ -1118,8 +1118,8 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[0]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[0]][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
asm volatile ("movdqa %xmm0,%xmm4");
asm volatile ("movdqa %xmm0,%xmm5");
asm volatile ("psrlw $4,%xmm5");
@ -1130,8 +1130,8 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("pxor %xmm2,%xmm6");
asm volatile ("pxor %xmm3,%xmm6");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[1]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[1]][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
asm volatile ("movdqa %xmm1,%xmm4");
asm volatile ("movdqa %xmm1,%xmm5");
asm volatile ("psrlw $4,%xmm5");
@ -1146,8 +1146,8 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[2]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[2]][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
asm volatile ("movdqa %xmm0,%xmm4");
asm volatile ("movdqa %xmm0,%xmm5");
asm volatile ("psrlw $4,%xmm5");
@ -1158,8 +1158,8 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("pxor %xmm2,%xmm6");
asm volatile ("pxor %xmm3,%xmm6");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[3]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[3]][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
asm volatile ("movdqa %xmm1,%xmm4");
asm volatile ("movdqa %xmm1,%xmm5");
asm volatile ("psrlw $4,%xmm5");
@ -1187,7 +1187,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t buffer[RAID_PARITY_MAX*16+16];
uint8_t buffer[RAID_PARITY_MAX * 16 + 16];
uint8_t *pd = __align_ptr(buffer, 16);
size_t i;
int j, k;
@ -1218,7 +1218,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[j][i]));
asm volatile ("pxor %xmm1,%xmm0");
asm volatile ("movdqa %%xmm0,%0" : "=m" (pd[j*16]));
asm volatile ("movdqa %%xmm0,%0" : "=m" (pd[j * 16]));
}
/* reconstruct */
@ -1229,9 +1229,9 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
for (k = 0; k < N; ++k) {
uint8_t m = V[j * N + k];
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[m][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[m][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (pd[k*16]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[m][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[m][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (pd[k * 16]));
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("pand %xmm7,%xmm4");
@ -1253,25 +1253,25 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_ssse3(void)
{
if (raid_cpu_has_ssse3()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3, RAID_POLY_ANY);
#ifdef CONFIG_X86_64
if (!raid_cpu_has_slow_extendedreg()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext, RAID_POLY_ANY);
}
#endif
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
}
}
#endif

31516
raid/tables.c
View file

File diff suppressed because it is too large Load diff

84
raid/test.c
View file

@ -164,7 +164,7 @@ int raid_test_rec(int mode, int nd, size_t size)
int np;
raid_mode(mode);
if (mode == RAID_MODE_CAUCHY)
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES)
np = RAID_PARITY_MAX;
else
np = 3;
@ -207,14 +207,12 @@ int raid_test_rec(int mode, int nd, size_t size)
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec1_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_rec1_avx512bw;
#ifdef USE_RAID_AES
if (raid_cpu_has_avx2gfni())
test_setup(i) = raid_rec1_avx2gfni;
if (raid_cpu_has_avx512gfni())
test_setup(i) = raid_rec1_avx512gfni;
#endif
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni())
test_setup(i) = raid_rec1_avx2gfni;
if (raid_cpu_has_avx512gfni())
test_setup(i) = raid_rec1_avx512gfni;
}
#endif
#endif
#ifdef CONFIG_NEON
@ -230,12 +228,12 @@ int raid_test_rec(int mode, int nd, size_t size)
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_rec2_avx512bw;
#ifdef USE_RAID_AES
if (raid_cpu_has_avx2gfni())
test_setup(i) = raid_rec2_avx2gfni;
if (raid_cpu_has_avx512gfni())
test_setup(i) = raid_rec2_avx512gfni;
#endif
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni())
test_setup(i) = raid_rec2_avx2gfni;
if (raid_cpu_has_avx512gfni())
test_setup(i) = raid_rec2_avx512gfni;
}
#endif
#endif
#ifdef CONFIG_NEON
@ -251,12 +249,12 @@ int raid_test_rec(int mode, int nd, size_t size)
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_recX_avx512bw;
#ifdef USE_RAID_AES
if (raid_cpu_has_avx2gfni())
test_setup(i) = raid_recX_avx2gfni;
if (raid_cpu_has_avx512gfni())
test_setup(i) = raid_recX_avx512gfni;
#endif
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni())
test_setup(i) = raid_recX_avx2gfni;
if (raid_cpu_has_avx512gfni())
test_setup(i) = raid_recX_avx512gfni;
}
#endif
#endif
#ifdef CONFIG_NEON
@ -338,7 +336,7 @@ int raid_test_par(int mode, int nd, size_t size)
int np;
raid_mode(mode);
if (mode == RAID_MODE_CAUCHY)
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES)
np = RAID_PARITY_MAX;
else
np = 3;
@ -401,18 +399,18 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(1) = raid_gen1_avx512bw;
test_setup(2) = raid_gen2_avx512bw;
}
#ifdef USE_RAID_AES
if (raid_cpu_has_avx2gfni()) {
test_setup(2) = raid_gen2_avx2gfni;
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
test_setup(2) = raid_gen2_avx2gfni;
}
if (raid_cpu_has_avx512gfni()) {
test_setup(2) = raid_gen2_avx512gfni;
}
}
if (raid_cpu_has_avx512gfni()) {
test_setup(2) = raid_gen2_avx512gfni;
}
#endif
#endif
#endif
if (mode == RAID_MODE_CAUCHY) {
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_int8;
test_setup(4) = raid_gen4_int8;
test_setup(5) = raid_gen5_int8;
@ -451,20 +449,20 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(5) = raid_gen5_avx512bw;
test_setup(6) = raid_gen6_avx512bw;
}
#ifdef USE_RAID_AES
if (raid_cpu_has_avx2gfni()) {
test_setup(3) = raid_gen3_avx2gfni;
test_setup(4) = raid_gen4_avx2gfni;
test_setup(5) = raid_gen5_avx2gfni;
test_setup(6) = raid_gen6_avx2gfni;
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
test_setup(3) = raid_gen3_avx2gfni;
test_setup(4) = raid_gen4_avx2gfni;
test_setup(5) = raid_gen5_avx2gfni;
test_setup(6) = raid_gen6_avx2gfni;
}
if (raid_cpu_has_avx512gfni()) {
test_setup(3) = raid_gen3_avx512gfni;
test_setup(4) = raid_gen4_avx512gfni;
test_setup(5) = raid_gen5_avx512gfni;
test_setup(6) = raid_gen6_avx512gfni;
}
}
if (raid_cpu_has_avx512gfni()) {
test_setup(3) = raid_gen3_avx512gfni;
test_setup(4) = raid_gen4_avx512gfni;
test_setup(5) = raid_gen5_avx512gfni;
test_setup(6) = raid_gen6_avx512gfni;
}
#endif
#endif
#endif
} else {

1
raid/test.h
View file

@ -54,4 +54,3 @@ int raid_test_rec(unsigned mode, int nd, size_t size);
int raid_test_par(unsigned mode, int nd, size_t size);
#endif

2
raid/test/Makefile
View file

@ -32,7 +32,7 @@ CC = gcc
LD = ld
CFLAGS = -I.. -Wall -Wextra -g
ifeq ($(COVERAGE),)
CFLAGS += -O2 -DUSE_RAID_AES
CFLAGS += -O2
else
CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1
endif

40
raid/test/fulltest.c
View file

@ -44,17 +44,9 @@ int main(void)
if (raid_cpu_has_avx512bw())
printf("Including x86 AVX512BW\n");
if (raid_cpu_has_avx2gfni())
#ifdef USE_RAID_AES
printf("Including x86 AVX2GFNI\n");
#else
printf("Excluding x86 AVX2GFNI due polynomial mismatch\n");
#endif
if (raid_cpu_has_avx512gfni())
#ifdef USE_RAID_AES
printf("Including x86 AVX512GFNI\n");
#else
printf("Excluding x86 AVX512GFNI due polynomial mismatch\n");
#endif
#endif
#endif
@ -81,36 +73,51 @@ int main(void)
/* LCOV_EXCL_STOP */
}
printf("Test Cauchy parity generation with 1-%u data disks...\n", RAID_DATA_MAX);
printf("Test Cauchy RAID parity generation with 1-%u data disks...\n", RAID_DATA_MAX);
for (int i = 1; i <= RAID_DATA_MAX; ++i) {
if (raid_test_par(RAID_MODE_CAUCHY, i, TEST_SIZE) != 0) {
if (raid_test_par(RAID_MODE_CAUCHY_RAID, i, TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
}
printf("Test Cauchy recovering with all combinations of %u data and 6 parity blocks...\n", TEST_COUNT);
if (raid_test_rec(RAID_MODE_CAUCHY, TEST_COUNT, TEST_SIZE) != 0) {
printf("Test Cauchy RAID recovering with all combinations of %u data and 6 parity blocks...\n", TEST_COUNT);
if (raid_test_rec(RAID_MODE_CAUCHY_RAID, TEST_COUNT, TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("Test Vandermonde parity generation with %u data disks...\n", RAID_DATA_MAX);
if (raid_test_par(RAID_MODE_VANDERMONDE, RAID_DATA_MAX, TEST_SIZE) != 0) {
printf("Test Cauchy AES parity generation with 1-%u data disks...\n", RAID_DATA_MAX);
for (int i = 1; i <= RAID_DATA_MAX; ++i) {
if (raid_test_par(RAID_MODE_CAUCHY_AES, i, TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
}
printf("Test Cauchy AES recovering with all combinations of %u data and 6 parity blocks...\n", TEST_COUNT);
if (raid_test_rec(RAID_MODE_CAUCHY_AES, TEST_COUNT, TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("Test Vandermonde recovering with all combinations of %u data and 3 parity blocks...\n", TEST_COUNT);
if (raid_test_rec(RAID_MODE_VANDERMONDE, TEST_COUNT, TEST_SIZE) != 0) {
printf("Test Vandermonde RAID parity generation with %u data disks...\n", RAID_DATA_MAX);
if (raid_test_par(RAID_MODE_VANDERMONDE_RAID, RAID_DATA_MAX, TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("Test Vandermonde RAID recovering with all combinations of %u data and 3 parity blocks...\n", TEST_COUNT);
if (raid_test_rec(RAID_MODE_VANDERMONDE_RAID, TEST_COUNT, TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("OK\n");
return 0;
@ -121,4 +128,3 @@ bail:
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

5
raid/test/invtest.c
View file

@ -98,7 +98,7 @@ static __always_inline int test_sub_matrix(int nr, int64_t *total)
/* setup the submatrix */
for (i = 0; i < nr; ++i)
for (j = 0; j < nr; ++j)
M[i * nr + j] = gfgen[ip[i]][id[j]];
M[i * nr + j] = raid_gfgen[ip[i]][id[j]];
/* invert */
if (raid_invert_fast(M, nr) != 0)
@ -155,7 +155,7 @@ int main(void)
{
printf("Matrix inversion test for the RAID Cauchy library\n\n");
/* required to set the gfgen table */
/* required to set the raid tables */
raid_init();
if (test_all_sub_matrix() != 0) {
@ -166,4 +166,3 @@ int main(void)
return 0;
}

23
raid/test/selftest.c
View file

@ -15,15 +15,28 @@ int main(void)
raid_init();
printf("Self-test...\n");
printf("Test Cauchy RAID\n");
if (raid_selftest() != 0) {
/* LCOV_EXCL_START */
printf("FAILED!\n");
exit(EXIT_FAILURE);
goto bail;
/* LCOV_EXCL_STOP */
}
printf("OK\n\n");
raid_mode(RAID_MODE_CAUCHY_AES);
printf("Test Cauchy AES\n");
if (raid_selftest() != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("OK\n");
return 0;
}
bail:
/* LCOV_EXCL_START */
printf("FAILED!\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

17
raid/test/speedtest.c
View file

@ -58,8 +58,8 @@ static int64_t diffgettimeofday(struct timeval *start, struct timeval *stop)
* Stop time measurement.
*/
#define SPEED_STOP \
count += delta; \
gettimeofday(&stop, 0); \
count += delta; \
gettimeofday(&stop, 0); \
} while (diffgettimeofday(&start, &stop) < period); \
ds = size * (int64_t)count * nd; \
dt = diffgettimeofday(&start, &stop);
@ -87,7 +87,7 @@ void speed_mem(int nd, void **v, int size, int delta, int period)
printf("\n");
}
void speed_gen(int nd, void **v, int size, int delta, int period, const char* msg)
void speed_gen(int nd, void **v, int size, int delta, int period, const char *msg)
{
struct timeval start;
struct timeval stop;
@ -1285,7 +1285,7 @@ void speed(void)
memset(v[nd + RAID_PARITY_MAX], 0, size);
memset(v_direct[nd + RAID_PARITY_MAX], 0, size);
raid_zero(v[nd + RAID_PARITY_MAX]);
speed_mem(nd, v, size, delta, period);
speed_gen(nd, v, size, delta, period, "");
speed_gen(nd, v_direct, size, delta, period, " (without displacement)");
@ -1316,17 +1316,9 @@ int main(void)
if (raid_cpu_has_avx512bw())
printf("Including x86 AVX512BW\n");
if (raid_cpu_has_avx2gfni())
#ifdef USE_RAID_AES
printf("Including x86 AVX2GFNI\n");
#else
printf("Including x86 AVX2GFNI (Performance reference only. Polynomial mismatch prevents use)\n");
#endif
if (raid_cpu_has_avx512gfni())
#ifdef USE_RAID_AES
printf("Including x86 AVX512GFNI\n");
#else
printf("Including x86 AVX512GFNI (Performance reference only. Polynomial mismatch prevents use)\n");
#endif
#endif
printf("\nPlease wait about 30 seconds...\n\n");
@ -1335,4 +1327,3 @@ int main(void)
return 0;
}