diff --git a/cmdline/speed.c b/cmdline/speed.c index 947b4b53..71030809 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -1212,7 +1212,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) /* ensure to use same hardware in the delta step */ raid_gen_force(1, raid_gen1_avx2); /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1223,7 +1223,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) /* ensure to use same hardware in the delta step */ raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1305,9 +1305,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx2gfni_aes); + raid_gen_force(2, raid_gen2_avx2gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1316,9 +1316,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx512gfni_aes); + raid_gen_force(2, raid_gen2_avx512gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1396,8 +1396,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx2gfni_aes); - raid_recX_avx2gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx2gfni_raid); + raid_recX_avx2gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1406,8 +1406,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx512gfni_aes); - raid_recX_avx512gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx512gfni_raid); + raid_recX_avx512gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1486,8 +1486,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx2gfni_aes); - raid_recX_avx2gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx2gfni_raid); + raid_recX_avx2gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1496,8 +1496,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx512gfni_aes); - raid_recX_avx512gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx512gfni_raid); + raid_recX_avx512gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1575,8 +1575,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx2gfni_aes); - raid_recX_avx2gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx2gfni_raid); + raid_recX_avx2gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1585,8 +1585,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx512gfni_aes); - raid_recX_avx512gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx512gfni_raid); + raid_recX_avx512gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1664,8 +1664,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx2gfni_aes); - raid_recX_avx2gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx2gfni_raid); + raid_recX_avx2gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1674,8 +1674,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx512gfni_aes); - raid_recX_avx512gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx512gfni_raid); + raid_recX_avx512gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); diff --git a/raid/gfni.c b/raid/gfni.c index 3a5616c4..c873c09b 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -551,7 +551,362 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) /* * RAID recovering for one disk AVX2 GFNI implementation */ -void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastq %0,%%ymm4" : : "m" (raid_gfaffine_raid[V][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm0,%ymm0"); + asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for one disk GFNI implementation + */ +void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfaffine_raid[V][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for two disks AVX2 GFNI implementation + */ +void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + asm volatile ("vpbroadcastq %0,%%ymm10" : : "m" (raid_gfaffine_raid[V[0]][0])); + asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[V[1]][0])); + asm volatile ("vpbroadcastq %0,%%ymm12" : : "m" (raid_gfaffine_raid[V[2]][0])); + asm volatile ("vpbroadcastq %0,%%ymm13" : : "m" (raid_gfaffine_raid[V[3]][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32])); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32])); + + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + + asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm0,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm1,%ymm5"); + + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm4,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); + + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32])); + + asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm0,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm1,%ymm7"); + + asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm2,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm6,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm3,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); + + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); + asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for two disks GFNI implementation + */ +void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[V[0]][0])); + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[V[1]][0])); + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[V[2]][0])); + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[V[3]][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); + asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); + + asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm0,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm1,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm0,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm1,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering AVX2 GFNI implementation + */ +void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); + for (k = 0; k < N; ++k) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32])); + + asm volatile ("vpbroadcastq %0,%%ymm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0])); + + asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm4,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm0,%ymm0"); + + asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm5,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + } + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32])); + } + } + + raid_avx_end(); +} + +/* + * RAID recovering GFNI implementation + */ +void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); + for (k = 0; k < N; ++k) { + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64])); + asm volatile ("vpbroadcastq %0,%%zmm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0])); + asm volatile ("vgf2p8affineqb $0,%zmm2,%zmm4,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + } + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +/* + * AES recovering for one disk AVX2 GFNI implementation + */ +void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p, *pa; @@ -598,9 +953,9 @@ void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv } /* - * RAID recovering for one disk GFNI implementation + * AES recovering for one disk GFNI implementation */ -void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p, *pa; @@ -638,9 +993,9 @@ void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } /* - * RAID recovering for two disks AVX2 GFNI implementation + * AES recovering for two disks AVX2 GFNI implementation */ -void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; const int N = 2; @@ -715,9 +1070,9 @@ void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv } /* - * RAID recovering for two disks GFNI implementation + * AES recovering for two disks GFNI implementation */ -void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; const int N = 2; @@ -779,9 +1134,9 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } /* - * RAID recovering AVX2 GFNI implementation + * AES recovering AVX2 GFNI implementation */ -void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; int N = nr; @@ -849,9 +1204,9 @@ void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv } /* - * RAID recovering GFNI implementation + * AES recovering GFNI implementation */ -void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; int N = nr; @@ -918,12 +1273,19 @@ void raid_register_avx2gfni(void) raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); } } @@ -943,12 +1305,19 @@ void raid_register_avx512gfni(void) raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); } } } diff --git a/raid/internal.h b/raid/internal.h index 68bf6a79..255e67c4 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -171,12 +171,18 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_gen1_neon(int nd, size_t size, void **vv); void raid_gen2_neon_raid(int nd, size_t size, void **vv); void raid_gen2_neon_aes(int nd, size_t size, void **vv); diff --git a/raid/test.c b/raid/test.c index 682747b2..a2155dee 100644 --- a/raid/test.c +++ b/raid/test.c @@ -399,11 +399,19 @@ int raid_test_rec(int mode, int nd, size_t size) if (raid_cpu_has_avx2()) test_setup(i) = raid_rec1_avx2; #ifdef CONFIG_X86_64 - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) - test_setup(i) = raid_rec1_avx2gfni; - if (raid_cpu_has_avx512gfni()) - test_setup(i) = raid_rec1_avx512gfni; + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec1_avx2gfni_aes; + else + test_setup(i) = raid_rec1_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec1_avx512gfni_aes; + else + test_setup(i) = raid_rec1_avx512gfni_raid; + } } #endif #endif @@ -420,11 +428,19 @@ int raid_test_rec(int mode, int nd, size_t size) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) test_setup(i) = raid_rec2_avx512bw; - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) - test_setup(i) = raid_rec2_avx2gfni; - if (raid_cpu_has_avx512gfni()) - test_setup(i) = raid_rec2_avx512gfni; + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec2_avx2gfni_aes; + else + test_setup(i) = raid_rec2_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec2_avx512gfni_aes; + else + test_setup(i) = raid_rec2_avx512gfni_raid; + } } #endif #endif @@ -441,11 +457,19 @@ int raid_test_rec(int mode, int nd, size_t size) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) test_setup(i) = raid_recX_avx512bw; - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) - test_setup(i) = raid_recX_avx2gfni; - if (raid_cpu_has_avx512gfni()) - test_setup(i) = raid_recX_avx512gfni; + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_recX_avx2gfni_aes; + else + test_setup(i) = raid_recX_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_recX_avx512gfni_aes; + else + test_setup(i) = raid_recX_avx512gfni_raid; + } } #endif #endif diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 0a489529..f534e063 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -1102,7 +1102,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) SPEED_START { raid_gen_force(1, raid_gen1_avx2); /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1112,7 +1112,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) SPEED_START { raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1189,9 +1189,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx2gfni_aes); + raid_gen_force(2, raid_gen2_avx2gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1199,9 +1199,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx512gfni_aes); + raid_gen_force(2, raid_gen2_avx512gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1268,8 +1268,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx2gfni_aes); - raid_recX_avx2gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx2gfni_raid); + raid_recX_avx2gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1277,8 +1277,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx512gfni_aes); - raid_recX_avx512gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx512gfni_raid); + raid_recX_avx512gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1349,8 +1349,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx2gfni_aes); - raid_recX_avx2gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx2gfni_raid); + raid_recX_avx2gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1358,8 +1358,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx512gfni_aes); - raid_recX_avx512gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx512gfni_raid); + raid_recX_avx512gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1430,8 +1430,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx2gfni_aes); - raid_recX_avx2gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx2gfni_raid); + raid_recX_avx2gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1439,8 +1439,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx512gfni_aes); - raid_recX_avx512gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx512gfni_raid); + raid_recX_avx512gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1511,8 +1511,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx2gfni_aes); - raid_recX_avx2gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx2gfni_raid); + raid_recX_avx2gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1520,8 +1520,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx512gfni_aes); - raid_recX_avx512gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx512gfni_raid); + raid_recX_avx512gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt);