diff --git a/raid/internal.h b/raid/internal.h index 4d1098dc..283f2477 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -73,20 +73,33 @@ #include #include -#ifndef BUG_ON +/* + * Inverse assert. + */ #define BUG_ON(a) assert(!(a)) -#endif +/* + * Forced inline. + */ #ifndef __always_inline #define __always_inline inline __attribute__((always_inline)) #endif +/* + * Forced alignment. + */ #ifndef __aligned #define __aligned(a) __attribute__((aligned(a))) #endif /* - * Includes the headers. + * Align a pointer at the specified size. + */ +#define __align_ptr(ptr, size) \ + (void*)((uintptr_t)(((uint8_t*)ptr) + size - 1) & ~(uintptr_t)(size - 1)) + +/* + * Includes the main interface header. */ #include "raid.h" diff --git a/raid/x86.c b/raid/x86.c index bafa4f91..506d8ab4 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -1119,10 +1119,6 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) /* * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation */ -#ifdef _WIN32 -/* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ -__attribute__((force_align_arg_pointer)) -#endif void raid_gen5_ssse3(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -1133,7 +1129,8 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv) uint8_t *t; int d, l; size_t i; - uint8_t p0[16] __aligned(16); + uint8_t buffer[16+16]; + uint8_t* p0 = __align_ptr(buffer, 16); l = nd - 1; p = v[nd]; @@ -1522,10 +1519,6 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv) /* * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation */ -#ifdef _WIN32 -/* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ -__attribute__((force_align_arg_pointer)) -#endif void raid_gen6_ssse3(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -1537,8 +1530,8 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) uint8_t *u; int d, l; size_t i; - uint8_t p0[16] __aligned(16); - uint8_t q0[16] __aligned(16); + uint8_t buffer[2*16+16]; + uint8_t* p0 = __align_ptr(buffer, 16); l = nd - 1; p = v[nd]; @@ -1563,7 +1556,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); asm volatile ("movdqa %%xmm4,%0" : "=m" (p0[0])); - asm volatile ("movdqa %%xmm4,%0" : "=m" (q0[0])); + asm volatile ("movdqa %%xmm4,%0" : "=m" (p0[16])); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %xmm4,%xmm5"); @@ -1598,7 +1591,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) /* intermediate disks */ for (d = l - 1; d > 0; --d) { asm volatile ("movdqa %0,%%xmm5" : : "m" (p0[0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (q0[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (p0[16])); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile ("pxor %xmm4,%xmm4"); @@ -1612,7 +1605,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) asm volatile ("pxor %xmm4,%xmm5"); asm volatile ("pxor %xmm4,%xmm6"); asm volatile ("movdqa %%xmm5,%0" : "=m" (p0[0])); - asm volatile ("movdqa %%xmm6,%0" : "=m" (q0[0])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (p0[16])); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %xmm4,%xmm5"); @@ -1651,7 +1644,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) /* first disk with all coefficients at 1 */ asm volatile ("movdqa %0,%%xmm5" : : "m" (p0[0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (q0[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (p0[16])); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile ("pxor %xmm4,%xmm4"); @@ -2154,6 +2147,8 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*16+16]; + uint8_t *PD = __align_ptr(buffer, 16); size_t i; int j, k; @@ -2178,14 +2173,12 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 16) { - uint8_t PD[RAID_PARITY_MAX][16] __aligned(16); - /* delta */ for (j = 0; j < N; ++j) { asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i])); asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[j][i])); asm volatile ("pxor %xmm1,%xmm0"); - asm volatile ("movdqa %%xmm0,%0" : "=m" (PD[j][0])); + asm volatile ("movdqa %%xmm0,%0" : "=m" (PD[j*16])); } /* reconstruct */ @@ -2198,7 +2191,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[m][0][0])); asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[m][1][0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (PD[k][0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (PD[k*16])); asm volatile ("movdqa %xmm4,%xmm5"); asm volatile ("psrlw $4,%xmm5"); asm volatile ("pand %xmm7,%xmm4"); @@ -2384,6 +2377,8 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*32+32]; + uint8_t *PD = __align_ptr(buffer, 32); size_t i; int j, k; @@ -2408,14 +2403,12 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { - uint8_t PD[RAID_PARITY_MAX][32] __aligned(32); - /* delta */ for (j = 0; j < N; ++j) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[j][i])); asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (PD[j][0])); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (PD[j*32])); } /* reconstruct */ @@ -2428,7 +2421,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[m][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[m][1][0])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (PD[k][0])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (PD[k*32])); asm volatile ("vpsrlw $4,%ymm4,%ymm5"); asm volatile ("vpand %ymm7,%ymm4,%ymm4"); asm volatile ("vpand %ymm7,%ymm5,%ymm5");