diff --git a/raid.c b/raid.c index 315c0a0d..ca5e48be 100644 --- a/raid.c +++ b/raid.c @@ -79,21 +79,60 @@ void raid5_mmxr2(unsigned char** buffer, unsigned diskmax, unsigned size) p = buffer[z0+1]; /* XOR parity */ for(d=0;d=0;--z) { - asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d])); - asm volatile("movq %0,%%mm7" : : "m" (buffer[z][d+8])); - asm volatile("pxor %mm5,%mm2"); - asm volatile("pxor %mm7,%mm3"); + asm volatile("movq %0,%%mm4" : : "m" (buffer[z][d])); + asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d+8])); + asm volatile("pxor %mm4,%mm0"); + asm volatile("pxor %mm5,%mm1"); } - asm volatile("movq %%mm2,%0" : "=m" (p[d])); - asm volatile("movq %%mm3,%0" : "=m" (p[d+8])); + asm volatile("movq %%mm0,%0" : "=m" (p[d])); + asm volatile("movq %%mm1,%0" : "=m" (p[d+8])); } asm volatile("emms" : : : "memory"); } +/* + * Unrolled-by-4 MMX implementation + */ +void raid5_mmxr4(unsigned char** buffer, unsigned diskmax, unsigned size) +{ + unsigned char* p; + int z, z0; + unsigned d; + + z0 = diskmax - 1; /* Highest data disk */ + p = buffer[z0+1]; /* XOR parity */ + + for(d=0;d=0;--z) { + asm volatile("movq %0,%%mm4" : : "m" (buffer[z][d])); + asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d+8])); + asm volatile("movq %0,%%mm6" : : "m" (buffer[z][d+16])); + asm volatile("movq %0,%%mm7" : : "m" (buffer[z][d+24])); + asm volatile("pxor %mm4,%mm0"); + asm volatile("pxor %mm5,%mm1"); + asm volatile("pxor %mm6,%mm2"); + asm volatile("pxor %mm7,%mm3"); + } + asm volatile("movq %%mm0,%0" : "=m" (p[d])); + asm volatile("movq %%mm1,%0" : "=m" (p[d+8])); + asm volatile("movq %%mm2,%0" : "=m" (p[d+16])); + asm volatile("movq %%mm3,%0" : "=m" (p[d+24])); + } + + asm volatile("emms" : : : "memory"); +} + +/* + * Unrolled-by-2 SSE2 implementation + */ void raid5_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) { unsigned char* p; @@ -104,16 +143,52 @@ void raid5_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) p = buffer[z0+1]; /* XOR parity */ for(d=0;d=0;--z) { - asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm7" : : "m" (buffer[z][d+16])); - asm volatile("pxor %xmm5,%xmm2"); + asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d])); + asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm5,%xmm1"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); + asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); + } + + asm volatile("sfence" : : : "memory"); +} + +/* + * Unrolled-by-4 SSE2 implementation + */ +void raid5_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) +{ + unsigned char* p; + int z, z0; + unsigned d; + + z0 = diskmax - 1; /* Highest data disk */ + p = buffer[z0+1]; /* XOR parity */ + + for(d=0;d=0;--z) { + asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d])); + asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16])); + asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d+32])); + asm volatile("movdqa %0,%%xmm7" : : "m" (buffer[z][d+48])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm5,%xmm1"); + asm volatile("pxor %xmm6,%xmm2"); asm volatile("pxor %xmm7,%xmm3"); } - asm volatile("movntdq %%xmm2,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); + asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32])); + asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48])); } asm volatile("sfence" : : : "memory"); @@ -282,13 +357,11 @@ void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) /* We uniformly assume a single prefetch covers at least 32 bytes */ for(d=0;d=0;--z) { - asm volatile("prefetchnta %0" : : "m" (buffer[z][d])); asm volatile("pcmpgtb %xmm4,%xmm5"); asm volatile("pcmpgtb %xmm6,%xmm7"); asm volatile("paddb %xmm4,%xmm4"); @@ -426,7 +499,7 @@ void raid_init(void) raid6_gen = raid6_mmxr2; } if (cpu_has_sse2()) { - raid5_gen = raid5_sse2r2; + raid5_gen = raid5_sse2r4; if (!cpu_has_slowsse2()) raid6_gen = raid6_sse2r2; } diff --git a/raid.h b/raid.h index 6db271e9..a78745e6 100644 --- a/raid.h +++ b/raid.h @@ -51,7 +51,9 @@ void raid_init(void); */ void raid5_int32r2(unsigned char** dptr, unsigned diskmax, unsigned size); void raid5_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned size); +void raid5_mmxr4(unsigned char** dptr, unsigned diskmax, unsigned size); void raid5_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned size); +void raid5_sse2r4(unsigned char** dptr, unsigned diskmax, unsigned size); void raid6_int32r2(unsigned char** dptr, unsigned diskmax, unsigned size); void raid6_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned size); void raid6_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned size); diff --git a/speed.c b/speed.c index e83e9c9c..a0de24d2 100644 --- a/speed.c +++ b/speed.c @@ -122,9 +122,9 @@ void speed(void) #if defined(__i386__) || defined(__x86_64__) printf("Expected fastest RAID5 is "); if (cpu_has_sse2()) - printf("sse2x2"); + printf("sse2x4"); else if (cpu_has_mmx()) - printf("mmxx2"); + printf("mmxx4"); else printf("int32x2"); printf("\n"); @@ -180,6 +180,12 @@ void speed(void) } SPEED_STOP printf("RAID5 mmxx2 %"PRIu64" [MB/s]\n", ds / dt); + + SPEED_START { + raid5_mmxr4(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAID5 mmxx4 %"PRIu64" [MB/s]\n", ds / dt); } if (cpu_has_sse2()) { @@ -188,6 +194,12 @@ void speed(void) } SPEED_STOP printf("RAID5 sse2x2 %"PRIu64" [MB/s]\n", ds / dt); + + SPEED_START { + raid5_sse2r4(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAID5 sse2x4 %"PRIu64" [MB/s]\n", ds / dt); } #endif