mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
Remove stream CRC and optimize CRC compute CRC32C using two lanes
This commit is contained in:
Andrea Mazzoleni
parent
581ec4e156
commit
a01ec45f14
7 changed files with 196 additions and 129 deletions
144
cmdline/util.h
144
cmdline/util.h
|
|
@ -91,31 +91,14 @@ extern uint32_t CRC32C_1[256];
|
|||
extern uint32_t CRC32C_2[256];
|
||||
extern uint32_t CRC32C_3[256];
|
||||
|
||||
/**
|
||||
* If the CPU support the CRC instructions.
|
||||
*/
|
||||
#if CONFIG_X86
|
||||
extern int crc_x86;
|
||||
#endif
|
||||
/** Minimum size worth splitting into two hardware CRC lanes. */
|
||||
#define CRC32C_2LANE_MIN 4096
|
||||
|
||||
/**
|
||||
* Compute CRC-32 (Castagnoli) for a single byte without the IV.
|
||||
* Advance a raw CRC-32C by a number of zero bytes.
|
||||
* Used to combine independent hardware CRC lanes.
|
||||
*/
|
||||
static inline uint32_t crc32c_plain_char(uint32_t crc, unsigned char c)
|
||||
{
|
||||
#if CONFIG_X86
|
||||
if (tommy_likely(crc_x86)) {
|
||||
asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (c));
|
||||
return crc;
|
||||
}
|
||||
#endif
|
||||
#if CONFIG_ARM_CRC
|
||||
asm ("crc32cb %w0, %w0, %w1\n" : "+r" (crc) : "r" (c));
|
||||
return crc;
|
||||
#else
|
||||
return CRC32C_0[(crc ^ c) & 0xff] ^ (crc >> 8);
|
||||
#endif
|
||||
}
|
||||
uint32_t crc32c_shift(uint32_t crc, size_t size);
|
||||
|
||||
/**
|
||||
* Compute the CRC-32 (Castagnoli) without the IV.
|
||||
|
|
@ -146,12 +129,31 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr,
|
|||
{
|
||||
#ifdef CONFIG_X86_64
|
||||
uint64_t crc64 = crc;
|
||||
if (size >= CRC32C_2LANE_MIN) {
|
||||
const unsigned char* ptr0 = ptr;
|
||||
size_t lane_size = (size / 2) & ~(size_t)31;
|
||||
const unsigned char* ptr1 = ptr + lane_size;
|
||||
const unsigned char* end = ptr1;
|
||||
uint64_t crc0 = crc;
|
||||
uint64_t crc1 = 0;
|
||||
|
||||
/* block hashes are normally 16 bytes and dominate content serialization */
|
||||
if (tommy_likely(size == 16)) {
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8)));
|
||||
return (uint32_t)crc64;
|
||||
/* Keep two dependency chains in flight and combine them only once. */
|
||||
while (ptr0 < end) {
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc0) : "m" (*(const uint64_t*)ptr0));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc1) : "m" (*(const uint64_t*)ptr1));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc0) : "m" (*(const uint64_t*)(ptr0 + 8)));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc1) : "m" (*(const uint64_t*)(ptr1 + 8)));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc0) : "m" (*(const uint64_t*)(ptr0 + 16)));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc1) : "m" (*(const uint64_t*)(ptr1 + 16)));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc0) : "m" (*(const uint64_t*)(ptr0 + 24)));
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc1) : "m" (*(const uint64_t*)(ptr1 + 24)));
|
||||
ptr0 += 32;
|
||||
ptr1 += 32;
|
||||
}
|
||||
|
||||
crc64 = crc32c_shift((uint32_t)crc0, lane_size) ^ (uint32_t)crc1;
|
||||
ptr += 2 * lane_size;
|
||||
size -= 2 * lane_size;
|
||||
}
|
||||
|
||||
while (size >= 32) {
|
||||
|
|
@ -175,6 +177,33 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr,
|
|||
}
|
||||
crc = (uint32_t)crc64;
|
||||
#else
|
||||
if (size >= CRC32C_2LANE_MIN) {
|
||||
const unsigned char* ptr0 = ptr;
|
||||
size_t lane_size = (size / 2) & ~(size_t)15;
|
||||
const unsigned char* ptr1 = ptr + lane_size;
|
||||
const unsigned char* end = ptr1;
|
||||
uint32_t crc0 = crc;
|
||||
uint32_t crc1 = 0;
|
||||
|
||||
/* Keep two dependency chains in flight and combine them only once. */
|
||||
while (ptr0 < end) {
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc0) : "m" (*(const uint32_t*)ptr0));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc1) : "m" (*(const uint32_t*)ptr1));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc0) : "m" (*(const uint32_t*)(ptr0 + 4)));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc1) : "m" (*(const uint32_t*)(ptr1 + 4)));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc0) : "m" (*(const uint32_t*)(ptr0 + 8)));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc1) : "m" (*(const uint32_t*)(ptr1 + 8)));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc0) : "m" (*(const uint32_t*)(ptr0 + 12)));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc1) : "m" (*(const uint32_t*)(ptr1 + 12)));
|
||||
ptr0 += 16;
|
||||
ptr1 += 16;
|
||||
}
|
||||
|
||||
crc = crc32c_shift(crc0, lane_size) ^ crc1;
|
||||
ptr += 2 * lane_size;
|
||||
size -= 2 * lane_size;
|
||||
}
|
||||
|
||||
while (size >= 16) {
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4)));
|
||||
|
|
@ -211,6 +240,50 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr,
|
|||
#if CONFIG_ARM_CRC
|
||||
static inline uint32_t crc32c_arm64_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
{
|
||||
if (size >= CRC32C_2LANE_MIN) {
|
||||
const unsigned char* ptr0 = ptr;
|
||||
size_t lane_size = (size / 2) & ~(size_t)31;
|
||||
const unsigned char* ptr1 = ptr + lane_size;
|
||||
const unsigned char* end = ptr1;
|
||||
uint32_t crc0 = crc;
|
||||
uint32_t crc1 = 0;
|
||||
|
||||
/* Keep two dependency chains in flight and combine them only once. */
|
||||
while (ptr0 < end) {
|
||||
uint64_t val00;
|
||||
uint64_t val01;
|
||||
uint64_t val02;
|
||||
uint64_t val03;
|
||||
uint64_t val10;
|
||||
uint64_t val11;
|
||||
uint64_t val12;
|
||||
uint64_t val13;
|
||||
|
||||
__builtin_memcpy(&val00, ptr0, 8);
|
||||
__builtin_memcpy(&val10, ptr1, 8);
|
||||
__builtin_memcpy(&val01, ptr0 + 8, 8);
|
||||
__builtin_memcpy(&val11, ptr1 + 8, 8);
|
||||
__builtin_memcpy(&val02, ptr0 + 16, 8);
|
||||
__builtin_memcpy(&val12, ptr1 + 16, 8);
|
||||
__builtin_memcpy(&val03, ptr0 + 24, 8);
|
||||
__builtin_memcpy(&val13, ptr1 + 24, 8);
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val00));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val10));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val01));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val11));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val02));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val12));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val03));
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val13));
|
||||
ptr0 += 32;
|
||||
ptr1 += 32;
|
||||
}
|
||||
|
||||
crc = crc32c_shift(crc0, lane_size) ^ crc1;
|
||||
ptr += 2 * lane_size;
|
||||
size -= 2 * lane_size;
|
||||
}
|
||||
|
||||
while (size >= 8) {
|
||||
uint64_t val;
|
||||
__builtin_memcpy(&val, ptr, 8);
|
||||
|
|
@ -235,23 +308,6 @@ static inline uint32_t crc32c_arm64_plain(uint32_t crc, const unsigned char* ptr
|
|||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute CRC-32 (Castagnoli) without the IV.
|
||||
*/
|
||||
static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
{
|
||||
#if CONFIG_X86
|
||||
if (tommy_likely(crc_x86)) {
|
||||
return crc32c_x86_plain(crc, ptr, size);
|
||||
}
|
||||
#endif
|
||||
#if CONFIG_ARM_CRC
|
||||
return crc32c_arm64_plain(crc, ptr, size);
|
||||
#else
|
||||
return crc32c_gen_plain(crc, ptr, size);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CRC-32 (Castagnoli)
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue