Optimize hardware CRC32C for common sizes and tails
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
Build / build-freebsd (push) Waiting to run
Build / build-musl (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run

This commit is contained in:
Andrea Mazzoleni 2026-09-10 00:28:24 +02:00
parent e45e467b1a
commit 581ec4e156
2 changed files with 46 additions and 6 deletions

View file

@ -267,6 +267,7 @@ static struct crc_test_vector TEST_CRC32C[] = {
{ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 32, 0x62a8ab43 },
{ "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18\x17\x16\x15\x14\x13\x12\x11\x10\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08\x07\x06\x05\x04\x03\x02\x01\x00", 32, 0x113fdb5c },
{ "\x01\xc0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x14\x00\x00\x00\x18\x28\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00", 48, 0xd9963a56 },
{ "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 16, 0xd9c908eb },
{ "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", 32, 0x46dd794e },
{ "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28", 40, 0x0e2c157f },
{ "\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50", 40, 0xe980ebf6 },

View file

@ -146,23 +146,62 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr,
{
#ifdef CONFIG_X86_64
uint64_t crc64 = crc;
while (size >= 8) {
/* block hashes are normally 16 bytes and dominate content serialization */
if (tommy_likely(size == 16)) {
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8)));
return (uint32_t)crc64;
}
while (size >= 32) {
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8)));
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 16)));
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 24)));
ptr += 32;
size -= 32;
}
if (size >= 16) {
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8)));
ptr += 16;
size -= 16;
}
if (size >= 8) {
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
ptr += 8;
size -= 8;
}
crc = (uint32_t)crc64;
#else
while (size >= 4) {
while (size >= 16) {
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4)));
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 8)));
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 12)));
ptr += 16;
size -= 16;
}
if (size >= 8) {
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4)));
ptr += 8;
size -= 8;
}
#endif
if (size >= 4) {
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
ptr += 4;
size -= 4;
}
#endif
while (size) {
if (size >= 2) {
asm ("crc32w %1, %0\n" : "+r" (crc) : "m" (*(const uint16_t*)ptr));
ptr += 2;
size -= 2;
}
if (size) {
asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr));
++ptr;
--size;
}
return crc;