diff --git a/cmdline/selftest.c b/cmdline/selftest.c index b0cd9daa..5d7b8f65 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -267,6 +267,7 @@ static struct crc_test_vector TEST_CRC32C[] = { { "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 32, 0x62a8ab43 }, { "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18\x17\x16\x15\x14\x13\x12\x11\x10\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08\x07\x06\x05\x04\x03\x02\x01\x00", 32, 0x113fdb5c }, { "\x01\xc0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x14\x00\x00\x00\x18\x28\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00", 48, 0xd9963a56 }, + { "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 16, 0xd9c908eb }, { "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", 32, 0x46dd794e }, { "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28", 40, 0x0e2c157f }, { "\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50", 40, 0xe980ebf6 }, diff --git a/cmdline/util.h b/cmdline/util.h index 3b6a3abb..9db2c402 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -146,23 +146,62 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, { #ifdef CONFIG_X86_64 uint64_t crc64 = crc; - while (size >= 8) { + + /* block hashes are normally 16 bytes and dominate content serialization */ + if (tommy_likely(size == 16)) { + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8))); + return (uint32_t)crc64; + } + + while (size >= 32) { + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8))); + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 16))); + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 24))); + ptr += 32; + size -= 32; + } + if (size >= 16) { + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); + asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8))); + ptr += 16; + size -= 16; + } + if (size >= 8) { asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); ptr += 8; size -= 8; } crc = (uint32_t)crc64; #else - while (size >= 4) { + while (size >= 16) { + asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr)); + asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4))); + asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 8))); + asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 12))); + ptr += 16; + size -= 16; + } + if (size >= 8) { + asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr)); + asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4))); + ptr += 8; + size -= 8; + } +#endif + if (size >= 4) { asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr)); ptr += 4; size -= 4; } -#endif - while (size) { + if (size >= 2) { + asm ("crc32w %1, %0\n" : "+r" (crc) : "m" (*(const uint16_t*)ptr)); + ptr += 2; + size -= 2; + } + if (size) { asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr)); - ++ptr; - --size; } return crc;