From 0ba7e649d13c83c97c9d47b701401475346b1034 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 8 Oct 2015 22:09:12 +0200 Subject: [PATCH] Better optimized CRC call when writing the content file This saves about 15% of CPU time in writing. --- cmdline/stream.c | 3 +- cmdline/stream.h | 2 +- cmdline/util.c | 47 ++++------------------------ cmdline/util.h | 81 +++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 83 insertions(+), 50 deletions(-) diff --git a/cmdline/stream.c b/cmdline/stream.c index 6efd4948..82849022 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -562,9 +562,10 @@ int swrite(const void* void_data, unsigned size, STREAM* f) /* optimized version with all the data in memory */ unsigned char* pos = sptrget(f); + f->crc_stream = crc32c_plain(f->crc_stream, data, size); + /* copy it */ while (size--) { - f->crc_stream = crc32c_plain(f->crc_stream, *data); *pos++ = *data++; } diff --git a/cmdline/stream.h b/cmdline/stream.h index cd4ec117..885f85ec 100644 --- a/cmdline/stream.h +++ b/cmdline/stream.h @@ -345,7 +345,7 @@ static inline int sputc(int c, STREAM* s) if (sflush(s) != 0) return -1; } - s->crc_stream = crc32c_plain(s->crc_stream, c); + s->crc_stream = crc32c_plain_char(s->crc_stream, c); *s->pos++ = c; return 0; } diff --git a/cmdline/util.c b/cmdline/util.c index 0af8a8b1..1e763269 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -350,22 +350,15 @@ uint32_t CRC32C_3[256] = { 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 }; +#if HAVE_SSE42 +int crc_x86; +#endif + uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, unsigned size) { crc ^= CRC_IV; - while (size >= 4) { - crc ^= ptr[0] | (uint32_t)ptr[1] << 8 | (uint32_t)ptr[2] << 16 | (uint32_t)ptr[3] << 24; - crc = CRC32C_3[crc & 0xff] ^ CRC32C_2[(crc >> 8) & 0xff] ^ CRC32C_1[(crc >> 16) & 0xff] ^ CRC32C_0[crc >> 24]; - ptr += 4; - size -= 4; - } - - while (size) { - crc = CRC32C_0[(crc ^ *ptr) & 0xff] ^ (crc >> 8); - ++ptr; - --size; - } + crc = crc32c_gen_plain(crc, ptr, size); crc ^= CRC_IV; @@ -375,33 +368,9 @@ uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, unsigned size) #if HAVE_SSE42 uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, unsigned size) { -#ifdef CONFIG_X86_64 - uint64_t crc64; -#endif - crc ^= CRC_IV; -#ifdef CONFIG_X86_64 - crc64 = crc; - while (size >= 8) { - asm("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); - ptr += 8; - size -= 8; - } - crc = crc64; -#else - while (size >= 4) { - asm("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr)); - ptr += 4; - size -= 4; - } -#endif - - while (size) { - asm("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr)); - ++ptr; - --size; - } + crc = crc32c_x86_plain(crc, ptr, size); crc ^= CRC_IV; @@ -411,10 +380,6 @@ uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, unsigned size) uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size); -#if HAVE_SSE42 -int crc_x86; -#endif - void crc32c_init(void) { crc32c = crc32c_gen; diff --git a/cmdline/util.h b/cmdline/util.h index c12eb976..43656f1a 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -61,33 +61,100 @@ extern uint32_t CRC32C_1[256]; extern uint32_t CRC32C_2[256]; extern uint32_t CRC32C_3[256]; -#if HAVE_SSE42 /** * If the CPU support the CRC instructions. */ +#if HAVE_SSE42 extern int crc_x86; #endif /** - * Compute CRC-32 (Castagnoli) for a single byte without IV. + * Compute CRC-32 (Castagnoli) for a single byte without the IV. */ -static inline uint32_t crc32c_plain(uint32_t crc, unsigned char c) +static inline uint32_t crc32c_plain_char(uint32_t crc, unsigned char c) { #if HAVE_SSE42 if (tommy_likely(crc_x86)) { asm("crc32b %1, %0\n" : "+r" (crc) : "m" (c)); return crc; - } else -#endif - { - return CRC32C_0[(crc ^ c) & 0xff] ^ (crc >> 8); } +#endif + return CRC32C_0[(crc ^ c) & 0xff] ^ (crc >> 8); +} + +/** + * Compute the CRC-32 (Castagnoli) without the IV. + */ +static inline uint32_t crc32c_gen_plain(uint32_t crc, const unsigned char* ptr, unsigned size) +{ + while (size >= 4) { + crc ^= ptr[0] | (uint32_t)ptr[1] << 8 | (uint32_t)ptr[2] << 16 | (uint32_t)ptr[3] << 24; + crc = CRC32C_3[crc & 0xff] ^ CRC32C_2[(crc >> 8) & 0xff] ^ CRC32C_1[(crc >> 16) & 0xff] ^ CRC32C_0[crc >> 24]; + ptr += 4; + size -= 4; + } + + while (size) { + crc = CRC32C_0[(crc ^ *ptr) & 0xff] ^ (crc >> 8); + ++ptr; + --size; + } + + return crc; +} + +/** + * Compute the CRC-32 (Castagnoli) without the IV. + */ +#if HAVE_SSE42 +static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, unsigned size) +{ +#ifdef CONFIG_X86_64 + uint64_t crc64 = crc; + while (size >= 8) { + asm("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr)); + ptr += 8; + size -= 8; + } + crc = crc64; +#else + while (size >= 4) { + asm("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr)); + ptr += 4; + size -= 4; + } +#endif + while (size) { + asm("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr)); + ++ptr; + --size; + } + + return crc; +} +#endif + +/** + * Compute CRC-32 (Castagnoli) without the IV. + */ +static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, unsigned size) +{ +#if HAVE_SSE42 + if (tommy_likely(crc_x86)) { + return crc32c_x86_plain(crc, ptr, size); + } +#endif + return crc32c_gen_plain(crc, ptr, size); } /** * Compute the CRC-32 (Castagnoli) */ uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size); + +/** + * Internal entry points for testing. + */ uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, unsigned size); uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, unsigned size);