diff --git a/cmdline/portable.h b/cmdline/portable.h index 8dacbbd1..4beaad40 100644 --- a/cmdline/portable.h +++ b/cmdline/portable.h @@ -79,6 +79,10 @@ #define CONFIG_X86 1 #define CONFIG_X86_64 1 #endif + +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) +#define CONFIG_ARM_CRC 1 +#endif #endif /** diff --git a/cmdline/speed.c b/cmdline/speed.c index 9653f084..056bb123 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -101,11 +101,11 @@ void speed_crc(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); printf("\n"); - printf("%8s", "intel"); - fflush(stdout); - #if CONFIG_X86 if (raid_cpu_has_crc32()) { + printf("%8s", "intel"); + fflush(stdout); + SPEED_START { for (j = 0; j < nd; ++j) side_effect += crc32c_x86(0, v[j], size); @@ -115,6 +115,20 @@ void speed_crc(int nd, void** v, int size, int delta, int period) printf("\n"); } #endif + +#if CONFIG_ARM_CRC + printf("%8s", "arm"); + fflush(stdout); + + SPEED_START { + for (j = 0; j < nd; ++j) + side_effect += crc32c_arm64(0, v[j], size); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + printf("\n"); +#endif + printf("\n"); } diff --git a/cmdline/util.c b/cmdline/util.c index 1a9db356..c412a9b0 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -426,6 +426,19 @@ uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) } #endif +#if CONFIG_ARM_CRC +uint32_t crc32c_arm64(uint32_t crc, const unsigned char* ptr, size_t size) +{ + crc ^= CRC_IV; + + crc = crc32c_arm64_plain(crc, ptr, size); + + crc ^= CRC_IV; + + return crc; +} +#endif + uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, size_t size); void crc32c_init(void) diff --git a/cmdline/util.h b/cmdline/util.h index cc974d8d..3b056daf 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -77,7 +77,12 @@ static inline uint32_t crc32c_plain_char(uint32_t crc, unsigned char c) return crc; } #endif +#if CONFIG_ARM_CRC + asm ("crc32cb %w0, %w0, %w1\n" : "+r" (crc) : "r" (c)); + return crc; +#else return CRC32C_0[(crc ^ c) & 0xff] ^ (crc >> 8); +#endif } /** @@ -132,6 +137,33 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, } #endif +#if CONFIG_ARM_CRC +static inline uint32_t crc32c_arm64_plain(uint32_t crc, const unsigned char* ptr, size_t size) +{ + while (size >= 8) { + uint64_t val; + __builtin_memcpy(&val, ptr, 8); + asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc) : "r" (val)); + ptr += 8; + size -= 8; + } + if (size >= 4) { + uint32_t val; + __builtin_memcpy(&val, ptr, 4); + asm ("crc32cw %w0, %w0, %w1\n" : "+r" (crc) : "r" (val)); + ptr += 4; + size -= 4; + } + while (size) { + asm ("crc32cb %w0, %w0, %w1\n" : "+r" (crc) : "r" (*ptr)); + ++ptr; + --size; + } + + return crc; +} +#endif + /** * Compute CRC-32 (Castagnoli) without the IV. */ @@ -142,7 +174,11 @@ static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, size return crc32c_x86_plain(crc, ptr, size); } #endif +#if CONFIG_ARM_CRC + return crc32c_arm64_plain(crc, ptr, size); +#else return crc32c_gen_plain(crc, ptr, size); +#endif } /** @@ -155,6 +191,7 @@ extern uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, size_t size); */ uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, size_t size); uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size); +uint32_t crc32c_arm64(uint32_t crc, const unsigned char* ptr, size_t size); /** * Initialize the CRC-32 (Castagnoli) support.