// SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2026 Andrea Mazzoleni /* * Derivative work of the XXH3-128 v8.4 implementation. * * This file implements the XXH3 long accumulator mode with support for * arbitrary input lengths. Unlike the original XXH3, which switches to separate * small/midsize routines for len <= 240 bytes, this implementation extends the * long mode: * - For len >= 64 bytes, it uses the striped accumulator loop directly. * - For len < 64 bytes, the input is zero-padded to 64 bytes, and the original * length is folded into the finalizer (preventing padding collisions). * * The motivation for extending the long mode to all lengths rather than using * the upstream short/midsize routines is that SnapRAID operates primarily on * large data blocks (typically 256 KB) where SIMD-accelerated long mode delivers * maximum throughput. Short inputs occur only as rare edge cases (such as * empty or small files), and maintaining a single code path reduces algorithmic * complexity and defect potential. * * seed points to 16 bytes, interpreted as two little-endian uint64_t words: * seed_lo then seed_hi. * - An all-zero seed uses the unmodified standard XXH3 secret. * - For len > 240, setting seed_lo == seed_hi matches standard * XXH3_128bits_withSeed() with that 64-bit seed value. * - When seed_lo != seed_hi, the 128-bit seed extends the secret by adding * seed_lo to even 64-bit words and subtracting seed_hi from odd words. * * For len <= 240, the output is not bit-compatible with original xxHash due to * using the padded long mode rather than the upstream short/midsize routines. */ /* * xxHash - Extremely Fast Hash algorithm * Development source file for `xxh3` * Copyright (C) 2019-2021 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at: * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ /** * Standalone SMHasher3 adapter for cmdline/xxh3.c. * Both entry points retain SnapRAID's canonical little-endian reads and output. * SMHasher3's 64-bit seed is duplicated into the two 64-bit seed halves. */ #include "Platform.h" #include "Hashlib.h" #include "Mathmult.h" /** * Change the #if values here to select an implementation manually. * SIMD entry points require a compatible CPU; there is no runtime dispatch. * The target attributes below preserve the existing SMHasher3 compiler flags. */ #if 0 #define XXH3_SNAPRAID_HASH_FN xxh3_128 #define XXH3_SNAPRAID_IMPL "scalar" #elif 0 #define XXH3_SNAPRAID_HASH_FN xxh3_128_avx512 #define XXH3_SNAPRAID_IMPL "avx512" #elif 0 #define XXH3_SNAPRAID_HASH_FN xxh3_128_avx2 #define XXH3_SNAPRAID_IMPL "avx2" #elif defined(__i386__) || defined(__x86_64__) #define XXH3_SNAPRAID_HASH_FN xxh3_128_sse2 #define XXH3_SNAPRAID_IMPL "sse2" #elif defined(__ARM_NEON) || defined(__ARM_NEON__) #define XXH3_SNAPRAID_HASH_FN xxh3_128_neon #define XXH3_SNAPRAID_IMPL "neon" #else #define XXH3_SNAPRAID_HASH_FN xxh3_128 #define XXH3_SNAPRAID_IMPL "scalar" #endif static FORCE_INLINE uint64_t xxh3_read64(const uint8_t* ptr) { return COND_BSWAP(GET_U64(ptr, 0), isBE()); } static FORCE_INLINE void xxh3_write64(uint8_t* ptr, uint64_t value) { PUT_U64(COND_BSWAP(value, isBE()), ptr, 0); } #if defined(__i386__) || defined(__x86_64__) #include #define XXH3_TARGET_SSE2 __attribute__((target("sse2"))) #define XXH3_TARGET_AVX2 __attribute__((target("avx2"))) #define XXH3_TARGET_AVX512 __attribute__((target("avx512f"))) #endif #if defined(__ARM_NEON) || defined(__ARM_NEON__) #include #endif #define XXH3_STRIPE_LEN 64 #define XXH3_SECRET_SIZE 192 #define XXH3_SECRET_CONSUME_RATE 8 #define XXH3_ACC_NB 8 #define XXH3_STRIPES_PER_BLOCK 16 #define XXH3_BLOCK_LEN 1024 #define XXH3_LAST_SECRET_OFFSET 121 #define XXH3_MERGE_OFFSET 11 #define XXH_PRIME32_1 0x9E3779B1U #define XXH_PRIME32_2 0x85EBCA77U #define XXH_PRIME32_3 0xC2B2AE3DU #define XXH_PRIME64_1 0x9E3779B185EBCA87ULL #define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL #define XXH_PRIME64_3 0x165667B19E3779F9ULL #define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL #define XXH_PRIME64_5 0x27D4EB2F165667C5ULL #define XXH_PRIME_MX1 0x165667919E3779F9ULL #define XXH3_INIT_ACC_VALUES \ { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } alignas(64) static const uint8_t xxh3_secret[XXH3_SECRET_SIZE] = { 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e }; static FORCE_INLINE uint64_t xxh3_xorshift64(uint64_t x, unsigned shift) { return x ^ (x >> shift); } static FORCE_INLINE uint64_t xxh3_avalanche(uint64_t x) { x = xxh3_xorshift64(x, 37); x *= XXH_PRIME_MX1; return xxh3_xorshift64(x, 32); } static FORCE_INLINE uint64_t xxh3_mul128_fold64(uint64_t a, uint64_t b) { MathMult::mult64_128(a, b, a, b); return a ^ b; } /* * 128-bit seed extension. * * Standard XXH3 with a 64-bit seed transforms every 16-byte secret pair as: * * secret[even] += seed; * secret[odd] -= seed; * * Here the 16-byte seed is split into two little-endian 64-bit words and the * same construction is extended naturally to: * * secret[even] += seed_lo; * secret[odd] -= seed_hi; * * Therefore both halves of the seed affect every 16-byte secret pair. */ static void xxh3_make_seeded_secret(uint8_t* out, uint64_t seed_lo, uint64_t seed_hi) { for (size_t i = 0; i < XXH3_SECRET_SIZE; i += 16) { xxh3_write64(out + i, xxh3_read64(xxh3_secret + i) + seed_lo); xxh3_write64(out + i + 8, xxh3_read64(xxh3_secret + i + 8) - seed_hi); } } static FORCE_INLINE const uint8_t* xxh3_select_secret(const uint8_t* seed, uint8_t* custom) { uint64_t seed_lo = xxh3_read64(seed); uint64_t seed_hi = xxh3_read64(seed + 8); if ((seed_lo | seed_hi) == 0) return xxh3_secret; xxh3_make_seeded_secret(custom, seed_lo, seed_hi); return custom; } static FORCE_INLINE uint64_t xxh3_mix2accs(const uint64_t* acc, const uint8_t* secret) { return xxh3_mul128_fold64(acc[0] ^ xxh3_read64(secret), acc[1] ^ xxh3_read64(secret + 8)); } static FORCE_INLINE uint64_t xxh3_merge_accs(const uint64_t* acc, const uint8_t* secret, uint64_t start) { uint64_t h = start; for (unsigned i = 0; i < 4; ++i) h += xxh3_mix2accs(acc + 2 * i, secret + 16 * i); return xxh3_avalanche(h); } static FORCE_INLINE void xxh3_finalize128(const uint64_t* acc, const uint8_t* secret, size_t len, uint8_t* out) { uint64_t low64 = xxh3_merge_accs(acc, secret + XXH3_MERGE_OFFSET, (uint64_t)len * XXH_PRIME64_1); uint64_t high64 = xxh3_merge_accs(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN - XXH3_MERGE_OFFSET, ~((uint64_t)len * XXH_PRIME64_2)); xxh3_write64(out, low64); xxh3_write64(out + 8, high64); } static FORCE_INLINE void xxh3_acc512_c(uint64_t* acc, const uint8_t* input, const uint8_t* secret) { for (unsigned lane = 0; lane < XXH3_ACC_NB; ++lane) { uint64_t data = xxh3_read64(input + 8 * lane); uint64_t keyed = data ^ xxh3_read64(secret + 8 * lane); uint64_t product = (uint64_t)(uint32_t)keyed * (uint32_t)(keyed >> 32); acc[lane ^ 1] += data; /* exchange adjacent 64-bit lanes */ acc[lane] += product; } } static FORCE_INLINE void xxh3_scramble_c(uint64_t* acc, const uint8_t* secret) { for (unsigned lane = 0; lane < XXH3_ACC_NB; ++lane) { uint64_t x = xxh3_xorshift64(acc[lane], 47); x ^= xxh3_read64(secret + 8 * lane); acc[lane] = x * XXH_PRIME32_1; } } static FORCE_INLINE void xxh3_short_core(const void* input, size_t len, const uint8_t* secret, uint8_t* out) { alignas(64) uint8_t buf[XXH3_STRIPE_LEN]; alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES; memset(buf, 0, XXH3_STRIPE_LEN); if (len > 0) memcpy(buf, input, len); xxh3_acc512_c(acc, buf, secret + XXH3_LAST_SECRET_OFFSET); xxh3_finalize128(acc, secret, len, out); } /* * With the 192-byte default secret: * stripes per block = (192 - 64) / 8 = 16 * block size = 16 * 64 = 1024 bytes */ static FORCE_INLINE void xxh3_long_core_c(const void* input, size_t len, const uint8_t* secret, uint8_t* out) { const uint8_t* p = static_cast(input); alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES; size_t blocks = (len - 1) / XXH3_BLOCK_LEN; for (size_t b = 0; b < blocks; ++b) { const uint8_t* block = p + b * XXH3_BLOCK_LEN; for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s) xxh3_acc512_c(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_scramble_c(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN); } size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN; const uint8_t* block = p + blocks * XXH3_BLOCK_LEN; for (size_t s = 0; s < stripes; ++s) xxh3_acc512_c(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_acc512_c(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET); xxh3_finalize128(acc, secret, len, out); } /* * seed points to 16 bytes: seed_lo then seed_hi, both little-endian. * out receives low64 then high64, both little-endian. * * No CPU detection is performed. The caller must invoke only a SIMD entry * point supported by the current CPU. */ static void xxh3_128(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) { alignas(64) uint8_t custom[XXH3_SECRET_SIZE]; const uint8_t* secret; secret = xxh3_select_secret(seed, custom); if (unlikely(len < XXH3_STRIPE_LEN)) xxh3_short_core(bytes, len, secret, out); else xxh3_long_core_c(bytes, len, secret, out); } #if defined(__ARM_NEON) || defined(__ARM_NEON__) static FORCE_INLINE uint64x2_t xxh3_vld1q_u64(const void* ptr) { return vreinterpretq_u64_u8(vld1q_u8((const uint8_t*)ptr)); } static FORCE_INLINE uint64x2_t xxh3_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); } static FORCE_INLINE uint64x2_t xxh3_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { #if defined(__aarch64__) return vmlal_high_u32(acc, lhs, rhs); #else return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); #endif } static FORCE_INLINE void xxh3_acc512_neon(uint64_t* acc, const uint8_t* input, const uint8_t* secret) { for (unsigned i = 0; i < 4; i += 2) { uint64x2_t data_vec_1 = xxh3_vld1q_u64(input + i * 16); uint64x2_t data_vec_2 = xxh3_vld1q_u64(input + (i + 1) * 16); uint64x2_t key_vec_1 = xxh3_vld1q_u64(secret + i * 16); uint64x2_t key_vec_2 = xxh3_vld1q_u64(secret + (i + 1) * 16); uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); uint32x4x2_t unzipped = vuzpq_u32( vreinterpretq_u32_u64(data_key_1), vreinterpretq_u32_u64(data_key_2) ); uint32x4_t data_key_lo = unzipped.val[0]; uint32x4_t data_key_hi = unzipped.val[1]; uint64x2_t sum_1 = xxh3_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); uint64x2_t sum_2 = xxh3_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); /* typed loads and stores preserve scalar lane order on big-endian ARM */ vst1q_u64(acc + i * 2, vaddq_u64(vld1q_u64(acc + i * 2), sum_1)); vst1q_u64(acc + (i + 1) * 2, vaddq_u64(vld1q_u64(acc + (i + 1) * 2), sum_2)); } } static FORCE_INLINE void xxh3_scramble_neon(uint64_t* acc, const uint8_t* secret) { const uint32x2_t prime_lo = vdup_n_u32(XXH_PRIME32_1); const uint32x4_t prime_hi = vreinterpretq_u32_u64(vdupq_n_u64((uint64_t)XXH_PRIME32_1 << 32)); for (unsigned i = 0; i < 4; ++i) { uint64x2_t acc_vec = vld1q_u64(acc + i * 2); uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); uint64x2_t data_vec = veorq_u64(acc_vec, shifted); uint64x2_t key_vec = xxh3_vld1q_u64(secret + i * 16); uint64x2_t data_key = veorq_u64(data_vec, key_vec); uint32x4_t prod_hi = vmulq_u32(vreinterpretq_u32_u64(data_key), prime_hi); uint32x2_t data_key_lo = vmovn_u64(data_key); vst1q_u64(acc + i * 2, vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, prime_lo)); } } static FORCE_INLINE void xxh3_long_core_neon(const void* input, size_t len, const uint8_t* secret, uint8_t* out) { const uint8_t* p = static_cast(input); alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES; size_t blocks = (len - 1) / XXH3_BLOCK_LEN; for (size_t b = 0; b < blocks; ++b) { const uint8_t* block = p + b * XXH3_BLOCK_LEN; for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s) xxh3_acc512_neon(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_scramble_neon(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN); } size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN; const uint8_t* block = p + blocks * XXH3_BLOCK_LEN; for (size_t s = 0; s < stripes; ++s) xxh3_acc512_neon(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_acc512_neon(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET); xxh3_finalize128(acc, secret, len, out); } static void xxh3_128_neon(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) { alignas(64) uint8_t custom[XXH3_SECRET_SIZE]; const uint8_t* secret; secret = xxh3_select_secret(seed, custom); if (unlikely(len < XXH3_STRIPE_LEN)) xxh3_short_core(bytes, len, secret, out); else xxh3_long_core_neon(bytes, len, secret, out); } #endif #if defined(__i386__) || defined(__x86_64__) static XXH3_TARGET_SSE2 FORCE_INLINE void xxh3_acc512_sse2(uint64_t* acc, const uint8_t* input, const uint8_t* secret) { __m128i* a = (__m128i*)(void*)acc; for (unsigned i = 0; i < 4; ++i) { __m128i data = _mm_loadu_si128((const __m128i*)(const void*)(input + 16 * i)); __m128i key = _mm_loadu_si128((const __m128i*)(const void*)(secret + 16 * i)); __m128i mix = _mm_xor_si128(data, key); __m128i hi32 = _mm_srli_epi64(mix, 32); __m128i prod = _mm_mul_epu32(mix, hi32); __m128i swap = _mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)); a[i] = _mm_add_epi64(_mm_add_epi64(a[i], swap), prod); } } static XXH3_TARGET_SSE2 FORCE_INLINE void xxh3_scramble_sse2(uint64_t* acc, const uint8_t* secret) { __m128i* a = (__m128i*)(void*)acc; const __m128i prime = _mm_set1_epi32((int)XXH_PRIME32_1); for (unsigned i = 0; i < 4; ++i) { __m128i x = a[i]; __m128i key = _mm_loadu_si128((const __m128i*)(const void*)(secret + 16 * i)); x = _mm_xor_si128(x, _mm_srli_epi64(x, 47)); x = _mm_xor_si128(x, key); /* 64-bit x * 32-bit prime, modulo 2^64, using SSE2 32x32 multiplies. */ __m128i lo = _mm_mul_epu32(x, prime); __m128i hi = _mm_mul_epu32(_mm_srli_epi64(x, 32), prime); a[i] = _mm_add_epi64(lo, _mm_slli_epi64(hi, 32)); } } static XXH3_TARGET_AVX2 FORCE_INLINE void xxh3_acc512_avx2(uint64_t* acc, const uint8_t* input, const uint8_t* secret) { __m256i* a = (__m256i*)(void*)acc; for (unsigned i = 0; i < 2; ++i) { __m256i data = _mm256_loadu_si256((const __m256i*)(const void*)(input + 32 * i)); __m256i key = _mm256_loadu_si256((const __m256i*)(const void*)(secret + 32 * i)); __m256i mix = _mm256_xor_si256(data, key); __m256i hi32 = _mm256_srli_epi64(mix, 32); __m256i prod = _mm256_mul_epu32(mix, hi32); __m256i swap = _mm256_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2)); a[i] = _mm256_add_epi64(_mm256_add_epi64(a[i], swap), prod); } } static XXH3_TARGET_AVX2 FORCE_INLINE void xxh3_scramble_avx2(uint64_t* acc, const uint8_t* secret) { __m256i* a = (__m256i*)(void*)acc; const __m256i prime = _mm256_set1_epi32((int)XXH_PRIME32_1); for (unsigned i = 0; i < 2; ++i) { __m256i x = a[i]; __m256i key = _mm256_loadu_si256((const __m256i*)(const void*)(secret + 32 * i)); x = _mm256_xor_si256(x, _mm256_srli_epi64(x, 47)); x = _mm256_xor_si256(x, key); __m256i lo = _mm256_mul_epu32(x, prime); __m256i hi = _mm256_mul_epu32(_mm256_srli_epi64(x, 32), prime); a[i] = _mm256_add_epi64(lo, _mm256_slli_epi64(hi, 32)); } } static XXH3_TARGET_AVX512 FORCE_INLINE void xxh3_acc512_avx512(uint64_t* acc, const uint8_t* input, const uint8_t* secret) { __m512i* a = (__m512i*)(void*)acc; __m512i data = _mm512_loadu_si512((const void*)input); __m512i key = _mm512_loadu_si512((const void*)secret); __m512i mix = _mm512_xor_si512(data, key); __m512i prod = _mm512_mul_epu32(mix, _mm512_srli_epi64(mix, 32)); __m512i swap = _mm512_shuffle_epi32(data, _MM_PERM_BADC); *a = _mm512_add_epi64(_mm512_add_epi64(*a, swap), prod); } static XXH3_TARGET_AVX512 FORCE_INLINE void xxh3_scramble_avx512(uint64_t* acc, const uint8_t* secret) { __m512i* a = (__m512i*)(void*)acc; const __m512i prime = _mm512_set1_epi32((int)XXH_PRIME32_1); __m512i x = *a; __m512i key = _mm512_loadu_si512((const void*)secret); __m512i shifted = _mm512_srli_epi64(x, 47); x = _mm512_ternarylogic_epi32(key, x, shifted, 0x96); __m512i lo = _mm512_mul_epu32(x, prime); __m512i hi = _mm512_mul_epu32(_mm512_srli_epi64(x, 32), prime); *a = _mm512_add_epi64(lo, _mm512_slli_epi64(hi, 32)); } static XXH3_TARGET_SSE2 FORCE_INLINE void xxh3_long_core_sse2(const void* input, size_t len, const uint8_t* secret, uint8_t* out) { const uint8_t* p = static_cast(input); alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES; size_t blocks = (len - 1) / XXH3_BLOCK_LEN; for (size_t b = 0; b < blocks; ++b) { const uint8_t* block = p + b * XXH3_BLOCK_LEN; for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s) xxh3_acc512_sse2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_scramble_sse2(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN); } size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN; const uint8_t* block = p + blocks * XXH3_BLOCK_LEN; for (size_t s = 0; s < stripes; ++s) xxh3_acc512_sse2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_acc512_sse2(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET); xxh3_finalize128(acc, secret, len, out); } static XXH3_TARGET_AVX2 FORCE_INLINE void xxh3_long_core_avx2(const void* input, size_t len, const uint8_t* secret, uint8_t* out) { const uint8_t* p = static_cast(input); alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES; size_t blocks = (len - 1) / XXH3_BLOCK_LEN; for (size_t b = 0; b < blocks; ++b) { const uint8_t* block = p + b * XXH3_BLOCK_LEN; for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s) xxh3_acc512_avx2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_scramble_avx2(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN); } size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN; const uint8_t* block = p + blocks * XXH3_BLOCK_LEN; for (size_t s = 0; s < stripes; ++s) xxh3_acc512_avx2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_acc512_avx2(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET); xxh3_finalize128(acc, secret, len, out); } static XXH3_TARGET_AVX512 FORCE_INLINE void xxh3_long_core_avx512(const void* input, size_t len, const uint8_t* secret, uint8_t* out) { const uint8_t* p = static_cast(input); alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES; size_t blocks = (len - 1) / XXH3_BLOCK_LEN; for (size_t b = 0; b < blocks; ++b) { const uint8_t* block = p + b * XXH3_BLOCK_LEN; for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s) xxh3_acc512_avx512(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_scramble_avx512(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN); } size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN; const uint8_t* block = p + blocks * XXH3_BLOCK_LEN; for (size_t s = 0; s < stripes; ++s) xxh3_acc512_avx512(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE); xxh3_acc512_avx512(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET); xxh3_finalize128(acc, secret, len, out); } static XXH3_TARGET_SSE2 void xxh3_128_sse2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) { alignas(64) uint8_t custom[XXH3_SECRET_SIZE]; const uint8_t* secret; secret = xxh3_select_secret(seed, custom); if (unlikely(len < XXH3_STRIPE_LEN)) xxh3_short_core(bytes, len, secret, out); else xxh3_long_core_sse2(bytes, len, secret, out); } static XXH3_TARGET_AVX2 void xxh3_128_avx2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) { alignas(64) uint8_t custom[XXH3_SECRET_SIZE]; const uint8_t* secret; secret = xxh3_select_secret(seed, custom); if (unlikely(len < XXH3_STRIPE_LEN)) xxh3_short_core(bytes, len, secret, out); else xxh3_long_core_avx2(bytes, len, secret, out); } static XXH3_TARGET_AVX512 void xxh3_128_avx512(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) { alignas(64) uint8_t custom[XXH3_SECRET_SIZE]; const uint8_t* secret; secret = xxh3_select_secret(seed, custom); if (unlikely(len < XXH3_STRIPE_LEN)) xxh3_short_core(bytes, len, secret, out); else xxh3_long_core_avx512(bytes, len, secret, out); } #endif static void xxh3_snapraid(const void* in, size_t len, seed_t seed, void* out) { uint8_t seed_bytes[16]; /* duplicate the seed, matching the other SnapRAID adapters */ xxh3_write64(seed_bytes, seed); xxh3_write64(seed_bytes + 8, seed); XXH3_SNAPRAID_HASH_FN(in, len, seed_bytes, static_cast(out)); } REGISTER_FAMILY(xxh3_snapraid, $.src_url = "https://github.com/amadvance/snapraid/", $.src_status = HashFamilyInfo::SRC_ACTIVE ); REGISTER_HASH(XXH3_128_snapraid, $.desc = "XXH3-128, SnapRAID long mode for all input lengths", $.impl = XXH3_SNAPRAID_IMPL, $.hash_flags = FLAG_HASH_ENDIAN_INDEPENDENT, $.impl_flags = FLAG_IMPL_MULTIPLY_64_128 | FLAG_IMPL_CANONICAL_BOTH | FLAG_IMPL_LICENSE_GPL3, $.bits = 128, $.verification_LE = 0xCFAD116F, $.verification_BE = 0xCFAD116F, $.hashfn_native = xxh3_snapraid, $.hashfn_bswap = xxh3_snapraid );