mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-25 18:40:19 -04:00
Add experimental XXH3 hash implementation with SIMD acceleration
Memory write speed using the C memset() function:
memset 57952
Hash used to check the data blocks integrity:
best murmur3 spooky2 museair xxh3 xsse2 xavx2
hash spooky2 6172 17914 32048 14627 23613 44034
This commit is contained in:
Andrea Mazzoleni
parent
f0ae584473
commit
cea9015933
17 changed files with 5498 additions and 13 deletions
|
|
@ -117,6 +117,8 @@ noinst_HEADERS = \
|
|||
cmdline/spooky2test.c \
|
||||
cmdline/museair.c \
|
||||
cmdline/museairtest.c \
|
||||
cmdline/xxh3.c \
|
||||
cmdline/xxh3test.c \
|
||||
cmdline/import.h \
|
||||
cmdline/search.h \
|
||||
cmdline/thermal.h \
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ void test(int advise_mode)
|
|||
uint32_t put_crc_stored;
|
||||
|
||||
crc32c_init();
|
||||
hash_init();
|
||||
|
||||
s = sopen_multi_write(STREAM_MAX, advise_mode | STREAM_FLAGS_CRC);
|
||||
for (i = 0; i < STREAM_MAX; ++i) {
|
||||
|
|
|
|||
|
|
@ -153,7 +153,15 @@ static struct hash_test_vector TEST_MUSEAIR[] = {
|
|||
{ 0, 0, { 0 }, { 0 } }
|
||||
};
|
||||
|
||||
#define HASH_TEST_MAX 512 /* tests are never longer than 512 bytes */
|
||||
/**
|
||||
* Test vectors for XXH3
|
||||
*/
|
||||
static struct hash_test_vector TEST_XXH3[] = {
|
||||
#include "xxh3test.c"
|
||||
{ 0, 0, { 0 }, { 0 } }
|
||||
};
|
||||
|
||||
#define HASH_TEST_MAX 4096 /* tests are never longer than 4096 bytes */
|
||||
|
||||
static void test_hash(void)
|
||||
{
|
||||
|
|
@ -241,6 +249,69 @@ static void test_hash(void)
|
|||
}
|
||||
}
|
||||
|
||||
for (i = 0; TEST_XXH3[i].data; ++i) {
|
||||
unsigned char digest[HASH_MAX];
|
||||
memcpy(buffer_aligned, TEST_XXH3[i].data, TEST_XXH3[i].len);
|
||||
memcpy(seed_aligned, TEST_XXH3[i].seed, HASH_MAX);
|
||||
|
||||
xxh3_128(buffer_aligned, TEST_XXH3[i].len, seed_aligned, digest);
|
||||
if (memcmp(digest, TEST_XXH3[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed XXH3 test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
#if CONFIG_X86
|
||||
if (raid_cpu_has_sse2()) {
|
||||
xxh3_128_sse2(buffer_aligned, TEST_XXH3[i].len, seed_aligned, digest);
|
||||
if (memcmp(digest, TEST_XXH3[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed XXH3 SSE2 test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
if (raid_cpu_has_avx2()) {
|
||||
xxh3_128_avx2(buffer_aligned, TEST_XXH3[i].len, seed_aligned, digest);
|
||||
if (memcmp(digest, TEST_XXH3[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed XXH3 AVX2 test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
if (raid_cpu_has_avx512f()) {
|
||||
xxh3_128_avx512(buffer_aligned, TEST_XXH3[i].len, seed_aligned, digest);
|
||||
if (memcmp(digest, TEST_XXH3[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed XXH3 AVX512 test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
xxh3_128_neon(buffer_aligned, TEST_XXH3[i].len, seed_aligned, digest);
|
||||
if (memcmp(digest, TEST_XXH3[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed XXH3 NEON test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
#endif
|
||||
|
||||
memhash(HASH_XXH3, seed_aligned, digest, buffer_aligned, TEST_XXH3[i].len);
|
||||
if (memcmp(digest, TEST_XXH3[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed XXH3 memhash test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
free(buffer_alloc);
|
||||
free(seed_alloc);
|
||||
}
|
||||
|
|
@ -2924,6 +2995,7 @@ void test(int argc, char* argv[])
|
|||
|
||||
lock_init();
|
||||
crc32c_init();
|
||||
hash_init();
|
||||
raid_init();
|
||||
|
||||
msg_progress("Test...\n");
|
||||
|
|
|
|||
|
|
@ -1067,6 +1067,7 @@ int snapraid_main(int argc, char* argv[])
|
|||
app_init();
|
||||
raid_init();
|
||||
crc32c_init();
|
||||
hash_init();
|
||||
random_reseed();
|
||||
|
||||
if (speedtest != 0) {
|
||||
|
|
|
|||
|
|
@ -151,6 +151,18 @@ void speed_hash(int nd, void** v, int size, int delta, int period)
|
|||
printf("%8s", "murmur3");
|
||||
printf("%8s", "spooky2");
|
||||
printf("%8s", "museair");
|
||||
printf("%8s", "xxh3");
|
||||
#if CONFIG_X86
|
||||
if (raid_cpu_has_sse2())
|
||||
printf("%8s", "xsse2");
|
||||
if (raid_cpu_has_avx2())
|
||||
printf("%8s", "xavx2");
|
||||
if (raid_cpu_has_avx512f())
|
||||
printf("%8s", "xavx512");
|
||||
#endif
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
printf("%8s", "xneon");
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
printf("%8s", "hash");
|
||||
|
|
@ -179,6 +191,57 @@ void speed_hash(int nd, void** v, int size, int delta, int period)
|
|||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
for (i = 0; i < nd; ++i)
|
||||
xxh3_128(v[i], size, seed, digest);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#if CONFIG_X86
|
||||
if (raid_cpu_has_sse2()) {
|
||||
SPEED_START {
|
||||
for (i = 0; i < nd; ++i)
|
||||
xxh3_128_sse2(v[i], size, seed, digest);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
for (i = 0; i < nd; ++i)
|
||||
xxh3_128_avx2(v[i], size, seed, digest);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
if (raid_cpu_has_avx512f()) {
|
||||
SPEED_START {
|
||||
for (i = 0; i < nd; ++i)
|
||||
xxh3_128_avx512(v[i], size, seed, digest);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
SPEED_START {
|
||||
for (i = 0; i < nd; ++i)
|
||||
xxh3_128_neon(v[i], size, seed, digest);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
|
||||
printf("\n");
|
||||
printf("\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -787,6 +787,7 @@ const char* memhashname(unsigned kind)
|
|||
case HASH_MURMUR3 : return "murmur3";
|
||||
case HASH_SPOOKY2 : return "spooky2";
|
||||
case HASH_MUSEAIR : return "museair";
|
||||
case HASH_XXH3 : return "xxh3";
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -810,6 +811,26 @@ unsigned membesthash(void)
|
|||
#include "murmur3.c"
|
||||
#include "spooky2.c"
|
||||
#include "museair.c"
|
||||
#include "xxh3.c"
|
||||
|
||||
static void (*xxh3)(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) = xxh3_128;
|
||||
|
||||
void hash_init(void)
|
||||
{
|
||||
xxh3 = xxh3_128;
|
||||
#if CONFIG_X86
|
||||
if (raid_cpu_has_avx512f()) {
|
||||
xxh3 = xxh3_128_avx512;
|
||||
} else if (raid_cpu_has_avx2()) {
|
||||
xxh3 = xxh3_128_avx2;
|
||||
} else if (raid_cpu_has_sse2()) {
|
||||
xxh3 = xxh3_128_sse2;
|
||||
}
|
||||
#endif
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
xxh3 = xxh3_128_neon;
|
||||
#endif
|
||||
}
|
||||
|
||||
void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size)
|
||||
{
|
||||
|
|
@ -823,6 +844,9 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void*
|
|||
case HASH_MUSEAIR :
|
||||
MuseAirLoong(src, size, seed, digest);
|
||||
break;
|
||||
case HASH_XXH3 :
|
||||
xxh3(src, size, seed, digest);
|
||||
break;
|
||||
default :
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Internal inconsistency in hash function %u\n", kind);
|
||||
|
|
@ -859,6 +883,7 @@ const char* hash_config_name(unsigned kind)
|
|||
case HASH_MURMUR3 : return "murmur3";
|
||||
case HASH_SPOOKY2 : return "spooky2";
|
||||
case HASH_MUSEAIR : return "museair";
|
||||
case HASH_XXH3 : return "xxh3";
|
||||
default :
|
||||
/* LCOV_EXCL_START */
|
||||
return "unknown";
|
||||
|
|
|
|||
|
|
@ -385,12 +385,26 @@ extern uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, size_t size);
|
|||
uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, size_t size);
|
||||
uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size);
|
||||
uint32_t crc32c_arm64(uint32_t crc, const unsigned char* ptr, size_t size);
|
||||
void xxh3_128(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out);
|
||||
#if CONFIG_X86
|
||||
void xxh3_128_sse2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out);
|
||||
void xxh3_128_avx2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out);
|
||||
void xxh3_128_avx512(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out);
|
||||
#endif
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
void xxh3_128_neon(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out);
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Initialize the CRC-32 (Castagnoli) support.
|
||||
*/
|
||||
void crc32c_init(void);
|
||||
|
||||
/**
|
||||
* Initialize the hash subsystem and select the best CPU backend.
|
||||
*/
|
||||
void hash_init(void);
|
||||
|
||||
/****************************************************************************/
|
||||
/* hash */
|
||||
|
||||
|
|
@ -406,6 +420,7 @@ void crc32c_init(void);
|
|||
#define HASH_MURMUR3 1
|
||||
#define HASH_SPOOKY2 2
|
||||
#define HASH_MUSEAIR 3
|
||||
#define HASH_XXH3 4
|
||||
|
||||
/**
|
||||
* Return the hash that is expected to be the fastest in this architecture
|
||||
|
|
@ -448,12 +463,12 @@ void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const
|
|||
*
|
||||
* Legacy hashes (Murmur3, Spooky2) preserve historical semantics and hash
|
||||
* only the logical file bytes.
|
||||
* Modern hashes (MuseAir) hash the complete zero-padded RAID block up to
|
||||
* Modern hashes (MuseAir, XXH3) hash the complete zero-padded RAID block up to
|
||||
* block_size.
|
||||
*/
|
||||
static inline int memhash_is_block(unsigned kind)
|
||||
{
|
||||
return kind == HASH_MUSEAIR;
|
||||
return kind == HASH_MUSEAIR || kind == HASH_XXH3;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
579
cmdline/xxh3.c
Normal file
579
cmdline/xxh3.c
Normal file
|
|
@ -0,0 +1,579 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Copyright (C) 2026 Andrea Mazzoleni
|
||||
|
||||
/*
|
||||
* Derivative work of the XXH3-128 v8.4 implementation.
|
||||
*
|
||||
* This file implements the XXH3 long accumulator mode with support for
|
||||
* arbitrary input lengths. Unlike the original XXH3, which switches to separate
|
||||
* small/midsize routines for len <= 240 bytes, this implementation extends the
|
||||
* long mode:
|
||||
* - For len >= 64 bytes, it uses the striped accumulator loop directly.
|
||||
* - For len < 64 bytes, the input is zero-padded to 64 bytes, and the original
|
||||
* length is folded into the finalizer (preventing padding collisions).
|
||||
*
|
||||
* The motivation for extending the long mode to all lengths rather than using
|
||||
* the upstream short/midsize routines is that SnapRAID operates primarily on
|
||||
* large data blocks (typically 256 KB) where SIMD-accelerated long mode delivers
|
||||
* maximum throughput. Short inputs occur only as rare edge cases (such as
|
||||
* empty or small files), and maintaining a single code path reduces algorithmic
|
||||
* complexity and defect potential.
|
||||
*
|
||||
* seed points to 16 bytes, interpreted as two little-endian uint64_t words:
|
||||
* seed_lo then seed_hi.
|
||||
* - An all-zero seed uses the unmodified standard XXH3 secret.
|
||||
* - For len > 240, setting seed_lo == seed_hi matches standard
|
||||
* XXH3_128bits_withSeed() with that 64-bit seed value.
|
||||
* - When seed_lo != seed_hi, the 128-bit seed extends the secret by adding
|
||||
* seed_lo to even 64-bit words and subtracting seed_hi from odd words.
|
||||
*
|
||||
* For len <= 240, the output is not bit-compatible with original xxHash due to
|
||||
* using the padded long mode rather than the upstream short/midsize routines.
|
||||
*/
|
||||
|
||||
/*
|
||||
* xxHash - Extremely Fast Hash algorithm
|
||||
* Development source file for `xxh3`
|
||||
* Copyright (C) 2019-2021 Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
#if CONFIG_X86
|
||||
#include <immintrin.h>
|
||||
|
||||
#define XXH3_TARGET_SSE2 __attribute__((target("sse2")))
|
||||
#define XXH3_TARGET_AVX2 __attribute__((target("avx2")))
|
||||
#define XXH3_TARGET_AVX512 __attribute__((target("avx512f")))
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
#define XXH3_STRIPE_LEN 64
|
||||
#define XXH3_SECRET_SIZE 192
|
||||
#define XXH3_SECRET_CONSUME_RATE 8
|
||||
#define XXH3_ACC_NB 8
|
||||
#define XXH3_STRIPES_PER_BLOCK 16
|
||||
#define XXH3_BLOCK_LEN 1024
|
||||
#define XXH3_LAST_SECRET_OFFSET 121
|
||||
#define XXH3_MERGE_OFFSET 11
|
||||
|
||||
#define XXH_PRIME32_1 0x9E3779B1U
|
||||
#define XXH_PRIME32_2 0x85EBCA77U
|
||||
#define XXH_PRIME32_3 0xC2B2AE3DU
|
||||
|
||||
#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL
|
||||
#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL
|
||||
#define XXH_PRIME64_3 0x165667B19E3779F9ULL
|
||||
#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL
|
||||
#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL
|
||||
|
||||
#define XXH_PRIME_MX1 0x165667919E3779F9ULL
|
||||
|
||||
#define XXH3_INIT_ACC_VALUES \
|
||||
{ XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
|
||||
XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
|
||||
|
||||
static const __attribute__((aligned(64))) uint8_t xxh3_secret[XXH3_SECRET_SIZE] = {
|
||||
0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
|
||||
0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
|
||||
0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
|
||||
0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
|
||||
0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
|
||||
0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
|
||||
0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
|
||||
0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
|
||||
0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
|
||||
0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
|
||||
0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
|
||||
0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e
|
||||
};
|
||||
|
||||
static __always_inline uint64_t xxh3_xorshift64(uint64_t x, unsigned shift)
|
||||
{
|
||||
return x ^ (x >> shift);
|
||||
}
|
||||
|
||||
static __always_inline uint64_t xxh3_avalanche(uint64_t x)
|
||||
{
|
||||
x = xxh3_xorshift64(x, 37);
|
||||
x *= XXH_PRIME_MX1;
|
||||
return xxh3_xorshift64(x, 32);
|
||||
}
|
||||
|
||||
static __always_inline uint64_t xxh3_mul128_fold64(uint64_t a, uint64_t b)
|
||||
{
|
||||
util_mum(&a, &b);
|
||||
return a ^ b;
|
||||
}
|
||||
|
||||
/*
|
||||
* 128-bit seed extension.
|
||||
*
|
||||
* Standard XXH3 with a 64-bit seed transforms every 16-byte secret pair as:
|
||||
*
|
||||
* secret[even] += seed;
|
||||
* secret[odd] -= seed;
|
||||
*
|
||||
* Here the 16-byte seed is split into two little-endian 64-bit words and the
|
||||
* same construction is extended naturally to:
|
||||
*
|
||||
* secret[even] += seed_lo;
|
||||
* secret[odd] -= seed_hi;
|
||||
*
|
||||
* Therefore both halves of the seed affect every 16-byte secret pair.
|
||||
*/
|
||||
static void xxh3_make_seeded_secret(uint8_t* out, uint64_t seed_lo, uint64_t seed_hi)
|
||||
{
|
||||
for (size_t i = 0; i < XXH3_SECRET_SIZE; i += 16) {
|
||||
util_write64(out + i, util_read64(xxh3_secret + i) + seed_lo);
|
||||
util_write64(out + i + 8, util_read64(xxh3_secret + i + 8) - seed_hi);
|
||||
}
|
||||
}
|
||||
|
||||
static __always_inline const uint8_t* xxh3_select_secret(const uint8_t* seed, uint8_t* custom)
|
||||
{
|
||||
uint64_t seed_lo = util_read64(seed);
|
||||
uint64_t seed_hi = util_read64(seed + 8);
|
||||
|
||||
if ((seed_lo | seed_hi) == 0)
|
||||
return xxh3_secret;
|
||||
|
||||
xxh3_make_seeded_secret(custom, seed_lo, seed_hi);
|
||||
return custom;
|
||||
}
|
||||
|
||||
static __always_inline uint64_t xxh3_mix2accs(const uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
return xxh3_mul128_fold64(acc[0] ^ util_read64(secret), acc[1] ^ util_read64(secret + 8));
|
||||
}
|
||||
|
||||
static __always_inline uint64_t xxh3_merge_accs(const uint64_t* acc, const uint8_t* secret, uint64_t start)
|
||||
{
|
||||
uint64_t h = start;
|
||||
for (unsigned i = 0; i < 4; ++i)
|
||||
h += xxh3_mix2accs(acc + 2 * i, secret + 16 * i);
|
||||
return xxh3_avalanche(h);
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_finalize128(const uint64_t* acc, const uint8_t* secret, size_t len, uint8_t* out)
|
||||
{
|
||||
uint64_t low64 = xxh3_merge_accs(acc, secret + XXH3_MERGE_OFFSET, (uint64_t)len * XXH_PRIME64_1);
|
||||
uint64_t high64 = xxh3_merge_accs(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN - XXH3_MERGE_OFFSET, ~((uint64_t)len * XXH_PRIME64_2));
|
||||
util_write64(out, low64);
|
||||
util_write64(out + 8, high64);
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_acc512_c(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
for (unsigned lane = 0; lane < XXH3_ACC_NB; ++lane) {
|
||||
uint64_t data = util_read64(input + 8 * lane);
|
||||
uint64_t keyed = data ^ util_read64(secret + 8 * lane);
|
||||
uint64_t product = (uint64_t)(uint32_t)keyed * (uint32_t)(keyed >> 32);
|
||||
|
||||
acc[lane ^ 1] += data; /* exchange adjacent 64-bit lanes */
|
||||
acc[lane] += product;
|
||||
}
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_scramble_c(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
for (unsigned lane = 0; lane < XXH3_ACC_NB; ++lane) {
|
||||
uint64_t x = xxh3_xorshift64(acc[lane], 47);
|
||||
x ^= util_read64(secret + 8 * lane);
|
||||
acc[lane] = x * XXH_PRIME32_1;
|
||||
}
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_short_core(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
__attribute__((aligned(64))) uint8_t buf[XXH3_STRIPE_LEN];
|
||||
__attribute__((aligned(64))) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
|
||||
memset(buf, 0, XXH3_STRIPE_LEN);
|
||||
if (len > 0)
|
||||
memcpy(buf, input, len);
|
||||
|
||||
xxh3_acc512_c(acc, buf, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
/*
|
||||
* With the 192-byte default secret:
|
||||
* stripes per block = (192 - 64) / 8 = 16
|
||||
* block size = 16 * 64 = 1024 bytes
|
||||
*/
|
||||
static __always_inline void xxh3_long_core_c(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = input;
|
||||
__attribute__((aligned(64))) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_c(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_c(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_c(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_c(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
/*
|
||||
* seed points to 16 bytes: seed_lo then seed_hi, both little-endian.
|
||||
* out receives low64 then high64, both little-endian.
|
||||
*
|
||||
* No CPU detection is performed. The caller must invoke only a SIMD entry
|
||||
* point supported by the current CPU.
|
||||
*/
|
||||
void xxh3_128(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
__attribute__((aligned(64))) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (tommy_unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_c(bytes, len, secret, out);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
static __always_inline uint64x2_t xxh3_vld1q_u64(const void* ptr)
|
||||
{
|
||||
return vreinterpretq_u64_u8(vld1q_u8((const uint8_t*)ptr));
|
||||
}
|
||||
|
||||
static __always_inline uint64x2_t xxh3_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
|
||||
{
|
||||
return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
|
||||
}
|
||||
|
||||
static __always_inline uint64x2_t xxh3_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
|
||||
{
|
||||
#if defined(CONFIG_NEON)
|
||||
return vmlal_high_u32(acc, lhs, rhs);
|
||||
#else
|
||||
return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
|
||||
#endif
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_acc512_neon(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
for (unsigned i = 0; i < 4; i += 2) {
|
||||
uint64x2_t data_vec_1 = xxh3_vld1q_u64(input + i * 16);
|
||||
uint64x2_t data_vec_2 = xxh3_vld1q_u64(input + (i + 1) * 16);
|
||||
uint64x2_t key_vec_1 = xxh3_vld1q_u64(secret + i * 16);
|
||||
uint64x2_t key_vec_2 = xxh3_vld1q_u64(secret + (i + 1) * 16);
|
||||
|
||||
uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
|
||||
uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
|
||||
|
||||
uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
|
||||
uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
|
||||
|
||||
uint32x4x2_t unzipped = vuzpq_u32(
|
||||
vreinterpretq_u32_u64(data_key_1),
|
||||
vreinterpretq_u32_u64(data_key_2)
|
||||
);
|
||||
uint32x4_t data_key_lo = unzipped.val[0];
|
||||
uint32x4_t data_key_hi = unzipped.val[1];
|
||||
|
||||
uint64x2_t sum_1 = xxh3_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
|
||||
uint64x2_t sum_2 = xxh3_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
|
||||
|
||||
/* typed loads and stores preserve scalar lane order on big-endian ARM */
|
||||
vst1q_u64(acc + i * 2, vaddq_u64(vld1q_u64(acc + i * 2), sum_1));
|
||||
vst1q_u64(acc + (i + 1) * 2, vaddq_u64(vld1q_u64(acc + (i + 1) * 2), sum_2));
|
||||
}
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_scramble_neon(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
const uint32x2_t prime_lo = vdup_n_u32(XXH_PRIME32_1);
|
||||
const uint32x4_t prime_hi = vreinterpretq_u32_u64(vdupq_n_u64((uint64_t)XXH_PRIME32_1 << 32));
|
||||
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
uint64x2_t acc_vec = vld1q_u64(acc + i * 2);
|
||||
uint64x2_t shifted = vshrq_n_u64(acc_vec, 47);
|
||||
uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
|
||||
|
||||
uint64x2_t key_vec = xxh3_vld1q_u64(secret + i * 16);
|
||||
uint64x2_t data_key = veorq_u64(data_vec, key_vec);
|
||||
|
||||
uint32x4_t prod_hi = vmulq_u32(vreinterpretq_u32_u64(data_key), prime_hi);
|
||||
uint32x2_t data_key_lo = vmovn_u64(data_key);
|
||||
vst1q_u64(acc + i * 2, vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, prime_lo));
|
||||
}
|
||||
}
|
||||
|
||||
static __always_inline void xxh3_long_core_neon(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = input;
|
||||
__attribute__((aligned(64))) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_neon(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_neon(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_neon(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_neon(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
void xxh3_128_neon(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
__attribute__((aligned(64))) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (tommy_unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_neon(bytes, len, secret, out);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if CONFIG_X86
|
||||
static XXH3_TARGET_SSE2 __always_inline void xxh3_acc512_sse2(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
__m128i* a = (__m128i*)(void*)acc;
|
||||
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
__m128i data = _mm_loadu_si128((const __m128i*)(const void*)(input + 16 * i));
|
||||
__m128i key = _mm_loadu_si128((const __m128i*)(const void*)(secret + 16 * i));
|
||||
__m128i mix = _mm_xor_si128(data, key);
|
||||
__m128i hi32 = _mm_srli_epi64(mix, 32);
|
||||
__m128i prod = _mm_mul_epu32(mix, hi32);
|
||||
__m128i swap = _mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
a[i] = _mm_add_epi64(_mm_add_epi64(a[i], swap), prod);
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_SSE2 __always_inline void xxh3_scramble_sse2(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
__m128i* a = (__m128i*)(void*)acc;
|
||||
const __m128i prime = _mm_set1_epi32((int)XXH_PRIME32_1);
|
||||
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
__m128i x = a[i];
|
||||
__m128i key = _mm_loadu_si128((const __m128i*)(const void*)(secret + 16 * i));
|
||||
x = _mm_xor_si128(x, _mm_srli_epi64(x, 47));
|
||||
x = _mm_xor_si128(x, key);
|
||||
|
||||
/* 64-bit x * 32-bit prime, modulo 2^64, using SSE2 32x32 multiplies. */
|
||||
__m128i lo = _mm_mul_epu32(x, prime);
|
||||
__m128i hi = _mm_mul_epu32(_mm_srli_epi64(x, 32), prime);
|
||||
a[i] = _mm_add_epi64(lo, _mm_slli_epi64(hi, 32));
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 __always_inline void xxh3_acc512_avx2(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
__m256i* a = (__m256i*)(void*)acc;
|
||||
|
||||
for (unsigned i = 0; i < 2; ++i) {
|
||||
__m256i data = _mm256_loadu_si256((const __m256i*)(const void*)(input + 32 * i));
|
||||
__m256i key = _mm256_loadu_si256((const __m256i*)(const void*)(secret + 32 * i));
|
||||
__m256i mix = _mm256_xor_si256(data, key);
|
||||
__m256i hi32 = _mm256_srli_epi64(mix, 32);
|
||||
__m256i prod = _mm256_mul_epu32(mix, hi32);
|
||||
__m256i swap = _mm256_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
a[i] = _mm256_add_epi64(_mm256_add_epi64(a[i], swap), prod);
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 __always_inline void xxh3_scramble_avx2(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
__m256i* a = (__m256i*)(void*)acc;
|
||||
const __m256i prime = _mm256_set1_epi32((int)XXH_PRIME32_1);
|
||||
|
||||
for (unsigned i = 0; i < 2; ++i) {
|
||||
__m256i x = a[i];
|
||||
__m256i key = _mm256_loadu_si256((const __m256i*)(const void*)(secret + 32 * i));
|
||||
x = _mm256_xor_si256(x, _mm256_srli_epi64(x, 47));
|
||||
x = _mm256_xor_si256(x, key);
|
||||
|
||||
__m256i lo = _mm256_mul_epu32(x, prime);
|
||||
__m256i hi = _mm256_mul_epu32(_mm256_srli_epi64(x, 32), prime);
|
||||
a[i] = _mm256_add_epi64(lo, _mm256_slli_epi64(hi, 32));
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 __always_inline void xxh3_acc512_avx512(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
__m512i* a = (__m512i*)(void*)acc;
|
||||
__m512i data = _mm512_loadu_si512((const void*)input);
|
||||
__m512i key = _mm512_loadu_si512((const void*)secret);
|
||||
__m512i mix = _mm512_xor_si512(data, key);
|
||||
__m512i prod = _mm512_mul_epu32(mix, _mm512_srli_epi64(mix, 32));
|
||||
__m512i swap = _mm512_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
*a = _mm512_add_epi64(_mm512_add_epi64(*a, swap), prod);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 __always_inline void xxh3_scramble_avx512(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
__m512i* a = (__m512i*)(void*)acc;
|
||||
const __m512i prime = _mm512_set1_epi32((int)XXH_PRIME32_1);
|
||||
__m512i x = *a;
|
||||
__m512i key = _mm512_loadu_si512((const void*)secret);
|
||||
__m512i shifted = _mm512_srli_epi64(x, 47);
|
||||
|
||||
x = _mm512_ternarylogic_epi32(key, x, shifted, 0x96);
|
||||
|
||||
__m512i lo = _mm512_mul_epu32(x, prime);
|
||||
__m512i hi = _mm512_mul_epu32(_mm512_srli_epi64(x, 32), prime);
|
||||
*a = _mm512_add_epi64(lo, _mm512_slli_epi64(hi, 32));
|
||||
}
|
||||
|
||||
static XXH3_TARGET_SSE2 __always_inline void xxh3_long_core_sse2(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = input;
|
||||
__attribute__((aligned(64))) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_sse2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_sse2(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_sse2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_sse2(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 __always_inline void xxh3_long_core_avx2(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = input;
|
||||
__attribute__((aligned(64))) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_avx2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_avx2(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_avx2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_avx2(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 __always_inline void xxh3_long_core_avx512(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = input;
|
||||
__attribute__((aligned(64))) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_avx512(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_avx512(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_avx512(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_avx512(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
XXH3_TARGET_SSE2 void xxh3_128_sse2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
__attribute__((aligned(64))) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (tommy_unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_sse2(bytes, len, secret, out);
|
||||
}
|
||||
|
||||
XXH3_TARGET_AVX2 void xxh3_128_avx2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
__attribute__((aligned(64))) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (tommy_unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_avx2(bytes, len, secret, out);
|
||||
}
|
||||
|
||||
XXH3_TARGET_AVX512 void xxh3_128_avx512(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
__attribute__((aligned(64))) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (tommy_unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_avx512(bytes, len, secret, out);
|
||||
}
|
||||
#endif
|
||||
|
||||
270
cmdline/xxh3test.c
Normal file
270
cmdline/xxh3test.c
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -84,11 +84,22 @@
|
|||
#define CONFIG_X86_64 1
|
||||
#endif
|
||||
|
||||
#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
|
||||
#if defined(__aarch64__)
|
||||
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
|
||||
#define CONFIG_NEON 1
|
||||
#endif
|
||||
#if defined(__ARM_FEATURE_CRC32)
|
||||
#define CONFIG_ARM_CRC 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__arm__)
|
||||
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
|
||||
#define CONFIG_NEON32 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Includes some platform specific headers.
|
||||
*/
|
||||
|
|
@ -132,7 +143,6 @@
|
|||
#include <linux/fiemap.h>
|
||||
#endif
|
||||
|
||||
|
||||
/**
|
||||
* Includes some standard headers.
|
||||
*/
|
||||
|
|
|
|||
20
raid/cpu.h
20
raid/cpu.h
|
|
@ -260,6 +260,26 @@ static inline int raid_cpu_has_avx2gfni(void)
|
|||
return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, leaf_7_ecx, xcr0);
|
||||
}
|
||||
|
||||
static inline int raid_cpu_has_avx512f(void)
|
||||
{
|
||||
/*
|
||||
* Intel Architecture Instruction Set Extensions Programming Reference
|
||||
* 319433-022 October 2014
|
||||
*
|
||||
* 2.2 Detection of 512-bit Instruction Groups of Intel AVX-512 Family
|
||||
* 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use)
|
||||
* 2) Execute XGETBV and verify that XCR0[7:5] = `111b' (OPMASK state, upper 256-bit of
|
||||
* ZMM0-ZMM15 and ZMM16-ZMM31 state are enabled by OS) and that XCR0[2:1] = `11b'
|
||||
* (XMM state and YMM state are enabled by OS).
|
||||
* 3) Verify CPUID.0x7.0:EBX.AVX512F[bit 16] = 1.
|
||||
*/
|
||||
uint32_t leaf_1_ecx = 1 << 27; /* XSAVE/XGETBV */
|
||||
uint32_t leaf_7_ebx = 1 << 16; /* AVX512F */
|
||||
uint32_t xcr0 = (3 << 1) | (7 << 5); /* OS saves XMM, YMM and ZMM registers */
|
||||
|
||||
return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, 0, xcr0);
|
||||
}
|
||||
|
||||
static inline int raid_cpu_has_avx512bw(void)
|
||||
{
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -1,8 +1,3 @@
|
|||
WARNING: Hash with verification code 4a6673e7 was already registered: MetroHash-128
|
||||
Are you certain MetroHash-128.snapraid is a unique implementation?
|
||||
WARNING: Hash with verification code d5f2cd8c was already registered: MetroHash-128
|
||||
Are you certain MetroHash-128.snapraid is a unique implementation?
|
||||
-------------------------------------------------------------------------------
|
||||
--- Testing MurmurHash3-128.snapraid "MurmurHash3 x86_128 with SnapRAID 128-bit seed interface"
|
||||
|
||||
[[[ Sanity Tests ]]]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
-------------------------------------------------------------------------------
|
||||
--- Testing MuseAir-128.bfast-snapraid "MuseAir v2, BFast-128, SnapRAID long-path short-input extension"
|
||||
|
||||
[[[ Sanity Tests ]]]
|
||||
|
|
|
|||
|
|
@ -21,6 +21,35 @@ Date: Tue Aug 4 11:58:32 2026 -0700
|
|||
Files and results
|
||||
=================
|
||||
|
||||
xxh3_snapraid.cpp
|
||||
XXH3-128 as used by SnapRAID. All inputs use the long accumulator
|
||||
mode. Inputs shorter than 64 bytes are zero-padded to 64 bytes, with
|
||||
the original length retained in finalization.
|
||||
|
||||
XXH3-128-snapraid.txt
|
||||
SMHasher3 output for xxh3_snapraid.cpp, using SSE2.
|
||||
|
||||
Result comparison:
|
||||
The SnapRAID variant passes 250 of 252 tests, compared with 214 of 250
|
||||
in the archived official XXH3-128 result. Always using the Long path
|
||||
removes the observed short-input BIC failures and many seed-test
|
||||
failures. Sanity, Avalanche and BIC all pass.
|
||||
|
||||
Two failures remain. Bitflip finds excess collisions in the high
|
||||
32 bits for 20-byte inputs when flipping input bit 84: 2291 collisions
|
||||
versus 2047.3 expected. This isolated anomaly needs confirmation with
|
||||
different seeds. SeedZeroes shows strong differential collision and
|
||||
distribution biases for all-zero inputs with lengths up to 8448 bytes
|
||||
and sparse or complemented sparse seeds. The direct hash distribution
|
||||
passes; the failure concerns XOR differences between hashes.
|
||||
|
||||
Official XXH3 also fails SeedZeroes. The remaining failure is consistent
|
||||
with a limitation of the shared Long path, although these results alone
|
||||
do not establish its cause. No full 128-bit collisions were observed.
|
||||
MuseAir and SpookyHash V2 have cleaner results, passing all 252 tests.
|
||||
The seed tests cover duplicated 64-bit seeds, not independent 128-bit
|
||||
seed halves.
|
||||
|
||||
museair_snapraid.cpp
|
||||
MuseAir BFast-128 as used by SnapRAID. Inputs shorter than 32 bytes
|
||||
are zero-padded to 32 bytes and processed with the Long path while
|
||||
|
|
@ -71,7 +100,7 @@ Seed mapping
|
|||
|
||||
SMHasher3 exposes one 64-bit seed_t, while SnapRAID uses 128-bit seeds.
|
||||
|
||||
For MuseAir and SpookyHash V2:
|
||||
For XXH3, MuseAir and SpookyHash V2:
|
||||
|
||||
seed[0..7] = seed_t
|
||||
seed[8..15] = seed_t
|
||||
|
|
@ -95,6 +124,7 @@ Copy the adapters into the SMHasher3 hashes directory:
|
|||
hashes/museair_snapraid.cpp
|
||||
hashes/spooky2_snapraid.cpp
|
||||
hashes/murmur3_snapraid.cpp
|
||||
hashes/xxh3_snapraid.cpp
|
||||
|
||||
Add them to hashes/Hashsrc.cmake and rebuild SMHasher3.
|
||||
|
||||
|
|
@ -106,6 +136,10 @@ Check the registrations with:
|
|||
Generating the result files
|
||||
===========================
|
||||
|
||||
./SMHasher3 XXH3-128-snapraid \
|
||||
--extra --notest=Speed,Hashmap --ncpu=16 \
|
||||
> ../../../../smhasher3/XXH3-128-snapraid.txt 2>&1
|
||||
|
||||
./SMHasher3 MuseAir-128.bfast-snapraid \
|
||||
--extra --notest=Speed,Hashmap --ncpu=16 \
|
||||
> MuseAir-128.bfast-snapraid.txt 2>&1
|
||||
|
|
@ -127,6 +161,7 @@ The authoritative implementations remain the SnapRAID source files:
|
|||
cmdline/museair.c
|
||||
cmdline/spooky2.c
|
||||
cmdline/murmur3.c
|
||||
cmdline/xxh3.c
|
||||
|
||||
SnapRAID repository:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
-------------------------------------------------------------------------------
|
||||
--- Testing SpookyHash2-128-snapraid "SpookyHash v2 128-bit, SnapRAID long-path-only variant"
|
||||
|
||||
[[[ Sanity Tests ]]]
|
||||
|
|
|
|||
3748
smhasher3/XXH3-128-snapraid.txt
Normal file
3748
smhasher3/XXH3-128-snapraid.txt
Normal file
File diff suppressed because it is too large
Load diff
651
smhasher3/xxh3_snapraid.cpp
Normal file
651
smhasher3/xxh3_snapraid.cpp
Normal file
|
|
@ -0,0 +1,651 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Copyright (C) 2026 Andrea Mazzoleni
|
||||
|
||||
/*
|
||||
* Derivative work of the XXH3-128 v8.4 implementation.
|
||||
*
|
||||
* This file implements the XXH3 long accumulator mode with support for
|
||||
* arbitrary input lengths. Unlike the original XXH3, which switches to separate
|
||||
* small/midsize routines for len <= 240 bytes, this implementation extends the
|
||||
* long mode:
|
||||
* - For len >= 64 bytes, it uses the striped accumulator loop directly.
|
||||
* - For len < 64 bytes, the input is zero-padded to 64 bytes, and the original
|
||||
* length is folded into the finalizer (preventing padding collisions).
|
||||
*
|
||||
* The motivation for extending the long mode to all lengths rather than using
|
||||
* the upstream short/midsize routines is that SnapRAID operates primarily on
|
||||
* large data blocks (typically 256 KB) where SIMD-accelerated long mode delivers
|
||||
* maximum throughput. Short inputs occur only as rare edge cases (such as
|
||||
* empty or small files), and maintaining a single code path reduces algorithmic
|
||||
* complexity and defect potential.
|
||||
*
|
||||
* seed points to 16 bytes, interpreted as two little-endian uint64_t words:
|
||||
* seed_lo then seed_hi.
|
||||
* - An all-zero seed uses the unmodified standard XXH3 secret.
|
||||
* - For len > 240, setting seed_lo == seed_hi matches standard
|
||||
* XXH3_128bits_withSeed() with that 64-bit seed value.
|
||||
* - When seed_lo != seed_hi, the 128-bit seed extends the secret by adding
|
||||
* seed_lo to even 64-bit words and subtracting seed_hi from odd words.
|
||||
*
|
||||
* For len <= 240, the output is not bit-compatible with original xxHash due to
|
||||
* using the padded long mode rather than the upstream short/midsize routines.
|
||||
*/
|
||||
|
||||
/*
|
||||
* xxHash - Extremely Fast Hash algorithm
|
||||
* Development source file for `xxh3`
|
||||
* Copyright (C) 2019-2021 Yann Collet
|
||||
*
|
||||
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following disclaimer
|
||||
* in the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
/**
|
||||
* Standalone SMHasher3 adapter for cmdline/xxh3.c.
|
||||
* Both entry points retain SnapRAID's canonical little-endian reads and output.
|
||||
* SMHasher3's 64-bit seed is duplicated into the two 64-bit seed halves.
|
||||
*/
|
||||
#include "Platform.h"
|
||||
#include "Hashlib.h"
|
||||
#include "Mathmult.h"
|
||||
|
||||
/**
|
||||
* Change the #if values here to select an implementation manually.
|
||||
* SIMD entry points require a compatible CPU; there is no runtime dispatch.
|
||||
* The target attributes below preserve the existing SMHasher3 compiler flags.
|
||||
*/
|
||||
#if 0
|
||||
#define XXH3_SNAPRAID_HASH_FN xxh3_128
|
||||
#define XXH3_SNAPRAID_IMPL "scalar"
|
||||
#elif 0
|
||||
#define XXH3_SNAPRAID_HASH_FN xxh3_128_avx512
|
||||
#define XXH3_SNAPRAID_IMPL "avx512"
|
||||
#elif 0
|
||||
#define XXH3_SNAPRAID_HASH_FN xxh3_128_avx2
|
||||
#define XXH3_SNAPRAID_IMPL "avx2"
|
||||
#elif defined(__i386__) || defined(__x86_64__)
|
||||
#define XXH3_SNAPRAID_HASH_FN xxh3_128_sse2
|
||||
#define XXH3_SNAPRAID_IMPL "sse2"
|
||||
#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
|
||||
#define XXH3_SNAPRAID_HASH_FN xxh3_128_neon
|
||||
#define XXH3_SNAPRAID_IMPL "neon"
|
||||
#else
|
||||
#define XXH3_SNAPRAID_HASH_FN xxh3_128
|
||||
#define XXH3_SNAPRAID_IMPL "scalar"
|
||||
#endif
|
||||
|
||||
static FORCE_INLINE uint64_t xxh3_read64(const uint8_t* ptr)
|
||||
{
|
||||
return COND_BSWAP(GET_U64<false>(ptr, 0), isBE());
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_write64(uint8_t* ptr, uint64_t value)
|
||||
{
|
||||
PUT_U64<false>(COND_BSWAP(value, isBE()), ptr, 0);
|
||||
}
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
#include <immintrin.h>
|
||||
|
||||
#define XXH3_TARGET_SSE2 __attribute__((target("sse2")))
|
||||
#define XXH3_TARGET_AVX2 __attribute__((target("avx2")))
|
||||
#define XXH3_TARGET_AVX512 __attribute__((target("avx512f")))
|
||||
#endif
|
||||
|
||||
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
#define XXH3_STRIPE_LEN 64
|
||||
#define XXH3_SECRET_SIZE 192
|
||||
#define XXH3_SECRET_CONSUME_RATE 8
|
||||
#define XXH3_ACC_NB 8
|
||||
#define XXH3_STRIPES_PER_BLOCK 16
|
||||
#define XXH3_BLOCK_LEN 1024
|
||||
#define XXH3_LAST_SECRET_OFFSET 121
|
||||
#define XXH3_MERGE_OFFSET 11
|
||||
|
||||
#define XXH_PRIME32_1 0x9E3779B1U
|
||||
#define XXH_PRIME32_2 0x85EBCA77U
|
||||
#define XXH_PRIME32_3 0xC2B2AE3DU
|
||||
|
||||
#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL
|
||||
#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL
|
||||
#define XXH_PRIME64_3 0x165667B19E3779F9ULL
|
||||
#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL
|
||||
#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL
|
||||
|
||||
#define XXH_PRIME_MX1 0x165667919E3779F9ULL
|
||||
|
||||
#define XXH3_INIT_ACC_VALUES \
|
||||
{ XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
|
||||
XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
|
||||
|
||||
alignas(64) static const uint8_t xxh3_secret[XXH3_SECRET_SIZE] = {
|
||||
0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
|
||||
0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
|
||||
0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
|
||||
0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
|
||||
0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
|
||||
0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
|
||||
0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
|
||||
0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
|
||||
0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
|
||||
0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
|
||||
0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
|
||||
0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e
|
||||
};
|
||||
|
||||
static FORCE_INLINE uint64_t xxh3_xorshift64(uint64_t x, unsigned shift)
|
||||
{
|
||||
return x ^ (x >> shift);
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint64_t xxh3_avalanche(uint64_t x)
|
||||
{
|
||||
x = xxh3_xorshift64(x, 37);
|
||||
x *= XXH_PRIME_MX1;
|
||||
return xxh3_xorshift64(x, 32);
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint64_t xxh3_mul128_fold64(uint64_t a, uint64_t b)
|
||||
{
|
||||
MathMult::mult64_128(a, b, a, b);
|
||||
return a ^ b;
|
||||
}
|
||||
|
||||
/*
|
||||
* 128-bit seed extension.
|
||||
*
|
||||
* Standard XXH3 with a 64-bit seed transforms every 16-byte secret pair as:
|
||||
*
|
||||
* secret[even] += seed;
|
||||
* secret[odd] -= seed;
|
||||
*
|
||||
* Here the 16-byte seed is split into two little-endian 64-bit words and the
|
||||
* same construction is extended naturally to:
|
||||
*
|
||||
* secret[even] += seed_lo;
|
||||
* secret[odd] -= seed_hi;
|
||||
*
|
||||
* Therefore both halves of the seed affect every 16-byte secret pair.
|
||||
*/
|
||||
static void xxh3_make_seeded_secret(uint8_t* out, uint64_t seed_lo, uint64_t seed_hi)
|
||||
{
|
||||
for (size_t i = 0; i < XXH3_SECRET_SIZE; i += 16) {
|
||||
xxh3_write64(out + i, xxh3_read64(xxh3_secret + i) + seed_lo);
|
||||
xxh3_write64(out + i + 8, xxh3_read64(xxh3_secret + i + 8) - seed_hi);
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE const uint8_t* xxh3_select_secret(const uint8_t* seed, uint8_t* custom)
|
||||
{
|
||||
uint64_t seed_lo = xxh3_read64(seed);
|
||||
uint64_t seed_hi = xxh3_read64(seed + 8);
|
||||
|
||||
if ((seed_lo | seed_hi) == 0)
|
||||
return xxh3_secret;
|
||||
|
||||
xxh3_make_seeded_secret(custom, seed_lo, seed_hi);
|
||||
return custom;
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint64_t xxh3_mix2accs(const uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
return xxh3_mul128_fold64(acc[0] ^ xxh3_read64(secret), acc[1] ^ xxh3_read64(secret + 8));
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint64_t xxh3_merge_accs(const uint64_t* acc, const uint8_t* secret, uint64_t start)
|
||||
{
|
||||
uint64_t h = start;
|
||||
for (unsigned i = 0; i < 4; ++i)
|
||||
h += xxh3_mix2accs(acc + 2 * i, secret + 16 * i);
|
||||
return xxh3_avalanche(h);
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_finalize128(const uint64_t* acc, const uint8_t* secret, size_t len, uint8_t* out)
|
||||
{
|
||||
uint64_t low64 = xxh3_merge_accs(acc, secret + XXH3_MERGE_OFFSET, (uint64_t)len * XXH_PRIME64_1);
|
||||
uint64_t high64 = xxh3_merge_accs(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN - XXH3_MERGE_OFFSET, ~((uint64_t)len * XXH_PRIME64_2));
|
||||
xxh3_write64(out, low64);
|
||||
xxh3_write64(out + 8, high64);
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_acc512_c(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
for (unsigned lane = 0; lane < XXH3_ACC_NB; ++lane) {
|
||||
uint64_t data = xxh3_read64(input + 8 * lane);
|
||||
uint64_t keyed = data ^ xxh3_read64(secret + 8 * lane);
|
||||
uint64_t product = (uint64_t)(uint32_t)keyed * (uint32_t)(keyed >> 32);
|
||||
|
||||
acc[lane ^ 1] += data; /* exchange adjacent 64-bit lanes */
|
||||
acc[lane] += product;
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_scramble_c(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
for (unsigned lane = 0; lane < XXH3_ACC_NB; ++lane) {
|
||||
uint64_t x = xxh3_xorshift64(acc[lane], 47);
|
||||
x ^= xxh3_read64(secret + 8 * lane);
|
||||
acc[lane] = x * XXH_PRIME32_1;
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_short_core(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
alignas(64) uint8_t buf[XXH3_STRIPE_LEN];
|
||||
alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
|
||||
memset(buf, 0, XXH3_STRIPE_LEN);
|
||||
if (len > 0)
|
||||
memcpy(buf, input, len);
|
||||
|
||||
xxh3_acc512_c(acc, buf, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
/*
|
||||
* With the 192-byte default secret:
|
||||
* stripes per block = (192 - 64) / 8 = 16
|
||||
* block size = 16 * 64 = 1024 bytes
|
||||
*/
|
||||
static FORCE_INLINE void xxh3_long_core_c(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = static_cast<const uint8_t*>(input);
|
||||
alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_c(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_c(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_c(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_c(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
/*
|
||||
* seed points to 16 bytes: seed_lo then seed_hi, both little-endian.
|
||||
* out receives low64 then high64, both little-endian.
|
||||
*
|
||||
* No CPU detection is performed. The caller must invoke only a SIMD entry
|
||||
* point supported by the current CPU.
|
||||
*/
|
||||
static void xxh3_128(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
alignas(64) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_c(bytes, len, secret, out);
|
||||
}
|
||||
|
||||
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
|
||||
static FORCE_INLINE uint64x2_t xxh3_vld1q_u64(const void* ptr)
|
||||
{
|
||||
return vreinterpretq_u64_u8(vld1q_u8((const uint8_t*)ptr));
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint64x2_t xxh3_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
|
||||
{
|
||||
return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint64x2_t xxh3_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
|
||||
{
|
||||
#if defined(__aarch64__)
|
||||
return vmlal_high_u32(acc, lhs, rhs);
|
||||
#else
|
||||
return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
|
||||
#endif
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_acc512_neon(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
for (unsigned i = 0; i < 4; i += 2) {
|
||||
uint64x2_t data_vec_1 = xxh3_vld1q_u64(input + i * 16);
|
||||
uint64x2_t data_vec_2 = xxh3_vld1q_u64(input + (i + 1) * 16);
|
||||
uint64x2_t key_vec_1 = xxh3_vld1q_u64(secret + i * 16);
|
||||
uint64x2_t key_vec_2 = xxh3_vld1q_u64(secret + (i + 1) * 16);
|
||||
|
||||
uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
|
||||
uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
|
||||
|
||||
uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
|
||||
uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
|
||||
|
||||
uint32x4x2_t unzipped = vuzpq_u32(
|
||||
vreinterpretq_u32_u64(data_key_1),
|
||||
vreinterpretq_u32_u64(data_key_2)
|
||||
);
|
||||
uint32x4_t data_key_lo = unzipped.val[0];
|
||||
uint32x4_t data_key_hi = unzipped.val[1];
|
||||
|
||||
uint64x2_t sum_1 = xxh3_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
|
||||
uint64x2_t sum_2 = xxh3_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
|
||||
|
||||
/* typed loads and stores preserve scalar lane order on big-endian ARM */
|
||||
vst1q_u64(acc + i * 2, vaddq_u64(vld1q_u64(acc + i * 2), sum_1));
|
||||
vst1q_u64(acc + (i + 1) * 2, vaddq_u64(vld1q_u64(acc + (i + 1) * 2), sum_2));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_scramble_neon(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
const uint32x2_t prime_lo = vdup_n_u32(XXH_PRIME32_1);
|
||||
const uint32x4_t prime_hi = vreinterpretq_u32_u64(vdupq_n_u64((uint64_t)XXH_PRIME32_1 << 32));
|
||||
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
uint64x2_t acc_vec = vld1q_u64(acc + i * 2);
|
||||
uint64x2_t shifted = vshrq_n_u64(acc_vec, 47);
|
||||
uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
|
||||
|
||||
uint64x2_t key_vec = xxh3_vld1q_u64(secret + i * 16);
|
||||
uint64x2_t data_key = veorq_u64(data_vec, key_vec);
|
||||
|
||||
uint32x4_t prod_hi = vmulq_u32(vreinterpretq_u32_u64(data_key), prime_hi);
|
||||
uint32x2_t data_key_lo = vmovn_u64(data_key);
|
||||
vst1q_u64(acc + i * 2, vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, prime_lo));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE void xxh3_long_core_neon(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = static_cast<const uint8_t*>(input);
|
||||
alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_neon(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_neon(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_neon(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_neon(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
static void xxh3_128_neon(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
alignas(64) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_neon(bytes, len, secret, out);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
static XXH3_TARGET_SSE2 FORCE_INLINE void xxh3_acc512_sse2(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
__m128i* a = (__m128i*)(void*)acc;
|
||||
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
__m128i data = _mm_loadu_si128((const __m128i*)(const void*)(input + 16 * i));
|
||||
__m128i key = _mm_loadu_si128((const __m128i*)(const void*)(secret + 16 * i));
|
||||
__m128i mix = _mm_xor_si128(data, key);
|
||||
__m128i hi32 = _mm_srli_epi64(mix, 32);
|
||||
__m128i prod = _mm_mul_epu32(mix, hi32);
|
||||
__m128i swap = _mm_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
a[i] = _mm_add_epi64(_mm_add_epi64(a[i], swap), prod);
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_SSE2 FORCE_INLINE void xxh3_scramble_sse2(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
__m128i* a = (__m128i*)(void*)acc;
|
||||
const __m128i prime = _mm_set1_epi32((int)XXH_PRIME32_1);
|
||||
|
||||
for (unsigned i = 0; i < 4; ++i) {
|
||||
__m128i x = a[i];
|
||||
__m128i key = _mm_loadu_si128((const __m128i*)(const void*)(secret + 16 * i));
|
||||
x = _mm_xor_si128(x, _mm_srli_epi64(x, 47));
|
||||
x = _mm_xor_si128(x, key);
|
||||
|
||||
/* 64-bit x * 32-bit prime, modulo 2^64, using SSE2 32x32 multiplies. */
|
||||
__m128i lo = _mm_mul_epu32(x, prime);
|
||||
__m128i hi = _mm_mul_epu32(_mm_srli_epi64(x, 32), prime);
|
||||
a[i] = _mm_add_epi64(lo, _mm_slli_epi64(hi, 32));
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 FORCE_INLINE void xxh3_acc512_avx2(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
__m256i* a = (__m256i*)(void*)acc;
|
||||
|
||||
for (unsigned i = 0; i < 2; ++i) {
|
||||
__m256i data = _mm256_loadu_si256((const __m256i*)(const void*)(input + 32 * i));
|
||||
__m256i key = _mm256_loadu_si256((const __m256i*)(const void*)(secret + 32 * i));
|
||||
__m256i mix = _mm256_xor_si256(data, key);
|
||||
__m256i hi32 = _mm256_srli_epi64(mix, 32);
|
||||
__m256i prod = _mm256_mul_epu32(mix, hi32);
|
||||
__m256i swap = _mm256_shuffle_epi32(data, _MM_SHUFFLE(1, 0, 3, 2));
|
||||
a[i] = _mm256_add_epi64(_mm256_add_epi64(a[i], swap), prod);
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 FORCE_INLINE void xxh3_scramble_avx2(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
__m256i* a = (__m256i*)(void*)acc;
|
||||
const __m256i prime = _mm256_set1_epi32((int)XXH_PRIME32_1);
|
||||
|
||||
for (unsigned i = 0; i < 2; ++i) {
|
||||
__m256i x = a[i];
|
||||
__m256i key = _mm256_loadu_si256((const __m256i*)(const void*)(secret + 32 * i));
|
||||
x = _mm256_xor_si256(x, _mm256_srli_epi64(x, 47));
|
||||
x = _mm256_xor_si256(x, key);
|
||||
|
||||
__m256i lo = _mm256_mul_epu32(x, prime);
|
||||
__m256i hi = _mm256_mul_epu32(_mm256_srli_epi64(x, 32), prime);
|
||||
a[i] = _mm256_add_epi64(lo, _mm256_slli_epi64(hi, 32));
|
||||
}
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 FORCE_INLINE void xxh3_acc512_avx512(uint64_t* acc, const uint8_t* input, const uint8_t* secret)
|
||||
{
|
||||
__m512i* a = (__m512i*)(void*)acc;
|
||||
__m512i data = _mm512_loadu_si512((const void*)input);
|
||||
__m512i key = _mm512_loadu_si512((const void*)secret);
|
||||
__m512i mix = _mm512_xor_si512(data, key);
|
||||
__m512i prod = _mm512_mul_epu32(mix, _mm512_srli_epi64(mix, 32));
|
||||
__m512i swap = _mm512_shuffle_epi32(data, _MM_PERM_BADC);
|
||||
*a = _mm512_add_epi64(_mm512_add_epi64(*a, swap), prod);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 FORCE_INLINE void xxh3_scramble_avx512(uint64_t* acc, const uint8_t* secret)
|
||||
{
|
||||
__m512i* a = (__m512i*)(void*)acc;
|
||||
const __m512i prime = _mm512_set1_epi32((int)XXH_PRIME32_1);
|
||||
__m512i x = *a;
|
||||
__m512i key = _mm512_loadu_si512((const void*)secret);
|
||||
__m512i shifted = _mm512_srli_epi64(x, 47);
|
||||
|
||||
x = _mm512_ternarylogic_epi32(key, x, shifted, 0x96);
|
||||
|
||||
__m512i lo = _mm512_mul_epu32(x, prime);
|
||||
__m512i hi = _mm512_mul_epu32(_mm512_srli_epi64(x, 32), prime);
|
||||
*a = _mm512_add_epi64(lo, _mm512_slli_epi64(hi, 32));
|
||||
}
|
||||
|
||||
static XXH3_TARGET_SSE2 FORCE_INLINE void xxh3_long_core_sse2(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = static_cast<const uint8_t*>(input);
|
||||
alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_sse2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_sse2(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_sse2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_sse2(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 FORCE_INLINE void xxh3_long_core_avx2(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = static_cast<const uint8_t*>(input);
|
||||
alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_avx2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_avx2(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_avx2(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_avx2(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 FORCE_INLINE void xxh3_long_core_avx512(const void* input, size_t len, const uint8_t* secret, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = static_cast<const uint8_t*>(input);
|
||||
alignas(64) uint64_t acc[XXH3_ACC_NB] = XXH3_INIT_ACC_VALUES;
|
||||
size_t blocks = (len - 1) / XXH3_BLOCK_LEN;
|
||||
|
||||
for (size_t b = 0; b < blocks; ++b) {
|
||||
const uint8_t* block = p + b * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < XXH3_STRIPES_PER_BLOCK; ++s)
|
||||
xxh3_acc512_avx512(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
xxh3_scramble_avx512(acc, secret + XXH3_SECRET_SIZE - XXH3_STRIPE_LEN);
|
||||
}
|
||||
|
||||
size_t stripes = ((len - 1) - blocks * XXH3_BLOCK_LEN) / XXH3_STRIPE_LEN;
|
||||
const uint8_t* block = p + blocks * XXH3_BLOCK_LEN;
|
||||
for (size_t s = 0; s < stripes; ++s)
|
||||
xxh3_acc512_avx512(acc, block + s * XXH3_STRIPE_LEN, secret + s * XXH3_SECRET_CONSUME_RATE);
|
||||
|
||||
xxh3_acc512_avx512(acc, p + len - XXH3_STRIPE_LEN, secret + XXH3_LAST_SECRET_OFFSET);
|
||||
xxh3_finalize128(acc, secret, len, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_SSE2 void xxh3_128_sse2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
alignas(64) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_sse2(bytes, len, secret, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX2 void xxh3_128_avx2(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
alignas(64) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_avx2(bytes, len, secret, out);
|
||||
}
|
||||
|
||||
static XXH3_TARGET_AVX512 void xxh3_128_avx512(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
alignas(64) uint8_t custom[XXH3_SECRET_SIZE];
|
||||
const uint8_t* secret;
|
||||
|
||||
secret = xxh3_select_secret(seed, custom);
|
||||
|
||||
if (unlikely(len < XXH3_STRIPE_LEN))
|
||||
xxh3_short_core(bytes, len, secret, out);
|
||||
else
|
||||
xxh3_long_core_avx512(bytes, len, secret, out);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void xxh3_snapraid(const void* in, size_t len, seed_t seed, void* out)
|
||||
{
|
||||
uint8_t seed_bytes[16];
|
||||
|
||||
/* duplicate the seed, matching the other SnapRAID adapters */
|
||||
xxh3_write64(seed_bytes, seed);
|
||||
xxh3_write64(seed_bytes + 8, seed);
|
||||
XXH3_SNAPRAID_HASH_FN(in, len, seed_bytes, static_cast<uint8_t*>(out));
|
||||
}
|
||||
|
||||
REGISTER_FAMILY(xxh3_snapraid,
|
||||
$.src_url = "https://github.com/amadvance/snapraid/",
|
||||
$.src_status = HashFamilyInfo::SRC_ACTIVE
|
||||
);
|
||||
|
||||
REGISTER_HASH(XXH3_128_snapraid,
|
||||
$.desc = "XXH3-128, SnapRAID long mode for all input lengths",
|
||||
$.impl = XXH3_SNAPRAID_IMPL,
|
||||
$.hash_flags = FLAG_HASH_ENDIAN_INDEPENDENT,
|
||||
$.impl_flags = FLAG_IMPL_MULTIPLY_64_128 |
|
||||
FLAG_IMPL_CANONICAL_BOTH |
|
||||
FLAG_IMPL_LICENSE_GPL3,
|
||||
$.bits = 128,
|
||||
$.verification_LE = 0xCFAD116F,
|
||||
$.verification_BE = 0xCFAD116F,
|
||||
$.hashfn_native = xxh3_snapraid,
|
||||
$.hashfn_bswap = xxh3_snapraid
|
||||
);
|
||||
Loading…
Add table
Add a link
Reference in a new issue