mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Parallel hashes
This commit is contained in:
Andrea Mazzoleni
parent
13844afe4c
commit
e63e79d51a
11 changed files with 1953 additions and 1 deletions
|
|
@ -109,10 +109,14 @@ noinst_HEADERS = \
|
|||
cmdline/murmur3test.c \
|
||||
cmdline/spooky2.c \
|
||||
cmdline/spooky2test.c \
|
||||
cmdline/spooky2sse.c \
|
||||
cmdline/spooky2avx.c \
|
||||
cmdline/metro.c \
|
||||
cmdline/metrotest.c \
|
||||
cmdline/museair.c \
|
||||
cmdline/museairtest.c \
|
||||
cmdline/museairsse.c \
|
||||
cmdline/museairavx.c \
|
||||
cmdline/import.h \
|
||||
cmdline/search.h \
|
||||
cmdline/thermal.h \
|
||||
|
|
|
|||
|
|
@ -5,3 +5,4 @@
|
|||
#define HAVE_CHECKER 1
|
||||
|
||||
#include "snapraid.c"
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ static const uint64_t MASK_I = 01555555555555555555555ULL;
|
|||
static const uint64_t MASK_J = 01333333333333333333333ULL;
|
||||
static const uint64_t MASK_K = 00666666666666666666666ULL;
|
||||
|
||||
static void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out)
|
||||
{
|
||||
const uint8_t* p = bytes;
|
||||
size_t q = len;
|
||||
|
|
|
|||
476
cmdline/museairavx.c
Normal file
476
cmdline/museairavx.c
Normal file
|
|
@ -0,0 +1,476 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Copyright (C) 2026 Andrea Mazzoleni
|
||||
|
||||
#if defined(__x86_64__)
|
||||
|
||||
#define AVX_LOAD(offset, dest) \
|
||||
"vmovq " #offset "*8(%[blocks0]), %%xmm14\n\t" \
|
||||
"vmovhps " #offset "*8(%[blocks1]), %%xmm14, %%xmm14\n\t" \
|
||||
"vmovq " #offset "*8(%[blocks2]), %%xmm15\n\t" \
|
||||
"vmovhps " #offset "*8(%[blocks3]), %%xmm15, %%xmm15\n\t" \
|
||||
"vinserti128 $1, %%xmm15, %%ymm14, " dest "\n\t"
|
||||
|
||||
#define AVX_LOAD_NEG(offset, dest) \
|
||||
"vmovq -" #offset "(%%r8), %%xmm14\n\t" \
|
||||
"vmovhps -" #offset "(%%r9), %%xmm14, %%xmm14\n\t" \
|
||||
"vmovq -" #offset "(%%r10), %%xmm15\n\t" \
|
||||
"vmovhps -" #offset "(%%r11), %%xmm15, %%xmm15\n\t" \
|
||||
"vinserti128 $1, %%xmm15, %%ymm14, " dest "\n\t"
|
||||
|
||||
#define AVX_MULT(ymm_a, ymm_b, ymm_lo, ymm_hi) \
|
||||
/* 1. ymm_ll = ymm_a * ymm_b (low parts) */ \
|
||||
"vpmuludq " ymm_b ", " ymm_a ", %%ymm8\n\t" /* ymm8 = P_ll */ \
|
||||
/* 2. ymm_lh = ymm_a * ymm_b_high */ \
|
||||
"vpsrlq $32, " ymm_b ", %%ymm9\n\t" /* ymm9 = B_h */ \
|
||||
"vpmuludq %%ymm9, " ymm_a ", %%ymm10\n\t" /* ymm10 = P_lh */ \
|
||||
/* 3. ymm_hl = ymm_a_high * ymm_b */ \
|
||||
"vpsrlq $32, " ymm_a ", %%ymm14\n\t" /* ymm14 = A_h */ \
|
||||
"vpmuludq " ymm_b ", %%ymm14, %%ymm15\n\t" /* ymm15 = P_hl */ \
|
||||
/* 4. ymm_hh = ymm_a_high * ymm_b_high */ \
|
||||
"vpmuludq %%ymm9, %%ymm14, %%ymm9\n\t" /* ymm9 = P_hh */ \
|
||||
/* 5. M = P_lh + P_hl */ \
|
||||
"vpaddq %%ymm15, %%ymm10, %%ymm15\n\t" /* ymm15 = M */ \
|
||||
/* Generate mask 0xffffffff in ymm14 */ \
|
||||
"vpcmpeqd %%ymm14, %%ymm14, %%ymm14\n\t" \
|
||||
"vpsrlq $32, %%ymm14, %%ymm14\n\t" \
|
||||
/* 6. Split M into ymm10 = M_low, ymm15 = M_high */ \
|
||||
"vpand %%ymm14, %%ymm15, %%ymm10\n\t" /* ymm10 = M_low */ \
|
||||
"vpsrlq $32, %%ymm15, %%ymm15\n\t" /* ymm15 = M_high */ \
|
||||
/* 7. Split P_ll (in ymm8) into ymm12 = P_lll, ymm13 = P_llh */ \
|
||||
"vpsrlq $32, %%ymm8, %%ymm13\n\t" /* ymm13 = P_llh */ \
|
||||
"vpand %%ymm14, %%ymm8, %%ymm12\n\t" /* ymm12 = P_lll */ \
|
||||
/* 8. S1 = P_llh + M_low */ \
|
||||
"vpaddq %%ymm10, %%ymm13, %%ymm10\n\t" /* ymm10 = S1 */ \
|
||||
/* 9. Split S1 into ymm13 = C1, ymm10 = S1_l */ \
|
||||
"vpsrlq $32, %%ymm10, %%ymm13\n\t" /* ymm13 = C1 */ \
|
||||
"vpand %%ymm14, %%ymm10, %%ymm10\n\t" /* ymm10 = S1_l */ \
|
||||
/* 10. Assemble final low = P_lll + S1_l * 2^32 */ \
|
||||
"vpsllq $32, %%ymm10, %%ymm10\n\t" \
|
||||
"vpaddq %%ymm12, %%ymm10, " ymm_lo "\n\t" /* ymm_lo = final low */ \
|
||||
/* 11. Split P_hh (in ymm9) into ymm12 = P_hhl, ymm10 = P_hhh */ \
|
||||
"vpsrlq $32, %%ymm9, %%ymm10\n\t" /* ymm10 = P_hhh */ \
|
||||
"vpand %%ymm14, %%ymm9, %%ymm12\n\t" /* ymm12 = P_hhl */ \
|
||||
/* 12. Assemble final high = M_high + P_hhl + C1 + P_hhh * 2^32 */ \
|
||||
"vpaddq %%ymm12, %%ymm15, %%ymm15\n\t" /* ymm15 = M_high + P_hhl */ \
|
||||
"vpaddq %%ymm13, %%ymm15, %%ymm15\n\t" /* ymm15 = M_high + P_hhl + C1 */ \
|
||||
"vpsllq $32, %%ymm10, %%ymm10\n\t" \
|
||||
"vpaddq %%ymm10, %%ymm15, " ymm_hi "\n\t" /* ymm_hi = final high */
|
||||
|
||||
void MuseAirLoongAVX(
|
||||
const void* data0, const void* data1, const void* data2, const void* data3,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3,
|
||||
uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3
|
||||
)
|
||||
{
|
||||
uint64_t buf0[12];
|
||||
uint64_t buf1[12];
|
||||
uint64_t buf2[12];
|
||||
uint64_t buf3[12];
|
||||
const uint8_t* blocks0 = data0;
|
||||
const uint8_t* blocks1 = data1;
|
||||
const uint8_t* blocks2 = data2;
|
||||
const uint8_t* blocks3 = data3;
|
||||
size_t q = size;
|
||||
|
||||
if (q < 32) {
|
||||
memset(buf0, 0, 32);
|
||||
memset(buf1, 0, 32);
|
||||
memset(buf2, 0, 32);
|
||||
memset(buf3, 0, 32);
|
||||
if (q > 0) {
|
||||
memcpy(buf0, blocks0, q);
|
||||
memcpy(buf1, blocks1, q);
|
||||
memcpy(buf2, blocks2, q);
|
||||
memcpy(buf3, blocks3, q);
|
||||
}
|
||||
blocks0 = (const uint8_t*)buf0;
|
||||
blocks1 = (const uint8_t*)buf1;
|
||||
blocks2 = (const uint8_t*)buf2;
|
||||
blocks3 = (const uint8_t*)buf3;
|
||||
q = 32;
|
||||
}
|
||||
|
||||
uint64_t rot = size & 63;
|
||||
uint64_t rot_diff = 64 - rot;
|
||||
static const uint64_t const_mask_i = 01555555555555555555555ULL;
|
||||
static const uint64_t const_mask_j = 01333333333333333333333ULL;
|
||||
static const uint64_t const_mask_k = 00666666666666666666666ULL;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
/* Load seeds */
|
||||
"movq %[seed0], %%rax\n\t"
|
||||
"vmovq (%%rax), %%xmm9\n\t"
|
||||
"movq %[seed1], %%rax\n\t"
|
||||
"vmovhps (%%rax), %%xmm9, %%xmm9\n\t"
|
||||
"movq %[seed2], %%rax\n\t"
|
||||
"vmovq (%%rax), %%xmm13\n\t"
|
||||
"movq %[seed3], %%rax\n\t"
|
||||
"vmovhps (%%rax), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm9, %%ymm9\n\t" /* ymm9 = seed_a */
|
||||
|
||||
"movq %[seed0], %%rax\n\t"
|
||||
"vmovq 8(%%rax), %%xmm10\n\t"
|
||||
"movq %[seed1], %%rax\n\t"
|
||||
"vmovhps 8(%%rax), %%xmm10, %%xmm10\n\t"
|
||||
"movq %[seed2], %%rax\n\t"
|
||||
"vmovq 8(%%rax), %%xmm13\n\t"
|
||||
"movq %[seed3], %%rax\n\t"
|
||||
"vmovhps 8(%%rax), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm10, %%ymm10\n\t" /* ymm10 = seed_b */
|
||||
|
||||
/* Load masks */
|
||||
"vmovq %[mask_i], %%xmm12\n\t"
|
||||
"vpunpcklqdq %%xmm12, %%xmm12, %%xmm12\n\t"
|
||||
"vinserti128 $1, %%xmm12, %%ymm12, %%ymm12\n\t"
|
||||
|
||||
"vmovq %[mask_j], %%xmm13\n\t"
|
||||
"vpunpcklqdq %%xmm13, %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm13, %%ymm13\n\t"
|
||||
|
||||
"vmovq %[mask_k], %%xmm14\n\t"
|
||||
"vpunpcklqdq %%xmm14, %%xmm14, %%xmm14\n\t"
|
||||
"vinserti128 $1, %%xmm14, %%ymm14, %%ymm14\n\t"
|
||||
|
||||
/* Initialize states */
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
|
||||
/* state0 */
|
||||
"vmovq 0*8(%%rax), %%xmm0\n\t"
|
||||
"vpunpcklqdq %%xmm0, %%xmm0, %%xmm0\n\t"
|
||||
"vinserti128 $1, %%xmm0, %%ymm0, %%ymm0\n\t"
|
||||
"vpand %%ymm12, %%ymm9, %%ymm15\n\t"
|
||||
"vpxor %%ymm15, %%ymm0, %%ymm0\n\t"
|
||||
|
||||
/* state1 */
|
||||
"vmovq 1*8(%%rax), %%xmm1\n\t"
|
||||
"vpunpcklqdq %%xmm1, %%xmm1, %%xmm1\n\t"
|
||||
"vinserti128 $1, %%xmm1, %%ymm1, %%ymm1\n\t"
|
||||
"vpand %%ymm13, %%ymm10, %%ymm15\n\t"
|
||||
"vpxor %%ymm15, %%ymm1, %%ymm1\n\t"
|
||||
|
||||
/* state2 */
|
||||
"vmovq 2*8(%%rax), %%xmm2\n\t"
|
||||
"vpunpcklqdq %%xmm2, %%xmm2, %%xmm2\n\t"
|
||||
"vinserti128 $1, %%xmm2, %%ymm2, %%ymm2\n\t"
|
||||
"vpand %%ymm14, %%ymm9, %%ymm15\n\t"
|
||||
"vpxor %%ymm15, %%ymm2, %%ymm2\n\t"
|
||||
|
||||
/* state3 */
|
||||
"vmovq 3*8(%%rax), %%xmm3\n\t"
|
||||
"vpunpcklqdq %%xmm3, %%xmm3, %%xmm3\n\t"
|
||||
"vinserti128 $1, %%xmm3, %%ymm3, %%ymm3\n\t"
|
||||
"vpand %%ymm12, %%ymm10, %%ymm15\n\t"
|
||||
"vpxor %%ymm15, %%ymm3, %%ymm3\n\t"
|
||||
|
||||
/* state4 */
|
||||
"vmovq 4*8(%%rax), %%xmm4\n\t"
|
||||
"vpunpcklqdq %%xmm4, %%xmm4, %%xmm4\n\t"
|
||||
"vinserti128 $1, %%xmm4, %%ymm4, %%ymm4\n\t"
|
||||
"vpand %%ymm13, %%ymm9, %%ymm15\n\t"
|
||||
"vpxor %%ymm15, %%ymm4, %%ymm4\n\t"
|
||||
|
||||
/* state5 */
|
||||
"vmovq 5*8(%%rax), %%xmm5\n\t"
|
||||
"vpunpcklqdq %%xmm5, %%xmm5, %%xmm5\n\t"
|
||||
"vinserti128 $1, %%xmm5, %%ymm5, %%ymm5\n\t"
|
||||
"vpand %%ymm14, %%ymm10, %%ymm15\n\t"
|
||||
"vpxor %%ymm15, %%ymm5, %%ymm5\n\t"
|
||||
|
||||
/* lo5 and hi5 */
|
||||
"vmovq 6*8(%%rax), %%xmm6\n\t"
|
||||
"vpunpcklqdq %%xmm6, %%xmm6, %%xmm6\n\t"
|
||||
"vinserti128 $1, %%xmm6, %%ymm6, %%ymm6\n\t"
|
||||
"vmovdqa %%ymm6, %%ymm7\n\t"
|
||||
|
||||
/* Conditional loop */
|
||||
"cmp $96, %[q]\n\t"
|
||||
"jbe 3f\n\t"
|
||||
|
||||
"1:\n\t"
|
||||
/* Loop step idx = 0 */
|
||||
AVX_LOAD(0, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm0, %%ymm0\n\t"
|
||||
AVX_LOAD(1, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm1, %%ymm1\n\t"
|
||||
AVX_MULT("%%ymm0", "%%ymm1", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm6, %%ymm0\n\t"
|
||||
"vmovdqa %%ymm8, %%ymm10\n\t"
|
||||
|
||||
/* Loop step idx = 1 */
|
||||
AVX_LOAD(2, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm1, %%ymm1\n\t"
|
||||
AVX_LOAD(3, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm2, %%ymm2\n\t"
|
||||
AVX_MULT("%%ymm1", "%%ymm2", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm10, %%ymm1\n\t"
|
||||
"vmovdqa %%ymm8, %%ymm10\n\t"
|
||||
|
||||
/* Loop step idx = 2 */
|
||||
AVX_LOAD(4, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm2, %%ymm2\n\t"
|
||||
AVX_LOAD(5, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm3, %%ymm3\n\t"
|
||||
AVX_MULT("%%ymm2", "%%ymm3", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm10, %%ymm2\n\t"
|
||||
"vmovdqa %%ymm8, %%ymm10\n\t"
|
||||
|
||||
/* Loop step idx = 3 */
|
||||
AVX_LOAD(6, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm3, %%ymm3\n\t"
|
||||
AVX_LOAD(7, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm4, %%ymm4\n\t"
|
||||
AVX_MULT("%%ymm3", "%%ymm4", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm10, %%ymm3\n\t"
|
||||
"vmovdqa %%ymm8, %%ymm10\n\t"
|
||||
|
||||
/* Loop step idx = 4 */
|
||||
AVX_LOAD(8, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm4, %%ymm4\n\t"
|
||||
AVX_LOAD(9, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm5, %%ymm5\n\t"
|
||||
AVX_MULT("%%ymm4", "%%ymm5", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm10, %%ymm4\n\t"
|
||||
"vmovdqa %%ymm8, %%ymm10\n\t"
|
||||
|
||||
/* Loop step idx = 5 */
|
||||
AVX_LOAD(10, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm5, %%ymm5\n\t"
|
||||
AVX_LOAD(11, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm0, %%ymm0\n\t"
|
||||
AVX_MULT("%%ymm5", "%%ymm0", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm10, %%ymm5\n\t"
|
||||
"vmovdqa %%ymm8, %%ymm6\n\t"
|
||||
"vmovdqa %%ymm9, %%ymm7\n\t"
|
||||
|
||||
"add $96, %[blocks0]\n\t"
|
||||
"add $96, %[blocks1]\n\t"
|
||||
"add $96, %[blocks2]\n\t"
|
||||
"add $96, %[blocks3]\n\t"
|
||||
"sub $96, %[q]\n\t"
|
||||
"cmp $96, %[q]\n\t"
|
||||
"ja 1b\n\t"
|
||||
|
||||
"vpxor %%ymm6, %%ymm0, %%ymm0\n\t"
|
||||
"3:\n\t"
|
||||
|
||||
/* Tail */
|
||||
"vmovdqa %%ymm1, %%ymm10\n\t" /* xor0 = state1 */
|
||||
"vmovdqa %%ymm2, %%ymm11\n\t" /* xor1 = state2 */
|
||||
"vmovdqa %%ymm3, %%ymm12\n\t" /* xor2 = state3 */
|
||||
"vmovdqa %%ymm4, %%ymm13\n\t" /* xor3 = state4 */
|
||||
|
||||
"cmp $32, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
AVX_LOAD(0, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm0, %%ymm0\n\t"
|
||||
AVX_LOAD(1, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm1, %%ymm1\n\t"
|
||||
AVX_MULT("%%ymm0", "%%ymm1", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm8, %%ymm10\n\t"
|
||||
|
||||
"cmp $48, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
AVX_LOAD(2, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm1, %%ymm1\n\t"
|
||||
AVX_LOAD(3, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm2, %%ymm2\n\t"
|
||||
AVX_MULT("%%ymm1", "%%ymm2", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm8, %%ymm11\n\t"
|
||||
|
||||
"cmp $64, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
AVX_LOAD(4, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm2, %%ymm2\n\t"
|
||||
AVX_LOAD(5, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm3, %%ymm3\n\t"
|
||||
AVX_MULT("%%ymm2", "%%ymm3", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm8, %%ymm12\n\t"
|
||||
|
||||
"cmp $80, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
AVX_LOAD(6, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm3, %%ymm3\n\t"
|
||||
AVX_LOAD(7, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm4, %%ymm4\n\t"
|
||||
AVX_MULT("%%ymm3", "%%ymm4", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm8, %%ymm13\n\t"
|
||||
|
||||
"4:\n\t"
|
||||
|
||||
/* Compute end pointers */
|
||||
"movq %[blocks0], %%r8\n\t"
|
||||
"add %[q], %%r8\n\t"
|
||||
"movq %[blocks1], %%r9\n\t"
|
||||
"add %[q], %%r9\n\t"
|
||||
"movq %[blocks2], %%r10\n\t"
|
||||
"add %[q], %%r10\n\t"
|
||||
"movq %[blocks3], %%r11\n\t"
|
||||
"add %[q], %%r11\n\t"
|
||||
|
||||
/* Unconditional tail step 1 */
|
||||
AVX_LOAD_NEG(32, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm4, %%ymm4\n\t"
|
||||
AVX_LOAD_NEG(24, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm5, %%ymm5\n\t"
|
||||
AVX_MULT("%%ymm4", "%%ymm5", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm8, %%ymm14\n\t"
|
||||
|
||||
/* Unconditional tail step 2 */
|
||||
AVX_LOAD_NEG(16, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm5, %%ymm5\n\t"
|
||||
AVX_LOAD_NEG(8, "%%ymm15")
|
||||
"vpxor %%ymm15, %%ymm0, %%ymm0\n\t"
|
||||
AVX_MULT("%%ymm5", "%%ymm0", "%%ymm8", "%%ymm9")
|
||||
"vpxor %%ymm9, %%ymm8, %%ymm15\n\t"
|
||||
|
||||
/* Subtract states to compute i, j, k */
|
||||
"vpsubq %%ymm1, %%ymm0, %%ymm0\n\t"
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
"vmovq 7*8(%%rax), %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
"vpxor %%ymm8, %%ymm0, %%ymm0\n\t" /* ymm0 = i */
|
||||
|
||||
"vpsubq %%ymm3, %%ymm2, %%ymm2\n\t"
|
||||
"vmovq 8*8(%%rax), %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
"vpxor %%ymm8, %%ymm2, %%ymm2\n\t" /* ymm2 = j */
|
||||
|
||||
"vpsubq %%ymm5, %%ymm4, %%ymm4\n\t"
|
||||
"vmovq 9*8(%%rax), %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
"vpxor %%ymm8, %%ymm4, %%ymm4\n\t" /* ymm4 = k */
|
||||
|
||||
/* Rotations and size subtraction */
|
||||
"vmovq %[rot], %%xmm8\n\t"
|
||||
"vmovq %[rot_diff], %%xmm9\n\t"
|
||||
|
||||
"vpsllq %%xmm8, %%ymm0, %%ymm1\n\t"
|
||||
"vpsrlq %%xmm9, %%ymm0, %%ymm0\n\t"
|
||||
"vpor %%ymm0, %%ymm1, %%ymm0\n\t"
|
||||
|
||||
"vpsrlq %%xmm8, %%ymm2, %%ymm3\n\t"
|
||||
"vpsllq %%xmm9, %%ymm2, %%ymm2\n\t"
|
||||
"vpor %%ymm2, %%ymm3, %%ymm2\n\t"
|
||||
|
||||
"vmovq %[size], %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
"vpsubq %%ymm8, %%ymm4, %%ymm4\n\t"
|
||||
|
||||
/* i, j, k corrections */
|
||||
"vpsubq %%ymm13, %%ymm0, %%ymm0\n\t"
|
||||
"vpsubq %%ymm14, %%ymm0, %%ymm0\n\t"
|
||||
|
||||
"vpsubq %%ymm15, %%ymm2, %%ymm2\n\t"
|
||||
"vpsubq %%ymm10, %%ymm2, %%ymm2\n\t"
|
||||
|
||||
"vpsubq %%ymm11, %%ymm4, %%ymm4\n\t"
|
||||
"vpsubq %%ymm12, %%ymm4, %%ymm4\n\t"
|
||||
|
||||
/* Final multiplications */
|
||||
AVX_MULT("%%ymm0", "%%ymm2", "%%ymm10", "%%ymm11")
|
||||
AVX_MULT("%%ymm2", "%%ymm4", "%%ymm12", "%%ymm13")
|
||||
AVX_MULT("%%ymm4", "%%ymm0", "%%ymm8", "%%ymm9")
|
||||
|
||||
"vpxor %%ymm11, %%ymm8, %%ymm0\n\t"
|
||||
"vpxor %%ymm13, %%ymm10, %%ymm2\n\t"
|
||||
"vpxor %%ymm9, %%ymm12, %%ymm4\n\t"
|
||||
|
||||
/* Multiply by constants 10, 11, 12 */
|
||||
"vmovq 10*8(%%rax), %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
AVX_MULT("%%ymm0", "%%ymm8", "%%ymm10", "%%ymm11")
|
||||
|
||||
"vmovq 11*8(%%rax), %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
AVX_MULT("%%ymm2", "%%ymm8", "%%ymm12", "%%ymm13")
|
||||
|
||||
"vmovq 12*8(%%rax), %%xmm8\n\t"
|
||||
"vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t"
|
||||
"vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t"
|
||||
AVX_MULT("%%ymm4", "%%ymm8", "%%ymm6", "%%ymm7")
|
||||
|
||||
/* Final digests XOR */
|
||||
"vpxor %%ymm13, %%ymm10, %%ymm10\n\t"
|
||||
"vpxor %%ymm6, %%ymm10, %%ymm10\n\t"
|
||||
|
||||
"vpxor %%ymm12, %%ymm11, %%ymm11\n\t"
|
||||
"vpxor %%ymm7, %%ymm11, %%ymm11\n\t"
|
||||
|
||||
/* Extract high parts of digests */
|
||||
"vextracti128 $1, %%ymm10, %%xmm12\n\t"
|
||||
"vextracti128 $1, %%ymm11, %%xmm13\n\t"
|
||||
|
||||
/* Store digest0 */
|
||||
"movq %[digest0], %%rax\n\t"
|
||||
"vmovq %%xmm10, (%%rax)\n\t"
|
||||
"vmovq %%xmm11, 8(%%rax)\n\t"
|
||||
|
||||
/* Store digest1 */
|
||||
"movq %[digest1], %%rcx\n\t"
|
||||
"vmovhps %%xmm10, (%%rcx)\n\t"
|
||||
"vmovhps %%xmm11, 8(%%rcx)\n\t"
|
||||
|
||||
/* Store digest2 */
|
||||
"movq %[digest2], %%rdx\n\t"
|
||||
"vmovq %%xmm12, (%%rdx)\n\t"
|
||||
"vmovq %%xmm13, 8(%%rdx)\n\t"
|
||||
|
||||
/* Store digest3 */
|
||||
"movq %[digest3], %%rsi\n\t"
|
||||
"vmovhps %%xmm12, (%%rsi)\n\t"
|
||||
"vmovhps %%xmm13, 8(%%rsi)\n\t"
|
||||
|
||||
: [blocks0] "+r" (blocks0),
|
||||
[blocks1] "+r" (blocks1),
|
||||
[blocks2] "+r" (blocks2),
|
||||
[blocks3] "+r" (blocks3),
|
||||
[q] "+r" (q)
|
||||
: [seed0] "m" (seed0),
|
||||
[seed1] "m" (seed1),
|
||||
[seed2] "m" (seed2),
|
||||
[seed3] "m" (seed3),
|
||||
[digest0] "m" (digest0),
|
||||
[digest1] "m" (digest1),
|
||||
[digest2] "m" (digest2),
|
||||
[digest3] "m" (digest3),
|
||||
[const_ptr] "m" (MUSEAIR_CONSTANT),
|
||||
[mask_i] "m" (const_mask_i),
|
||||
[mask_j] "m" (const_mask_j),
|
||||
[mask_k] "m" (const_mask_k),
|
||||
[rot] "m" (rot),
|
||||
[rot_diff] "m" (rot_diff),
|
||||
[size] "m" (size)
|
||||
: "rax", "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11",
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
|
||||
"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory"
|
||||
);
|
||||
}
|
||||
|
||||
#else /* !__x86_64__ */
|
||||
|
||||
void MuseAirLoongAVX(
|
||||
const void* data0, const void* data1, const void* data2, const void* data3,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3,
|
||||
uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3
|
||||
)
|
||||
{
|
||||
MuseAirLoong(data0, size, seed0, digest0);
|
||||
MuseAirLoong(data1, size, seed1, digest1);
|
||||
MuseAirLoong(data2, size, seed2, digest2);
|
||||
MuseAirLoong(data3, size, seed3, digest3);
|
||||
}
|
||||
|
||||
#endif /* __x86_64__ */
|
||||
|
||||
486
cmdline/museairsse.c
Normal file
486
cmdline/museairsse.c
Normal file
|
|
@ -0,0 +1,486 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Copyright (C) 2026 Andrea Mazzoleni
|
||||
|
||||
#if defined(__x86_64__)
|
||||
|
||||
#define SSE_LOAD(offset, dest) \
|
||||
"movq " #offset "*8(%[blocks0]), " dest "\n\t" \
|
||||
"movhps " #offset "*8(%[blocks1]), " dest "\n\t"
|
||||
|
||||
#define SSE_LOAD_NEG(offset, dest) \
|
||||
"movq -" #offset "(%%r8), " dest "\n\t" \
|
||||
"movhps -" #offset "(%%r9), " dest "\n\t"
|
||||
|
||||
#define SSE_MULT(a, b, lo, hi, s0, s1) \
|
||||
/* 1. Compute B_h in s0 */ \
|
||||
"movdqa " b ", " s0 "\n\t" \
|
||||
"psrlq $32, " s0 "\n\t" \
|
||||
/* 2. Compute P_lh in s1 */ \
|
||||
"movdqa " a ", " s1 "\n\t" \
|
||||
"pmuludq " s0 ", " s1 "\n\t" \
|
||||
/* 3. Compute A_h in hi */ \
|
||||
"movdqa " a ", " hi "\n\t" \
|
||||
"psrlq $32, " hi "\n\t" \
|
||||
/* 4. Compute P_hh in lo */ \
|
||||
"movdqa " hi ", " lo "\n\t" \
|
||||
"pmuludq " s0 ", " lo "\n\t" \
|
||||
/* 5. Compute P_hl in hi */ \
|
||||
"pmuludq " b ", " hi "\n\t" \
|
||||
/* 6. Compute P_ll in s0 */ \
|
||||
"movdqa " a ", " s0 "\n\t" \
|
||||
"pmuludq " b ", " s0 "\n\t" \
|
||||
/* 7. M = P_lh + P_hl */ \
|
||||
"paddq " hi ", " s1 "\n\t" /* s1 = M */ \
|
||||
/* 8. Move P_hh from lo to hi */ \
|
||||
"movdqa " lo ", " hi "\n\t" /* hi = P_hh */ \
|
||||
/* 9. Generate mask 0xffffffff in xmm15 */ \
|
||||
"pcmpeqd %%xmm15, %%xmm15\n\t" \
|
||||
"psrlq $32, %%xmm15\n\t" \
|
||||
/* 10. Split M into s1 = M_low, lo = M_high */ \
|
||||
"movdqa " s1 ", " lo "\n\t" \
|
||||
"pand %%xmm15, " s1 "\n\t" /* s1 = M_low */ \
|
||||
"psrlq $32, " lo "\n\t" /* lo = M_high */ \
|
||||
/* 11. Split P_ll (in s0) into s0 = P_lll, xmm15 = P_llh (overwriting mask) */ \
|
||||
"movdqa " s0 ", %%xmm15\n\t" \
|
||||
"psrlq $32, %%xmm15\n\t" /* xmm15 = P_llh */ \
|
||||
"pand %%xmm15, " s0 "\n\t" /* s0 = P_lll */ /* Wait! xmm15 contains P_llh which has 0 in high 32 bits, but low 32 bits is P_llh, not 0xffffffff! */ \
|
||||
/* Ah! We cannot pand with xmm15 because xmm15 is NOT the mask anymore! */ \
|
||||
/* To fix this, let's just use the shift method for s0! */ \
|
||||
/* "psllq $32, s0; psrlq $32, s0" is safer! */ \
|
||||
|
||||
#define SSE_MULT_FIXED(a, b, lo, hi, s0, s1) \
|
||||
/* 1. Compute B_h in s0 */ \
|
||||
"movdqa " b ", " s0 "\n\t" \
|
||||
"psrlq $32, " s0 "\n\t" \
|
||||
/* 2. Compute P_lh in s1 */ \
|
||||
"movdqa " a ", " s1 "\n\t" \
|
||||
"pmuludq " s0 ", " s1 "\n\t" \
|
||||
/* 3. Compute A_h in hi */ \
|
||||
"movdqa " a ", " hi "\n\t" \
|
||||
"psrlq $32, " hi "\n\t" \
|
||||
/* 4. Compute P_hh in lo */ \
|
||||
"movdqa " hi ", " lo "\n\t" \
|
||||
"pmuludq " s0 ", " lo "\n\t" \
|
||||
/* 5. Compute P_hl in hi */ \
|
||||
"pmuludq " b ", " hi "\n\t" \
|
||||
/* 6. Compute P_ll in s0 */ \
|
||||
"movdqa " a ", " s0 "\n\t" \
|
||||
"pmuludq " b ", " s0 "\n\t" \
|
||||
/* 7. M = P_lh + P_hl */ \
|
||||
"paddq " hi ", " s1 "\n\t" /* s1 = M */ \
|
||||
/* 8. Move P_hh from lo to hi */ \
|
||||
"movdqa " lo ", " hi "\n\t" /* hi = P_hh */ \
|
||||
/* 9. Generate mask 0xffffffff in xmm15 */ \
|
||||
"pcmpeqd %%xmm15, %%xmm15\n\t" \
|
||||
"psrlq $32, %%xmm15\n\t" \
|
||||
/* 10. Split M into s1 = M_low, lo = M_high */ \
|
||||
"movdqa " s1 ", " lo "\n\t" \
|
||||
"pand %%xmm15, " s1 "\n\t" /* s1 = M_low */ \
|
||||
"psrlq $32, " lo "\n\t" /* lo = M_high */ \
|
||||
/* 11. Split P_ll (in s0) into s0 = P_lll, xmm15 = P_llh (overwriting mask) */ \
|
||||
"movdqa " s0 ", %%xmm15\n\t" \
|
||||
"psrlq $32, %%xmm15\n\t" /* xmm15 = P_llh */ \
|
||||
"psllq $32, " s0 "\n\t" \
|
||||
"psrlq $32, " s0 "\n\t" /* s0 = P_lll */ \
|
||||
/* 12. Compute S1 = P_llh + M_low */ \
|
||||
"paddq " s1 ", %%xmm15\n\t" /* xmm15 = S1 */ \
|
||||
/* 13. Split S1 into s1 = C1, xmm15 = S1_l */ \
|
||||
"movdqa %%xmm15, " s1 "\n\t" \
|
||||
"psllq $32, %%xmm15\n\t" \
|
||||
"psrlq $32, %%xmm15\n\t" /* xmm15 = S1_l */ \
|
||||
"psrlq $32, " s1 "\n\t" /* s1 = C1 */ \
|
||||
/* 14. Assemble final low (in lo) */ \
|
||||
"psllq $32, %%xmm15\n\t" \
|
||||
"paddq " s0 ", %%xmm15\n\t" \
|
||||
"movdqa %%xmm15, " lo "\n\t" /* lo = final low */ \
|
||||
/* 15. Split P_hh (in hi) into s0 = P_hhl, hi = P_hhh */ \
|
||||
"movdqa " hi ", " s0 "\n\t" \
|
||||
"psllq $32, " s0 "\n\t" \
|
||||
"psrlq $32, " s0 "\n\t" /* s0 = P_hhl */ \
|
||||
"psrlq $32, " hi "\n\t" /* hi = P_hhh */ \
|
||||
/* 16. Compute final high (in hi) */ \
|
||||
"paddq " lo ", " s0 "\n\t" /* s0 = M_high + P_hhl */ \
|
||||
"paddq " s1 ", " s0 "\n\t" /* s0 = M_high + P_hhl + C1 */ \
|
||||
"psllq $32, " hi "\n\t" \
|
||||
"paddq " s0 ", " hi "\n\t" /* hi = final high */
|
||||
|
||||
void MuseAirLoongSSE(
|
||||
const void* data0, const void* data1,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1,
|
||||
uint8_t* digest0, uint8_t* digest1
|
||||
)
|
||||
{
|
||||
uint64_t buf0[12];
|
||||
uint64_t buf1[12];
|
||||
const uint8_t* blocks0 = data0;
|
||||
const uint8_t* blocks1 = data1;
|
||||
size_t q = size;
|
||||
|
||||
if (q < 32) {
|
||||
memset(buf0, 0, 32);
|
||||
memset(buf1, 0, 32);
|
||||
if (q > 0) {
|
||||
memcpy(buf0, blocks0, q);
|
||||
memcpy(buf1, blocks1, q);
|
||||
}
|
||||
blocks0 = (const uint8_t*)buf0;
|
||||
blocks1 = (const uint8_t*)buf1;
|
||||
q = 32;
|
||||
}
|
||||
|
||||
uint64_t rot = size & 63;
|
||||
uint64_t rot_diff = 64 - rot;
|
||||
static const uint64_t const_mask_i = 01555555555555555555555ULL;
|
||||
static const uint64_t const_mask_j = 01333333333333333333333ULL;
|
||||
static const uint64_t const_mask_k = 00666666666666666666666ULL;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
/* Load seed0 and seed1 */
|
||||
"movq %[seed0], %%rax\n\t"
|
||||
"movq (%%rax), %%xmm9\n\t"
|
||||
"movq %[seed1], %%rcx\n\t"
|
||||
"movhps (%%rcx), %%xmm9\n\t" /* xmm9 = seed_a */
|
||||
|
||||
"movq %[seed0], %%rax\n\t"
|
||||
"movq 8(%%rax), %%xmm10\n\t"
|
||||
"movq %[seed1], %%rcx\n\t"
|
||||
"movhps 8(%%rcx), %%xmm10\n\t" /* xmm10 = seed_b */
|
||||
|
||||
/* Load and duplicate masks */
|
||||
"movq %[mask_i], %%xmm12\n\t"
|
||||
"punpcklqdq %%xmm12, %%xmm12\n\t"
|
||||
|
||||
"movq %[mask_j], %%xmm13\n\t"
|
||||
"punpcklqdq %%xmm13, %%xmm13\n\t"
|
||||
|
||||
"movq %[mask_k], %%xmm14\n\t"
|
||||
"punpcklqdq %%xmm14, %%xmm14\n\t"
|
||||
|
||||
/* Initialize states */
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
|
||||
/* state0 */
|
||||
"movq 0*8(%%rax), %%xmm0\n\t"
|
||||
"punpcklqdq %%xmm0, %%xmm0\n\t"
|
||||
"movdqa %%xmm9, %%xmm15\n\t"
|
||||
"pand %%xmm12, %%xmm15\n\t"
|
||||
"pxor %%xmm15, %%xmm0\n\t"
|
||||
|
||||
/* state1 */
|
||||
"movq 1*8(%%rax), %%xmm1\n\t"
|
||||
"punpcklqdq %%xmm1, %%xmm1\n\t"
|
||||
"movdqa %%xmm10, %%xmm15\n\t"
|
||||
"pand %%xmm13, %%xmm15\n\t"
|
||||
"pxor %%xmm15, %%xmm1\n\t"
|
||||
|
||||
/* state2 */
|
||||
"movq 2*8(%%rax), %%xmm2\n\t"
|
||||
"punpcklqdq %%xmm2, %%xmm2\n\t"
|
||||
"movdqa %%xmm9, %%xmm15\n\t"
|
||||
"pand %%xmm14, %%xmm15\n\t"
|
||||
"pxor %%xmm15, %%xmm2\n\t"
|
||||
|
||||
/* state3 */
|
||||
"movq 3*8(%%rax), %%xmm3\n\t"
|
||||
"punpcklqdq %%xmm3, %%xmm3\n\t"
|
||||
"movdqa %%xmm10, %%xmm15\n\t"
|
||||
"pand %%xmm12, %%xmm15\n\t"
|
||||
"pxor %%xmm15, %%xmm3\n\t"
|
||||
|
||||
/* state4 */
|
||||
"movq 4*8(%%rax), %%xmm4\n\t"
|
||||
"punpcklqdq %%xmm4, %%xmm4\n\t"
|
||||
"movdqa %%xmm9, %%xmm15\n\t"
|
||||
"pand %%xmm13, %%xmm15\n\t"
|
||||
"pxor %%xmm15, %%xmm4\n\t"
|
||||
|
||||
/* state5 */
|
||||
"movq 5*8(%%rax), %%xmm5\n\t"
|
||||
"punpcklqdq %%xmm5, %%xmm5\n\t"
|
||||
"movdqa %%xmm10, %%xmm15\n\t"
|
||||
"pand %%xmm14, %%xmm15\n\t"
|
||||
"pxor %%xmm15, %%xmm5\n\t"
|
||||
|
||||
/* lo5 and hi5 */
|
||||
"movq 6*8(%%rax), %%xmm6\n\t"
|
||||
"punpcklqdq %%xmm6, %%xmm6\n\t"
|
||||
"movdqa %%xmm6, %%xmm7\n\t"
|
||||
|
||||
/* Conditional loop */
|
||||
"cmp $96, %[q]\n\t"
|
||||
"jbe 3f\n\t"
|
||||
|
||||
"1:\n\t"
|
||||
/* Loop step idx = 0 */
|
||||
SSE_LOAD(0, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm0\n\t"
|
||||
SSE_LOAD(1, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm1\n\t"
|
||||
SSE_MULT_FIXED("%%xmm0", "%%xmm1", "%%xmm8", "%%xmm9", "%%xmm10", "%%xmm11")
|
||||
"movdqa %%xmm6, %%xmm0\n\t"
|
||||
"pxor %%xmm9, %%xmm0\n\t"
|
||||
"movdqa %%xmm8, %%xmm10\n\t"
|
||||
|
||||
/* Loop step idx = 1 */
|
||||
SSE_LOAD(2, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm1\n\t"
|
||||
SSE_LOAD(3, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm2\n\t"
|
||||
SSE_MULT_FIXED("%%xmm1", "%%xmm2", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15")
|
||||
"movdqa %%xmm10, %%xmm1\n\t"
|
||||
"pxor %%xmm9, %%xmm1\n\t"
|
||||
"movdqa %%xmm8, %%xmm10\n\t"
|
||||
|
||||
/* Loop step idx = 2 */
|
||||
SSE_LOAD(4, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm2\n\t"
|
||||
SSE_LOAD(5, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm3\n\t"
|
||||
SSE_MULT_FIXED("%%xmm2", "%%xmm3", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15")
|
||||
"movdqa %%xmm10, %%xmm2\n\t"
|
||||
"pxor %%xmm9, %%xmm2\n\t"
|
||||
"movdqa %%xmm8, %%xmm10\n\t"
|
||||
|
||||
/* Loop step idx = 3 */
|
||||
SSE_LOAD(6, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm3\n\t"
|
||||
SSE_LOAD(7, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm4\n\t"
|
||||
SSE_MULT_FIXED("%%xmm3", "%%xmm4", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15")
|
||||
"movdqa %%xmm10, %%xmm3\n\t"
|
||||
"pxor %%xmm9, %%xmm3\n\t"
|
||||
"movdqa %%xmm8, %%xmm10\n\t"
|
||||
|
||||
/* Loop step idx = 4 */
|
||||
SSE_LOAD(8, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm4\n\t"
|
||||
SSE_LOAD(9, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm5\n\t"
|
||||
SSE_MULT_FIXED("%%xmm4", "%%xmm5", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15")
|
||||
"movdqa %%xmm10, %%xmm4\n\t"
|
||||
"pxor %%xmm9, %%xmm4\n\t"
|
||||
"movdqa %%xmm8, %%xmm10\n\t"
|
||||
|
||||
/* Loop step idx = 5 */
|
||||
SSE_LOAD(10, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm5\n\t"
|
||||
SSE_LOAD(11, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm0\n\t"
|
||||
SSE_MULT_FIXED("%%xmm5", "%%xmm0", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15")
|
||||
"movdqa %%xmm10, %%xmm5\n\t"
|
||||
"pxor %%xmm9, %%xmm5\n\t"
|
||||
"movdqa %%xmm8, %%xmm6\n\t"
|
||||
"movdqa %%xmm9, %%xmm7\n\t"
|
||||
|
||||
"add $96, %[blocks0]\n\t"
|
||||
"add $96, %[blocks1]\n\t"
|
||||
"sub $96, %[q]\n\t"
|
||||
"cmp $96, %[q]\n\t"
|
||||
"ja 1b\n\t"
|
||||
|
||||
"pxor %%xmm6, %%xmm0\n\t"
|
||||
"3:\n\t"
|
||||
|
||||
/* Tail */
|
||||
"movdqa %%xmm1, %%xmm10\n\t" /* xor0 = state1 */
|
||||
"movdqa %%xmm2, %%xmm11\n\t" /* xor1 = state2 */
|
||||
"movdqa %%xmm3, %%xmm12\n\t" /* xor2 = state3 */
|
||||
"movdqa %%xmm4, %%xmm13\n\t" /* xor3 = state4 */
|
||||
|
||||
"cmp $32, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
SSE_LOAD(0, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm0\n\t"
|
||||
SSE_LOAD(1, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm1\n\t"
|
||||
SSE_MULT_FIXED("%%xmm0", "%%xmm1", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15")
|
||||
"movdqa %%xmm8, %%xmm10\n\t"
|
||||
"pxor %%xmm9, %%xmm10\n\t"
|
||||
|
||||
"cmp $48, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
SSE_LOAD(2, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm1\n\t"
|
||||
SSE_LOAD(3, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm2\n\t"
|
||||
SSE_MULT_FIXED("%%xmm1", "%%xmm2", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15")
|
||||
"movdqa %%xmm8, %%xmm11\n\t"
|
||||
"pxor %%xmm9, %%xmm11\n\t"
|
||||
|
||||
"cmp $64, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
SSE_LOAD(4, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm2\n\t"
|
||||
SSE_LOAD(5, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm3\n\t"
|
||||
SSE_MULT_FIXED("%%xmm2", "%%xmm3", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15")
|
||||
"movdqa %%xmm8, %%xmm12\n\t"
|
||||
"pxor %%xmm9, %%xmm12\n\t"
|
||||
|
||||
"cmp $80, %[q]\n\t"
|
||||
"jbe 4f\n\t"
|
||||
|
||||
SSE_LOAD(6, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm3\n\t"
|
||||
SSE_LOAD(7, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm4\n\t"
|
||||
SSE_MULT_FIXED("%%xmm3", "%%xmm4", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15")
|
||||
"movdqa %%xmm8, %%xmm13\n\t"
|
||||
"pxor %%xmm9, %%xmm13\n\t"
|
||||
|
||||
"4:\n\t"
|
||||
|
||||
/* Compute end pointers */
|
||||
"movq %[blocks0], %%r8\n\t"
|
||||
"add %[q], %%r8\n\t"
|
||||
"movq %[blocks1], %%r9\n\t"
|
||||
"add %[q], %%r9\n\t"
|
||||
|
||||
/* Unconditional tail step 1 */
|
||||
SSE_LOAD_NEG(32, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm4\n\t"
|
||||
SSE_LOAD_NEG(24, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm5\n\t"
|
||||
SSE_MULT_FIXED("%%xmm4", "%%xmm5", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15")
|
||||
"movdqa %%xmm8, %%xmm14\n\t"
|
||||
"pxor %%xmm9, %%xmm14\n\t"
|
||||
|
||||
/* Unconditional tail step 2 */
|
||||
SSE_LOAD_NEG(16, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm5\n\t"
|
||||
SSE_LOAD_NEG(8, "%%xmm15")
|
||||
"pxor %%xmm15, %%xmm0\n\t"
|
||||
SSE_MULT_FIXED("%%xmm5", "%%xmm0", "%%xmm8", "%%xmm9", "%%xmm12", "%%xmm15")
|
||||
"movdqa %%xmm8, %%xmm15\n\t"
|
||||
"pxor %%xmm9, %%xmm15\n\t"
|
||||
|
||||
/* Subtract states to compute i, j, k */
|
||||
"psubq %%xmm1, %%xmm0\n\t"
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
"movq 7*8(%%rax), %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
"pxor %%xmm8, %%xmm0\n\t" /* xmm0 = i */
|
||||
|
||||
"psubq %%xmm3, %%xmm2\n\t"
|
||||
"movq 8*8(%%rax), %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
"pxor %%xmm8, %%xmm2\n\t" /* xmm2 = j */
|
||||
|
||||
"psubq %%xmm5, %%xmm4\n\t"
|
||||
"movq 9*8(%%rax), %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
"pxor %%xmm8, %%xmm4\n\t" /* xmm4 = k */
|
||||
|
||||
/* Rotations and size subtraction */
|
||||
"movq %[rot], %%xmm8\n\t"
|
||||
"movq %[rot_diff], %%xmm9\n\t"
|
||||
|
||||
"movdqa %%xmm0, %%xmm1\n\t"
|
||||
"psllq %%xmm8, %%xmm0\n\t"
|
||||
"psrlq %%xmm9, %%xmm1\n\t"
|
||||
"por %%xmm1, %%xmm0\n\t"
|
||||
|
||||
"movdqa %%xmm2, %%xmm3\n\t"
|
||||
"psrlq %%xmm8, %%xmm2\n\t"
|
||||
"psllq %%xmm9, %%xmm3\n\t"
|
||||
"por %%xmm3, %%xmm2\n\t"
|
||||
|
||||
"movq %[size], %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
"psubq %%xmm8, %%xmm4\n\t"
|
||||
|
||||
/* i, j, k corrections */
|
||||
"psubq %%xmm13, %%xmm0\n\t"
|
||||
"psubq %%xmm14, %%xmm0\n\t"
|
||||
|
||||
"psubq %%xmm15, %%xmm2\n\t"
|
||||
"psubq %%xmm10, %%xmm2\n\t"
|
||||
|
||||
"psubq %%xmm11, %%xmm4\n\t"
|
||||
"psubq %%xmm12, %%xmm4\n\t"
|
||||
|
||||
/* Final multiplications */
|
||||
SSE_MULT_FIXED("%%xmm0", "%%xmm2", "%%xmm10", "%%xmm11", "%%xmm8", "%%xmm9")
|
||||
SSE_MULT_FIXED("%%xmm2", "%%xmm4", "%%xmm12", "%%xmm13", "%%xmm8", "%%xmm9")
|
||||
SSE_MULT_FIXED("%%xmm4", "%%xmm0", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15")
|
||||
|
||||
"movdqa %%xmm8, %%xmm0\n\t"
|
||||
"pxor %%xmm11, %%xmm0\n\t"
|
||||
|
||||
"movdqa %%xmm10, %%xmm2\n\t"
|
||||
"pxor %%xmm13, %%xmm2\n\t"
|
||||
|
||||
"movdqa %%xmm12, %%xmm4\n\t"
|
||||
"pxor %%xmm9, %%xmm4\n\t"
|
||||
|
||||
/* Multiply by constants 10, 11, 12 */
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
"movq 10*8(%%rax), %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
SSE_MULT_FIXED("%%xmm0", "%%xmm8", "%%xmm10", "%%xmm11", "%%xmm12", "%%xmm13")
|
||||
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
"movq 11*8(%%rax), %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
SSE_MULT_FIXED("%%xmm2", "%%xmm8", "%%xmm12", "%%xmm13", "%%xmm10", "%%xmm11")
|
||||
|
||||
"leaq %[const_ptr], %%rax\n\t"
|
||||
"movq 12*8(%%rax), %%xmm8\n\t"
|
||||
"punpcklqdq %%xmm8, %%xmm8\n\t"
|
||||
SSE_MULT_FIXED("%%xmm4", "%%xmm8", "%%xmm6", "%%xmm7", "%%xmm10", "%%xmm11")
|
||||
|
||||
/* Final digests XOR */
|
||||
"pxor %%xmm13, %%xmm10\n\t"
|
||||
"pxor %%xmm6, %%xmm10\n\t"
|
||||
|
||||
"pxor %%xmm12, %%xmm11\n\t"
|
||||
"pxor %%xmm7, %%xmm11\n\t"
|
||||
|
||||
/* Store back digests */
|
||||
"movq %[digest0], %%rax\n\t"
|
||||
"movq %%xmm10, (%%rax)\n\t"
|
||||
"movq %%xmm11, 8(%%rax)\n\t"
|
||||
|
||||
"movq %[digest1], %%rcx\n\t"
|
||||
"movhps %%xmm10, (%%rcx)\n\t"
|
||||
"movhps %%xmm11, 8(%%rcx)\n\t"
|
||||
|
||||
: [blocks0] "+r" (blocks0),
|
||||
[blocks1] "+r" (blocks1),
|
||||
[q] "+r" (q)
|
||||
: [seed0] "m" (seed0),
|
||||
[seed1] "m" (seed1),
|
||||
[digest0] "m" (digest0),
|
||||
[digest1] "m" (digest1),
|
||||
[const_ptr] "m" (MUSEAIR_CONSTANT),
|
||||
[mask_i] "m" (const_mask_i),
|
||||
[mask_j] "m" (const_mask_j),
|
||||
[mask_k] "m" (const_mask_k),
|
||||
[rot] "m" (rot),
|
||||
[rot_diff] "m" (rot_diff),
|
||||
[size] "m" (size)
|
||||
: "rax", "rcx", "rdx", "r8", "r9",
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
|
||||
"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory"
|
||||
);
|
||||
}
|
||||
|
||||
#else /* !__x86_64__ */
|
||||
|
||||
void MuseAirLoongSSE(
|
||||
const void* data0, const void* data1,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1,
|
||||
uint8_t* digest0, uint8_t* digest1
|
||||
)
|
||||
{
|
||||
MuseAirLoong(data0, size, seed0, digest0);
|
||||
MuseAirLoong(data1, size, seed1, digest1);
|
||||
}
|
||||
|
||||
#endif /* __x86_64__ */
|
||||
|
||||
|
|
@ -235,6 +235,132 @@ static void test_hash(void)
|
|||
}
|
||||
}
|
||||
|
||||
for (i = 0; TEST_SPOOKY2[i].data; ++i) {
|
||||
unsigned char digest0[HASH_MAX];
|
||||
unsigned char digest1[HASH_MAX];
|
||||
unsigned char buffer0[4096];
|
||||
unsigned char buffer1[4096];
|
||||
unsigned char seed0[HASH_MAX];
|
||||
unsigned char seed1[HASH_MAX];
|
||||
int len = TEST_SPOOKY2[i].len;
|
||||
unsigned k;
|
||||
|
||||
if (len > 4096) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Test vector too long\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
for (k = 0; TEST_SPOOKY2[k].data; ++k) {
|
||||
if (TEST_SPOOKY2[k].len == len && k != i) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_SPOOKY2[k].data) {
|
||||
k = i;
|
||||
}
|
||||
|
||||
memcpy(buffer0, TEST_SPOOKY2[i].data, len);
|
||||
memcpy(seed0, TEST_SPOOKY2[i].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer1, TEST_SPOOKY2[k].data, len);
|
||||
memcpy(seed1, TEST_SPOOKY2[k].seed, HASH_MAX);
|
||||
|
||||
SpookyHash128SSE(buffer0, buffer1, len, seed0, seed1, digest0, digest1);
|
||||
|
||||
if (memcmp(digest0, TEST_SPOOKY2[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed SpookyHash128SSE test 0 on vector %d\n", i);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest1, TEST_SPOOKY2[k].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed SpookyHash128SSE test 1 on vector %d and %d\n", i, k);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
/* SpookyHash128AVX self-test */
|
||||
for (i = 0; TEST_SPOOKY2[i].data; ++i) {
|
||||
unsigned char buffer0[4096];
|
||||
unsigned char buffer1[4096];
|
||||
unsigned char buffer2[4096];
|
||||
unsigned char buffer3[4096];
|
||||
unsigned char seed0[HASH_MAX];
|
||||
unsigned char seed1[HASH_MAX];
|
||||
unsigned char seed2[HASH_MAX];
|
||||
unsigned char seed3[HASH_MAX];
|
||||
unsigned char digest0[HASH_MAX];
|
||||
unsigned char digest1[HASH_MAX];
|
||||
unsigned char digest2[HASH_MAX];
|
||||
unsigned char digest3[HASH_MAX];
|
||||
int len = TEST_SPOOKY2[i].len;
|
||||
unsigned k, l, m;
|
||||
|
||||
for (k = 0; TEST_SPOOKY2[k].data; ++k) {
|
||||
if (TEST_SPOOKY2[k].len == len && k != i) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_SPOOKY2[k].data) k = i;
|
||||
|
||||
for (l = 0; TEST_SPOOKY2[l].data; ++l) {
|
||||
if (TEST_SPOOKY2[l].len == len && l != i && l != k) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_SPOOKY2[l].data) l = i;
|
||||
|
||||
for (m = 0; TEST_SPOOKY2[m].data; ++m) {
|
||||
if (TEST_SPOOKY2[m].len == len && m != i && m != k && m != l) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_SPOOKY2[m].data) m = i;
|
||||
|
||||
memcpy(buffer0, TEST_SPOOKY2[i].data, len);
|
||||
memcpy(seed0, TEST_SPOOKY2[i].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer1, TEST_SPOOKY2[k].data, len);
|
||||
memcpy(seed1, TEST_SPOOKY2[k].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer2, TEST_SPOOKY2[l].data, len);
|
||||
memcpy(seed2, TEST_SPOOKY2[l].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer3, TEST_SPOOKY2[m].data, len);
|
||||
memcpy(seed3, TEST_SPOOKY2[m].seed, HASH_MAX);
|
||||
|
||||
SpookyHash128AVX(buffer0, buffer1, buffer2, buffer3, len, seed0, seed1, seed2, seed3, digest0, digest1, digest2, digest3);
|
||||
|
||||
if (memcmp(digest0, TEST_SPOOKY2[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 0 on vector %d\n", i);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest1, TEST_SPOOKY2[k].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 1 on vector %d and %d\n", i, k);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest2, TEST_SPOOKY2[l].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 2 on vector %d and %d\n", i, l);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest3, TEST_SPOOKY2[m].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 3 on vector %d and %d\n", i, m);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; TEST_METRO[i].data; ++i) {
|
||||
unsigned char digest[HASH_MAX];
|
||||
memcpy(buffer_aligned, TEST_METRO[i].data, TEST_METRO[i].len);
|
||||
|
|
@ -261,6 +387,140 @@ static void test_hash(void)
|
|||
}
|
||||
}
|
||||
|
||||
/* MuseAir SSE self-test */
|
||||
for (i = 0; TEST_MUSEAIR[i].data; ++i) {
|
||||
unsigned char digest0[HASH_MAX];
|
||||
unsigned char digest1[HASH_MAX];
|
||||
unsigned char buffer0[4096];
|
||||
unsigned char buffer1[4096];
|
||||
unsigned char seed0[HASH_MAX];
|
||||
unsigned char seed1[HASH_MAX];
|
||||
int len = TEST_MUSEAIR[i].len;
|
||||
unsigned k;
|
||||
|
||||
if (len > 4096) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Test vector too long\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
for (k = 0; TEST_MUSEAIR[k].data; ++k) {
|
||||
if (TEST_MUSEAIR[k].len == len && k != i) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_MUSEAIR[k].data) {
|
||||
k = i;
|
||||
}
|
||||
|
||||
memcpy(buffer0, TEST_MUSEAIR[i].data, len);
|
||||
memcpy(seed0, TEST_MUSEAIR[i].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer1, TEST_MUSEAIR[k].data, len);
|
||||
memcpy(seed1, TEST_MUSEAIR[k].seed, HASH_MAX);
|
||||
|
||||
MuseAirLoongSSE(buffer0, buffer1, len, seed0, seed1, digest0, digest1);
|
||||
|
||||
if (memcmp(digest0, TEST_MUSEAIR[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed MuseAirLoongSSE test 0 on vector %d\n", i);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest1, TEST_MUSEAIR[k].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed MuseAirLoongSSE test 1 on vector %d and %d\n", i, k);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
/* MuseAir AVX self-test */
|
||||
for (i = 0; TEST_MUSEAIR[i].data; ++i) {
|
||||
unsigned char buffer0[4096];
|
||||
unsigned char buffer1[4096];
|
||||
unsigned char buffer2[4096];
|
||||
unsigned char buffer3[4096];
|
||||
unsigned char seed0[HASH_MAX];
|
||||
unsigned char seed1[HASH_MAX];
|
||||
unsigned char seed2[HASH_MAX];
|
||||
unsigned char seed3[HASH_MAX];
|
||||
unsigned char digest0[HASH_MAX];
|
||||
unsigned char digest1[HASH_MAX];
|
||||
unsigned char digest2[HASH_MAX];
|
||||
unsigned char digest3[HASH_MAX];
|
||||
int len = TEST_MUSEAIR[i].len;
|
||||
unsigned k, l, m;
|
||||
|
||||
if (len > 4096) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Test vector too long\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
for (k = 0; TEST_MUSEAIR[k].data; ++k) {
|
||||
if (TEST_MUSEAIR[k].len == len && k != i) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_MUSEAIR[k].data) k = i;
|
||||
|
||||
for (l = 0; TEST_MUSEAIR[l].data; ++l) {
|
||||
if (TEST_MUSEAIR[l].len == len && l != i && l != k) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_MUSEAIR[l].data) l = i;
|
||||
|
||||
for (m = 0; TEST_MUSEAIR[m].data; ++m) {
|
||||
if (TEST_MUSEAIR[m].len == len && m != i && m != k && m != l) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!TEST_MUSEAIR[m].data) m = i;
|
||||
|
||||
memcpy(buffer0, TEST_MUSEAIR[i].data, len);
|
||||
memcpy(seed0, TEST_MUSEAIR[i].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer1, TEST_MUSEAIR[k].data, len);
|
||||
memcpy(seed1, TEST_MUSEAIR[k].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer2, TEST_MUSEAIR[l].data, len);
|
||||
memcpy(seed2, TEST_MUSEAIR[l].seed, HASH_MAX);
|
||||
|
||||
memcpy(buffer3, TEST_MUSEAIR[m].data, len);
|
||||
memcpy(seed3, TEST_MUSEAIR[m].seed, HASH_MAX);
|
||||
|
||||
MuseAirLoongAVX(buffer0, buffer1, buffer2, buffer3, len, seed0, seed1, seed2, seed3, digest0, digest1, digest2, digest3);
|
||||
|
||||
if (memcmp(digest0, TEST_MUSEAIR[i].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 0 on vector %d\n", i);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest1, TEST_MUSEAIR[k].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 1 on vector %d and %d\n", i, k);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest2, TEST_MUSEAIR[l].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 2 on vector %d and %d\n", i, l);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (memcmp(digest3, TEST_MUSEAIR[m].digest, HASH_MAX) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 3 on vector %d and %d\n", i, m);
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
free(buffer_alloc);
|
||||
free(seed_alloc);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,8 +154,12 @@ void speed_hash(int nd, void** v, int size, int delta, int period)
|
|||
printf("%8s", "best");
|
||||
printf("%8s", "murmur3");
|
||||
printf("%8s", "spooky2");
|
||||
printf("%8s", "spk2sse");
|
||||
printf("%8s", "spk2avx");
|
||||
printf("%8s", "metro");
|
||||
printf("%8s", "museair");
|
||||
printf("%8s", "musesse");
|
||||
printf("%8s", "museavx");
|
||||
printf("\n");
|
||||
|
||||
printf("%8s", "hash");
|
||||
|
|
@ -178,6 +182,38 @@ void speed_hash(int nd, void** v, int size, int delta, int period)
|
|||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
if (raid_cpu_has_sse2()) {
|
||||
SPEED_START {
|
||||
for (j = 0; j < nd - 1; j += 2) {
|
||||
SpookyHash128SSE(v[j], v[j + 1], size, seed, seed, digest, digest);
|
||||
}
|
||||
if (j < nd) {
|
||||
SpookyHash128(v[j], size, seed, digest);
|
||||
}
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "-");
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
for (j = 0; j < nd - 3; j += 4) {
|
||||
SpookyHash128AVX(v[j], v[j + 1], v[j + 2], v[j + 3], size, seed, seed, seed, seed, digest, digest, digest, digest);
|
||||
}
|
||||
for (; j < nd; ++j) {
|
||||
SpookyHash128(v[j], size, seed, digest);
|
||||
}
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "-");
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
for (j = 0; j < nd; ++j)
|
||||
memhash(HASH_METRO, seed, digest, v[j], size);
|
||||
|
|
@ -192,6 +228,38 @@ void speed_hash(int nd, void** v, int size, int delta, int period)
|
|||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
if (raid_cpu_has_sse2()) {
|
||||
SPEED_START {
|
||||
for (j = 0; j < nd - 1; j += 2) {
|
||||
MuseAirLoongSSE(v[j], v[j + 1], size, seed, seed, digest, digest);
|
||||
}
|
||||
if (j < nd) {
|
||||
MuseAirLoong(v[j], size, seed, digest);
|
||||
}
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "-");
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
for (j = 0; j < nd - 3; j += 4) {
|
||||
MuseAirLoongAVX(v[j], v[j + 1], v[j + 2], v[j + 3], size, seed, seed, seed, seed, digest, digest, digest, digest);
|
||||
}
|
||||
for (; j < nd; ++j) {
|
||||
MuseAirLoong(v[j], size, seed, digest);
|
||||
}
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "-");
|
||||
}
|
||||
printf("\n");
|
||||
printf("\n");
|
||||
}
|
||||
|
|
|
|||
324
cmdline/spooky2avx.c
Normal file
324
cmdline/spooky2avx.c
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Copyright (C) 2013 Andrea Mazzoleni
|
||||
|
||||
#if defined(__x86_64__)
|
||||
|
||||
#ifndef sc_numVars
|
||||
#define sc_numVars 12
|
||||
#endif
|
||||
#ifndef sc_blockSize
|
||||
#define sc_blockSize (sc_numVars * 8)
|
||||
#endif
|
||||
#ifndef sc_const
|
||||
#define sc_const 0xdeadbeefdeadbeefLL
|
||||
#endif
|
||||
|
||||
#define ASM_STEP_AVX2(idx, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, r, r_diff) \
|
||||
"vmovq " #idx "*8(%[blocks0]), %%xmm12\n\t" \
|
||||
"vmovhps " #idx "*8(%[blocks1]), %%xmm12, %%xmm12\n\t" \
|
||||
"vmovq " #idx "*8(%[blocks2]), %%xmm13\n\t" \
|
||||
"vmovhps " #idx "*8(%[blocks3]), %%xmm13, %%xmm13\n\t" \
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" \
|
||||
"vpaddq %%ymm12, " s0 ", " s0 "\n\t" \
|
||||
"vpxor " s10 ", " s2 ", " s2 "\n\t" \
|
||||
"vpxor " s0 ", " s11 ", " s11 "\n\t" \
|
||||
"vmovdqa " s0 ", %%ymm13\n\t" \
|
||||
"vpsllq $" #r ", " s0 ", " s0 "\n\t" \
|
||||
"vpsrlq $" #r_diff ", %%ymm13, %%ymm13\n\t" \
|
||||
"vpor %%ymm13, " s0 ", " s0 "\n\t" \
|
||||
"vpaddq " s1 ", " s11 ", " s11 "\n\t"
|
||||
|
||||
#define ASM_MIX_AVX2 \
|
||||
ASM_STEP_AVX2(0, "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", 11, 53) \
|
||||
ASM_STEP_AVX2(1, "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", 32, 32) \
|
||||
ASM_STEP_AVX2(2, "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", 43, 21) \
|
||||
ASM_STEP_AVX2(3, "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", 31, 33) \
|
||||
ASM_STEP_AVX2(4, "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", 17, 47) \
|
||||
ASM_STEP_AVX2(5, "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", 28, 36) \
|
||||
ASM_STEP_AVX2(6, "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", 39, 25) \
|
||||
ASM_STEP_AVX2(7, "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", 57, 7) \
|
||||
ASM_STEP_AVX2(8, "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", 55, 9) \
|
||||
ASM_STEP_AVX2(9, "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", 54, 10) \
|
||||
ASM_STEP_AVX2(10, "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", 22, 42) \
|
||||
ASM_STEP_AVX2(11, "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", 46, 18)
|
||||
|
||||
#define ASM_END_STEP_AVX2(s1, s11, s2, r, r_diff) \
|
||||
"vpaddq " s1 ", " s11 ", " s11 "\n\t" \
|
||||
"vpxor " s11 ", " s2 ", " s2 "\n\t" \
|
||||
"vmovdqa " s1 ", %%ymm13\n\t" \
|
||||
"vpsllq $" #r ", " s1 ", " s1 "\n\t" \
|
||||
"vpsrlq $" #r_diff ", %%ymm13, %%ymm13\n\t" \
|
||||
"vpor %%ymm13, " s1 ", " s1 "\n\t"
|
||||
|
||||
#define ASM_END_PARTIAL_AVX2 \
|
||||
ASM_END_STEP_AVX2("%%ymm1", "%%ymm11", "%%ymm2", 44, 20) \
|
||||
ASM_END_STEP_AVX2("%%ymm2", "%%ymm0", "%%ymm3", 15, 49) \
|
||||
ASM_END_STEP_AVX2("%%ymm3", "%%ymm1", "%%ymm4", 34, 30) \
|
||||
ASM_END_STEP_AVX2("%%ymm4", "%%ymm2", "%%ymm5", 21, 43) \
|
||||
ASM_END_STEP_AVX2("%%ymm5", "%%ymm3", "%%ymm6", 38, 26) \
|
||||
ASM_END_STEP_AVX2("%%ymm6", "%%ymm4", "%%ymm7", 33, 31) \
|
||||
ASM_END_STEP_AVX2("%%ymm7", "%%ymm5", "%%ymm8", 10, 54) \
|
||||
ASM_END_STEP_AVX2("%%ymm8", "%%ymm6", "%%ymm9", 13, 51) \
|
||||
ASM_END_STEP_AVX2("%%ymm9", "%%ymm7", "%%ymm10", 38, 26) \
|
||||
ASM_END_STEP_AVX2("%%ymm10", "%%ymm8", "%%ymm11", 53, 11) \
|
||||
ASM_END_STEP_AVX2("%%ymm11", "%%ymm9", "%%ymm0", 42, 22) \
|
||||
ASM_END_STEP_AVX2("%%ymm0", "%%ymm10", "%%ymm1", 54, 10)
|
||||
|
||||
void SpookyHash128AVX(
|
||||
const void* data0, const void* data1, const void* data2, const void* data3,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3,
|
||||
uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3
|
||||
)
|
||||
{
|
||||
uint64_t buf0[sc_numVars];
|
||||
uint64_t buf1[sc_numVars];
|
||||
uint64_t buf2[sc_numVars];
|
||||
uint64_t buf3[sc_numVars];
|
||||
size_t nblocks;
|
||||
const uint8_t* blocks0;
|
||||
const uint8_t* blocks1;
|
||||
const uint8_t* blocks2;
|
||||
const uint8_t* blocks3;
|
||||
const uint8_t* end0;
|
||||
size_t size_remainder;
|
||||
static const uint64_t sc_const_val = sc_const;
|
||||
|
||||
nblocks = size / sc_blockSize;
|
||||
blocks0 = (const uint8_t*)data0;
|
||||
blocks1 = (const uint8_t*)data1;
|
||||
blocks2 = (const uint8_t*)data2;
|
||||
blocks3 = (const uint8_t*)data3;
|
||||
end0 = blocks0 + nblocks * sc_blockSize;
|
||||
|
||||
/* tail data is computed beforehand */
|
||||
size_remainder = size - nblocks * sc_blockSize;
|
||||
memcpy(buf0, blocks0 + nblocks * sc_blockSize, size_remainder);
|
||||
memset(((uint8_t*)buf0) + size_remainder, 0, sc_blockSize - size_remainder);
|
||||
((uint8_t*)buf0)[sc_blockSize - 1] = size_remainder;
|
||||
|
||||
memcpy(buf1, blocks1 + nblocks * sc_blockSize, size_remainder);
|
||||
memset(((uint8_t*)buf1) + size_remainder, 0, sc_blockSize - size_remainder);
|
||||
((uint8_t*)buf1)[sc_blockSize - 1] = size_remainder;
|
||||
|
||||
memcpy(buf2, blocks2 + nblocks * sc_blockSize, size_remainder);
|
||||
memset(((uint8_t*)buf2) + size_remainder, 0, sc_blockSize - size_remainder);
|
||||
((uint8_t*)buf2)[sc_blockSize - 1] = size_remainder;
|
||||
|
||||
memcpy(buf3, blocks3 + nblocks * sc_blockSize, size_remainder);
|
||||
memset(((uint8_t*)buf3) + size_remainder, 0, sc_blockSize - size_remainder);
|
||||
((uint8_t*)buf3)[sc_blockSize - 1] = size_remainder;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
/* Load seed0 into XMM registers */
|
||||
"movq %[seed0], %%rax\n\t"
|
||||
"vmovq (%%rax), %%xmm9\n\t"
|
||||
"movq %[seed1], %%rax\n\t"
|
||||
"vmovhps (%%rax), %%xmm9, %%xmm9\n\t"
|
||||
"movq %[seed2], %%rax\n\t"
|
||||
"vmovq (%%rax), %%xmm13\n\t"
|
||||
"movq %[seed3], %%rax\n\t"
|
||||
"vmovhps (%%rax), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm9, %%ymm9\n\t"
|
||||
|
||||
"movq %[seed0], %%rax\n\t"
|
||||
"vmovq 8(%%rax), %%xmm10\n\t"
|
||||
"movq %[seed1], %%rax\n\t"
|
||||
"vmovhps 8(%%rax), %%xmm10, %%xmm10\n\t"
|
||||
"movq %[seed2], %%rax\n\t"
|
||||
"vmovq 8(%%rax), %%xmm13\n\t"
|
||||
"movq %[seed3], %%rax\n\t"
|
||||
"vmovhps 8(%%rax), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm10, %%ymm10\n\t"
|
||||
|
||||
/* Load sc_const into XMM15 and duplicate it across YMM15 */
|
||||
"vmovq %[sc_const_val], %%xmm15\n\t"
|
||||
"vpunpcklqdq %%xmm15, %%xmm15, %%xmm15\n\t"
|
||||
"vinserti128 $1, %%xmm15, %%ymm15, %%ymm15\n\t"
|
||||
|
||||
/* Initialize h0..h11 */
|
||||
"vmovdqa %%ymm9, %%ymm0\n\t"
|
||||
"vmovdqa %%ymm9, %%ymm3\n\t"
|
||||
"vmovdqa %%ymm9, %%ymm6\n\t"
|
||||
"vmovdqa %%ymm10, %%ymm1\n\t"
|
||||
"vmovdqa %%ymm10, %%ymm4\n\t"
|
||||
"vmovdqa %%ymm10, %%ymm7\n\t"
|
||||
"vmovdqa %%ymm15, %%ymm2\n\t"
|
||||
"vmovdqa %%ymm15, %%ymm5\n\t"
|
||||
"vmovdqa %%ymm15, %%ymm8\n\t"
|
||||
"vmovdqa %%ymm15, %%ymm11\n\t"
|
||||
|
||||
/* Main loop */
|
||||
"1:\n\t"
|
||||
"cmp %[end0], %[blocks0]\n\t"
|
||||
"jae 2f\n\t"
|
||||
|
||||
ASM_MIX_AVX2
|
||||
|
||||
"add $96, %[blocks0]\n\t"
|
||||
"add $96, %[blocks1]\n\t"
|
||||
"add $96, %[blocks2]\n\t"
|
||||
"add $96, %[blocks3]\n\t"
|
||||
"jmp 1b\n\t"
|
||||
|
||||
"2:\n\t"
|
||||
|
||||
/* Copy buf0..buf3 pointers into blocks0..blocks3 to reuse registers */
|
||||
"leaq %[buf0], %[blocks0]\n\t"
|
||||
"leaq %[buf1], %[blocks1]\n\t"
|
||||
"leaq %[buf2], %[blocks2]\n\t"
|
||||
"leaq %[buf3], %[blocks3]\n\t"
|
||||
|
||||
/* Load and add buf0..buf3 elements */
|
||||
"vmovq 0*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 0*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 0*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 0*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm0, %%ymm0\n\t"
|
||||
|
||||
"vmovq 1*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 1*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 1*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 1*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm1, %%ymm1\n\t"
|
||||
|
||||
"vmovq 2*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 2*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 2*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 2*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm2, %%ymm2\n\t"
|
||||
|
||||
"vmovq 3*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 3*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 3*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 3*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm3, %%ymm3\n\t"
|
||||
|
||||
"vmovq 4*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 4*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 4*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 4*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm4, %%ymm4\n\t"
|
||||
|
||||
"vmovq 5*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 5*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 5*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 5*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm5, %%ymm5\n\t"
|
||||
|
||||
"vmovq 6*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 6*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 6*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 6*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm6, %%ymm6\n\t"
|
||||
|
||||
"vmovq 7*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 7*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 7*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 7*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm7, %%ymm7\n\t"
|
||||
|
||||
"vmovq 8*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 8*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 8*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 8*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm8, %%ymm8\n\t"
|
||||
|
||||
"vmovq 9*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 9*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 9*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 9*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm9, %%ymm9\n\t"
|
||||
|
||||
"vmovq 10*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 10*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 10*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 10*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm10, %%ymm10\n\t"
|
||||
|
||||
"vmovq 11*8(%[blocks0]), %%xmm12\n\t"
|
||||
"vmovhps 11*8(%[blocks1]), %%xmm12, %%xmm12\n\t"
|
||||
"vmovq 11*8(%[blocks2]), %%xmm13\n\t"
|
||||
"vmovhps 11*8(%[blocks3]), %%xmm13, %%xmm13\n\t"
|
||||
"vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t"
|
||||
"vpaddq %%ymm12, %%ymm11, %%ymm11\n\t"
|
||||
|
||||
/* EndPartial 3 times */
|
||||
ASM_END_PARTIAL_AVX2
|
||||
ASM_END_PARTIAL_AVX2
|
||||
ASM_END_PARTIAL_AVX2
|
||||
|
||||
/* Extract upper halves of h0 and h1 */
|
||||
"vextracti128 $1, %%ymm0, %%xmm12\n\t"
|
||||
"vextracti128 $1, %%ymm1, %%xmm13\n\t"
|
||||
|
||||
/* Store digest0 and digest1 using XMM registers */
|
||||
"movq %[digest0], %%rax\n\t"
|
||||
"vmovq %%xmm0, (%%rax)\n\t"
|
||||
"vmovq %%xmm1, 8(%%rax)\n\t"
|
||||
|
||||
"movq %[digest1], %%rcx\n\t"
|
||||
"vmovhps %%xmm0, (%%rcx)\n\t"
|
||||
"vmovhps %%xmm1, 8(%%rcx)\n\t"
|
||||
|
||||
/* Store digest2 and digest3 using extracted XMM registers */
|
||||
"movq %[digest2], %%rdx\n\t"
|
||||
"vmovq %%xmm12, (%%rdx)\n\t"
|
||||
"vmovq %%xmm13, 8(%%rdx)\n\t"
|
||||
|
||||
"movq %[digest3], %%r8\n\t"
|
||||
"vmovhps %%xmm12, (%%r8)\n\t"
|
||||
"vmovhps %%xmm13, 8(%%r8)\n\t"
|
||||
|
||||
: [blocks0] "+r" (blocks0),
|
||||
[blocks1] "+r" (blocks1),
|
||||
[blocks2] "+r" (blocks2),
|
||||
[blocks3] "+r" (blocks3)
|
||||
: [end0] "r" (end0),
|
||||
[seed0] "m" (seed0),
|
||||
[seed1] "m" (seed1),
|
||||
[seed2] "m" (seed2),
|
||||
[seed3] "m" (seed3),
|
||||
[digest0] "m" (digest0),
|
||||
[digest1] "m" (digest1),
|
||||
[digest2] "m" (digest2),
|
||||
[digest3] "m" (digest3),
|
||||
[sc_const_val] "m" (sc_const_val),
|
||||
[buf0] "m" (buf0),
|
||||
[buf1] "m" (buf1),
|
||||
[buf2] "m" (buf2),
|
||||
[buf3] "m" (buf3)
|
||||
: "rax", "rcx", "rdx", "r8", "r9", "r10", "r11",
|
||||
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
|
||||
"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory"
|
||||
);
|
||||
}
|
||||
|
||||
#else /* !__x86_64__ */
|
||||
|
||||
void SpookyHash128AVX(
|
||||
const void* data0, const void* data1, const void* data2, const void* data3,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3,
|
||||
uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3
|
||||
)
|
||||
{
|
||||
SpookyHash128(data0, size, seed0, digest0);
|
||||
SpookyHash128(data1, size, seed1, digest1);
|
||||
SpookyHash128(data2, size, seed2, digest2);
|
||||
SpookyHash128(data3, size, seed3, digest3);
|
||||
}
|
||||
|
||||
#endif /* __x86_64__ */
|
||||
|
||||
323
cmdline/spooky2sse.c
Normal file
323
cmdline/spooky2sse.c
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// Copyright (C) 2013 Andrea Mazzoleni
|
||||
|
||||
#if defined(__x86_64__)
|
||||
|
||||
#ifndef sc_numVars
|
||||
#define sc_numVars 12
|
||||
#endif
|
||||
#ifndef sc_blockSize
|
||||
#define sc_blockSize (sc_numVars * 8)
|
||||
#endif
|
||||
#ifndef sc_const
|
||||
#define sc_const 0xdeadbeefdeadbeefLL
|
||||
#endif
|
||||
|
||||
#define ASM_END_STEP(s1, s11, s2, r, r_diff) \
|
||||
"paddq " s1 ", " s11 "\n\t" \
|
||||
"pxor " s11 ", " s2 "\n\t" \
|
||||
"movdqa " s1 ", %%xmm13\n\t" \
|
||||
"psllq $" #r ", " s1 "\n\t" \
|
||||
"psrlq $" #r_diff ", %%xmm13\n\t" \
|
||||
"por %%xmm13, " s1 "\n\t"
|
||||
|
||||
#define ASM_END_PARTIAL \
|
||||
ASM_END_STEP("%%xmm1", "%%xmm11", "%%xmm2", 44, 20) \
|
||||
ASM_END_STEP("%%xmm2", "%%xmm0", "%%xmm3", 15, 49) \
|
||||
ASM_END_STEP("%%xmm3", "%%xmm1", "%%xmm4", 34, 30) \
|
||||
ASM_END_STEP("%%xmm4", "%%xmm2", "%%xmm5", 21, 43) \
|
||||
ASM_END_STEP("%%xmm5", "%%xmm3", "%%xmm6", 38, 26) \
|
||||
ASM_END_STEP("%%xmm6", "%%xmm4", "%%xmm7", 33, 31) \
|
||||
ASM_END_STEP("%%xmm7", "%%xmm5", "%%xmm8", 10, 54) \
|
||||
ASM_END_STEP("%%xmm8", "%%xmm6", "%%xmm9", 13, 51) \
|
||||
ASM_END_STEP("%%xmm9", "%%xmm7", "%%xmm10", 38, 26) \
|
||||
ASM_END_STEP("%%xmm10", "%%xmm8", "%%xmm11", 53, 11) \
|
||||
ASM_END_STEP("%%xmm11", "%%xmm9", "%%xmm0", 42, 22) \
|
||||
ASM_END_STEP("%%xmm0", "%%xmm10", "%%xmm1", 54, 10)
|
||||
|
||||
void SpookyHash128SSE(
|
||||
const void* data0, const void* data1,
|
||||
size_t size,
|
||||
const uint8_t* seed0, const uint8_t* seed1,
|
||||
uint8_t* digest0, uint8_t* digest1
|
||||
)
|
||||
{
|
||||
uint64_t buf0[sc_numVars];
|
||||
uint64_t buf1[sc_numVars];
|
||||
size_t nblocks;
|
||||
const uint8_t* blocks0;
|
||||
const uint8_t* blocks1;
|
||||
const uint8_t* end0;
|
||||
size_t size_remainder;
|
||||
static const uint64_t sc_const_val = sc_const;
|
||||
|
||||
nblocks = size / sc_blockSize;
|
||||
blocks0 = (const uint8_t*)data0;
|
||||
blocks1 = (const uint8_t*)data1;
|
||||
end0 = blocks0 + nblocks * sc_blockSize;
|
||||
|
||||
/* tail data is computed beforehand */
|
||||
size_remainder = size - nblocks * sc_blockSize;
|
||||
memcpy(buf0, blocks0 + nblocks * sc_blockSize, size_remainder);
|
||||
memset(((uint8_t*)buf0) + size_remainder, 0, sc_blockSize - size_remainder);
|
||||
((uint8_t*)buf0)[sc_blockSize - 1] = size_remainder;
|
||||
|
||||
memcpy(buf1, blocks1 + nblocks * sc_blockSize, size_remainder);
|
||||
memset(((uint8_t*)buf1) + size_remainder, 0, sc_blockSize - size_remainder);
|
||||
((uint8_t*)buf1)[sc_blockSize - 1] = size_remainder;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
/* Load seed0 into XMM registers */
|
||||
"movq (%[seed0]), %%xmm9\n\t"
|
||||
"movhps (%[seed1]), %%xmm9\n\t"
|
||||
"movq 8(%[seed0]), %%xmm10\n\t"
|
||||
"movhps 8(%[seed1]), %%xmm10\n\t"
|
||||
|
||||
/* Load sc_const into XMM15 and duplicate it */
|
||||
"movq (%[sc_const_ptr]), %%xmm15\n\t"
|
||||
"punpcklqdq %%xmm15, %%xmm15\n\t"
|
||||
|
||||
/* Initialize h0..h11 */
|
||||
"movdqa %%xmm9, %%xmm0\n\t"
|
||||
"movdqa %%xmm9, %%xmm3\n\t"
|
||||
"movdqa %%xmm9, %%xmm6\n\t"
|
||||
"movdqa %%xmm10, %%xmm1\n\t"
|
||||
"movdqa %%xmm10, %%xmm4\n\t"
|
||||
"movdqa %%xmm10, %%xmm7\n\t"
|
||||
"movdqa %%xmm15, %%xmm2\n\t"
|
||||
"movdqa %%xmm15, %%xmm5\n\t"
|
||||
"movdqa %%xmm15, %%xmm8\n\t"
|
||||
"movdqa %%xmm15, %%xmm11\n\t"
|
||||
|
||||
/* Main loop */
|
||||
"1:\n\t"
|
||||
"cmp %[end0], %[blocks0]\n\t"
|
||||
"jae 2f\n\t"
|
||||
|
||||
/* Step 0 */
|
||||
"movq 0*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 0*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm0\n\t"
|
||||
"pxor %%xmm10, %%xmm2\n\t"
|
||||
"pxor %%xmm0, %%xmm11\n\t"
|
||||
"movdqa %%xmm0, %%xmm13\n\t"
|
||||
"psllq $11, %%xmm0\n\t"
|
||||
"psrlq $53, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm0\n\t"
|
||||
"paddq %%xmm1, %%xmm11\n\t"
|
||||
|
||||
/* Step 1 */
|
||||
"movq 1*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 1*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm1\n\t"
|
||||
"pxor %%xmm11, %%xmm3\n\t"
|
||||
"pxor %%xmm1, %%xmm0\n\t"
|
||||
"movdqa %%xmm1, %%xmm13\n\t"
|
||||
"psllq $32, %%xmm1\n\t"
|
||||
"psrlq $32, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm1\n\t"
|
||||
"paddq %%xmm2, %%xmm0\n\t"
|
||||
|
||||
/* Step 2 */
|
||||
"movq 2*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 2*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm2\n\t"
|
||||
"pxor %%xmm0, %%xmm4\n\t"
|
||||
"pxor %%xmm2, %%xmm1\n\t"
|
||||
"movdqa %%xmm2, %%xmm13\n\t"
|
||||
"psllq $43, %%xmm2\n\t"
|
||||
"psrlq $21, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm2\n\t"
|
||||
"paddq %%xmm3, %%xmm1\n\t"
|
||||
|
||||
/* Step 3 */
|
||||
"movq 3*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 3*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm3\n\t"
|
||||
"pxor %%xmm1, %%xmm5\n\t"
|
||||
"pxor %%xmm3, %%xmm2\n\t"
|
||||
"movdqa %%xmm3, %%xmm13\n\t"
|
||||
"psllq $31, %%xmm3\n\t"
|
||||
"psrlq $33, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm3\n\t"
|
||||
"paddq %%xmm4, %%xmm2\n\t"
|
||||
|
||||
/* Step 4 */
|
||||
"movq 4*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 4*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm4\n\t"
|
||||
"pxor %%xmm2, %%xmm6\n\t"
|
||||
"pxor %%xmm4, %%xmm3\n\t"
|
||||
"movdqa %%xmm4, %%xmm13\n\t"
|
||||
"psllq $17, %%xmm4\n\t"
|
||||
"psrlq $47, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm4\n\t"
|
||||
"paddq %%xmm5, %%xmm3\n\t"
|
||||
|
||||
/* Step 5 */
|
||||
"movq 5*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 5*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm5\n\t"
|
||||
"pxor %%xmm3, %%xmm7\n\t"
|
||||
"pxor %%xmm5, %%xmm4\n\t"
|
||||
"movdqa %%xmm5, %%xmm13\n\t"
|
||||
"psllq $28, %%xmm5\n\t"
|
||||
"psrlq $36, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm5\n\t"
|
||||
"paddq %%xmm6, %%xmm4\n\t"
|
||||
|
||||
/* Step 6 */
|
||||
"movq 6*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 6*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm6\n\t"
|
||||
"pxor %%xmm4, %%xmm8\n\t"
|
||||
"pxor %%xmm6, %%xmm5\n\t"
|
||||
"movdqa %%xmm6, %%xmm13\n\t"
|
||||
"psllq $39, %%xmm6\n\t"
|
||||
"psrlq $25, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm6\n\t"
|
||||
"paddq %%xmm7, %%xmm5\n\t"
|
||||
|
||||
/* Step 7 */
|
||||
"movq 7*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 7*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm7\n\t"
|
||||
"pxor %%xmm5, %%xmm9\n\t"
|
||||
"pxor %%xmm7, %%xmm6\n\t"
|
||||
"movdqa %%xmm7, %%xmm13\n\t"
|
||||
"psllq $57, %%xmm7\n\t"
|
||||
"psrlq $7, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm7\n\t"
|
||||
"paddq %%xmm8, %%xmm6\n\t"
|
||||
|
||||
/* Step 8 */
|
||||
"movq 8*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 8*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm8\n\t"
|
||||
"pxor %%xmm6, %%xmm10\n\t"
|
||||
"pxor %%xmm8, %%xmm7\n\t"
|
||||
"movdqa %%xmm8, %%xmm13\n\t"
|
||||
"psllq $55, %%xmm8\n\t"
|
||||
"psrlq $9, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm8\n\t"
|
||||
"paddq %%xmm9, %%xmm7\n\t"
|
||||
|
||||
/* Step 9 */
|
||||
"movq 9*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 9*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm9\n\t"
|
||||
"pxor %%xmm7, %%xmm11\n\t"
|
||||
"pxor %%xmm9, %%xmm8\n\t"
|
||||
"movdqa %%xmm9, %%xmm13\n\t"
|
||||
"psllq $54, %%xmm9\n\t"
|
||||
"psrlq $10, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm9\n\t"
|
||||
"paddq %%xmm10, %%xmm8\n\t"
|
||||
|
||||
/* Step 10 */
|
||||
"movq 10*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 10*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm10\n\t"
|
||||
"pxor %%xmm8, %%xmm0\n\t"
|
||||
"pxor %%xmm10, %%xmm9\n\t"
|
||||
"movdqa %%xmm10, %%xmm13\n\t"
|
||||
"psllq $22, %%xmm10\n\t"
|
||||
"psrlq $42, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm10\n\t"
|
||||
"paddq %%xmm11, %%xmm9\n\t"
|
||||
|
||||
/* Step 11 */
|
||||
"movq 11*8(%[blocks0]), %%xmm12\n\t"
|
||||
"movhps 11*8(%[blocks1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm11\n\t"
|
||||
"pxor %%xmm9, %%xmm1\n\t"
|
||||
"pxor %%xmm11, %%xmm10\n\t"
|
||||
"movdqa %%xmm11, %%xmm13\n\t"
|
||||
"psllq $46, %%xmm11\n\t"
|
||||
"psrlq $18, %%xmm13\n\t"
|
||||
"por %%xmm13, %%xmm11\n\t"
|
||||
"paddq %%xmm0, %%xmm10\n\t"
|
||||
|
||||
"add $96, %[blocks0]\n\t"
|
||||
"add $96, %[blocks1]\n\t"
|
||||
"jmp 1b\n\t"
|
||||
|
||||
"2:\n\t"
|
||||
|
||||
/* Load and add buf0 and buf1 elements */
|
||||
"movq 0*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 0*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm0\n\t"
|
||||
|
||||
"movq 1*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 1*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm1\n\t"
|
||||
|
||||
"movq 2*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 2*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm2\n\t"
|
||||
|
||||
"movq 3*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 3*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm3\n\t"
|
||||
|
||||
"movq 4*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 4*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm4\n\t"
|
||||
|
||||
"movq 5*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 5*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm5\n\t"
|
||||
|
||||
"movq 6*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 6*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm6\n\t"
|
||||
|
||||
"movq 7*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 7*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm7\n\t"
|
||||
|
||||
"movq 8*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 8*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm8\n\t"
|
||||
|
||||
"movq 9*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 9*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm9\n\t"
|
||||
|
||||
"movq 10*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 10*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm10\n\t"
|
||||
|
||||
"movq 11*8(%[buf0]), %%xmm12\n\t"
|
||||
"movhps 11*8(%[buf1]), %%xmm12\n\t"
|
||||
"paddq %%xmm12, %%xmm11\n\t"
|
||||
|
||||
/* EndPartial 3 times */
|
||||
ASM_END_PARTIAL
|
||||
ASM_END_PARTIAL
|
||||
ASM_END_PARTIAL
|
||||
|
||||
/* Store back h0 and h1 to output digests */
|
||||
"movq %%xmm0, (%[digest0])\n\t"
|
||||
"movhps %%xmm0, (%[digest1])\n\t"
|
||||
"movq %%xmm1, 8(%[digest0])\n\t"
|
||||
"movhps %%xmm1, 8(%[digest1])\n\t"
|
||||
|
||||
: [blocks0] "+r" (blocks0),
|
||||
[blocks1] "+r" (blocks1)
|
||||
: [end0] "r" (end0),
|
||||
[seed0] "r" (seed0),
|
||||
[seed1] "r" (seed1),
|
||||
[digest0] "r" (digest0),
|
||||
[digest1] "r" (digest1),
|
||||
[sc_const_ptr] "r" (&sc_const_val),
|
||||
[buf0] "r" (buf0),
|
||||
[buf1] "r" (buf1)
|
||||
: "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
|
||||
"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory"
|
||||
);
|
||||
}
|
||||
|
||||
#endif /* __x86_64__ */
|
||||
|
||||
|
|
@ -760,8 +760,12 @@ unsigned membesthash(void)
|
|||
|
||||
#include "murmur3.c"
|
||||
#include "spooky2.c"
|
||||
#include "spooky2sse.c"
|
||||
#include "spooky2avx.c"
|
||||
#include "metro.c"
|
||||
#include "museair.c"
|
||||
#include "museairsse.c"
|
||||
#include "museairavx.c"
|
||||
|
||||
void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -230,6 +230,12 @@ const char* memhashname(unsigned kind);
|
|||
* Compute the HASH of a memory block.
|
||||
* Seed is a 128 bit vector.
|
||||
*/
|
||||
void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest);
|
||||
void SpookyHash128SSE(const void* data0, const void* data1, size_t size, const uint8_t* seed0, const uint8_t* seed1, uint8_t* digest0, uint8_t* digest1);
|
||||
void SpookyHash128AVX(const void* data0, const void* data1, const void* data2, const void* data3, size_t size, const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3);
|
||||
void MuseAirLoongSSE(const void* data0, const void* data1, size_t size, const uint8_t* seed0, const uint8_t* seed1, uint8_t* digest0, uint8_t* digest1);
|
||||
void MuseAirLoongAVX(const void* data0, const void* data1, const void* data2, const void* data3, size_t size, const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3);
|
||||
void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out);
|
||||
void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size);
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue