From e63e79d51abeebd8a919f4208d5267adfe434bc8 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 13 Jul 2026 00:44:18 +0200 Subject: [PATCH] Parallel hashes --- Makefile.am | 4 + cmdline/checker.c | 1 + cmdline/museair.c | 2 +- cmdline/museairavx.c | 476 ++++++++++++++++++++++++++++++++++++++++++ cmdline/museairsse.c | 486 +++++++++++++++++++++++++++++++++++++++++++ cmdline/selftest.c | 260 +++++++++++++++++++++++ cmdline/speed.c | 68 ++++++ cmdline/spooky2avx.c | 324 +++++++++++++++++++++++++++++ cmdline/spooky2sse.c | 323 ++++++++++++++++++++++++++++ cmdline/util.c | 4 + cmdline/util.h | 6 + 11 files changed, 1953 insertions(+), 1 deletion(-) create mode 100644 cmdline/museairavx.c create mode 100644 cmdline/museairsse.c create mode 100644 cmdline/spooky2avx.c create mode 100644 cmdline/spooky2sse.c diff --git a/Makefile.am b/Makefile.am index 838dd729..8250916b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -109,10 +109,14 @@ noinst_HEADERS = \ cmdline/murmur3test.c \ cmdline/spooky2.c \ cmdline/spooky2test.c \ + cmdline/spooky2sse.c \ + cmdline/spooky2avx.c \ cmdline/metro.c \ cmdline/metrotest.c \ cmdline/museair.c \ cmdline/museairtest.c \ + cmdline/museairsse.c \ + cmdline/museairavx.c \ cmdline/import.h \ cmdline/search.h \ cmdline/thermal.h \ diff --git a/cmdline/checker.c b/cmdline/checker.c index ae00e755..56ce10e4 100644 --- a/cmdline/checker.c +++ b/cmdline/checker.c @@ -5,3 +5,4 @@ #define HAVE_CHECKER 1 #include "snapraid.c" + diff --git a/cmdline/museair.c b/cmdline/museair.c index 1c67fdee..38dca864 100644 --- a/cmdline/museair.c +++ b/cmdline/museair.c @@ -56,7 +56,7 @@ static const uint64_t MASK_I = 01555555555555555555555ULL; static const uint64_t MASK_J = 01333333333333333333333ULL; static const uint64_t MASK_K = 00666666666666666666666ULL; -static void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) +void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out) { const uint8_t* p = bytes; size_t q = len; diff --git a/cmdline/museairavx.c b/cmdline/museairavx.c new file mode 100644 index 00000000..6cd5714a --- /dev/null +++ b/cmdline/museairavx.c @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#if defined(__x86_64__) + +#define AVX_LOAD(offset, dest) \ + "vmovq " #offset "*8(%[blocks0]), %%xmm14\n\t" \ + "vmovhps " #offset "*8(%[blocks1]), %%xmm14, %%xmm14\n\t" \ + "vmovq " #offset "*8(%[blocks2]), %%xmm15\n\t" \ + "vmovhps " #offset "*8(%[blocks3]), %%xmm15, %%xmm15\n\t" \ + "vinserti128 $1, %%xmm15, %%ymm14, " dest "\n\t" + +#define AVX_LOAD_NEG(offset, dest) \ + "vmovq -" #offset "(%%r8), %%xmm14\n\t" \ + "vmovhps -" #offset "(%%r9), %%xmm14, %%xmm14\n\t" \ + "vmovq -" #offset "(%%r10), %%xmm15\n\t" \ + "vmovhps -" #offset "(%%r11), %%xmm15, %%xmm15\n\t" \ + "vinserti128 $1, %%xmm15, %%ymm14, " dest "\n\t" + +#define AVX_MULT(ymm_a, ymm_b, ymm_lo, ymm_hi) \ + /* 1. ymm_ll = ymm_a * ymm_b (low parts) */ \ + "vpmuludq " ymm_b ", " ymm_a ", %%ymm8\n\t" /* ymm8 = P_ll */ \ + /* 2. ymm_lh = ymm_a * ymm_b_high */ \ + "vpsrlq $32, " ymm_b ", %%ymm9\n\t" /* ymm9 = B_h */ \ + "vpmuludq %%ymm9, " ymm_a ", %%ymm10\n\t" /* ymm10 = P_lh */ \ + /* 3. ymm_hl = ymm_a_high * ymm_b */ \ + "vpsrlq $32, " ymm_a ", %%ymm14\n\t" /* ymm14 = A_h */ \ + "vpmuludq " ymm_b ", %%ymm14, %%ymm15\n\t" /* ymm15 = P_hl */ \ + /* 4. ymm_hh = ymm_a_high * ymm_b_high */ \ + "vpmuludq %%ymm9, %%ymm14, %%ymm9\n\t" /* ymm9 = P_hh */ \ + /* 5. M = P_lh + P_hl */ \ + "vpaddq %%ymm15, %%ymm10, %%ymm15\n\t" /* ymm15 = M */ \ + /* Generate mask 0xffffffff in ymm14 */ \ + "vpcmpeqd %%ymm14, %%ymm14, %%ymm14\n\t" \ + "vpsrlq $32, %%ymm14, %%ymm14\n\t" \ + /* 6. Split M into ymm10 = M_low, ymm15 = M_high */ \ + "vpand %%ymm14, %%ymm15, %%ymm10\n\t" /* ymm10 = M_low */ \ + "vpsrlq $32, %%ymm15, %%ymm15\n\t" /* ymm15 = M_high */ \ + /* 7. Split P_ll (in ymm8) into ymm12 = P_lll, ymm13 = P_llh */ \ + "vpsrlq $32, %%ymm8, %%ymm13\n\t" /* ymm13 = P_llh */ \ + "vpand %%ymm14, %%ymm8, %%ymm12\n\t" /* ymm12 = P_lll */ \ + /* 8. S1 = P_llh + M_low */ \ + "vpaddq %%ymm10, %%ymm13, %%ymm10\n\t" /* ymm10 = S1 */ \ + /* 9. Split S1 into ymm13 = C1, ymm10 = S1_l */ \ + "vpsrlq $32, %%ymm10, %%ymm13\n\t" /* ymm13 = C1 */ \ + "vpand %%ymm14, %%ymm10, %%ymm10\n\t" /* ymm10 = S1_l */ \ + /* 10. Assemble final low = P_lll + S1_l * 2^32 */ \ + "vpsllq $32, %%ymm10, %%ymm10\n\t" \ + "vpaddq %%ymm12, %%ymm10, " ymm_lo "\n\t" /* ymm_lo = final low */ \ + /* 11. Split P_hh (in ymm9) into ymm12 = P_hhl, ymm10 = P_hhh */ \ + "vpsrlq $32, %%ymm9, %%ymm10\n\t" /* ymm10 = P_hhh */ \ + "vpand %%ymm14, %%ymm9, %%ymm12\n\t" /* ymm12 = P_hhl */ \ + /* 12. Assemble final high = M_high + P_hhl + C1 + P_hhh * 2^32 */ \ + "vpaddq %%ymm12, %%ymm15, %%ymm15\n\t" /* ymm15 = M_high + P_hhl */ \ + "vpaddq %%ymm13, %%ymm15, %%ymm15\n\t" /* ymm15 = M_high + P_hhl + C1 */ \ + "vpsllq $32, %%ymm10, %%ymm10\n\t" \ + "vpaddq %%ymm10, %%ymm15, " ymm_hi "\n\t" /* ymm_hi = final high */ + +void MuseAirLoongAVX( + const void* data0, const void* data1, const void* data2, const void* data3, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, + uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3 +) +{ + uint64_t buf0[12]; + uint64_t buf1[12]; + uint64_t buf2[12]; + uint64_t buf3[12]; + const uint8_t* blocks0 = data0; + const uint8_t* blocks1 = data1; + const uint8_t* blocks2 = data2; + const uint8_t* blocks3 = data3; + size_t q = size; + + if (q < 32) { + memset(buf0, 0, 32); + memset(buf1, 0, 32); + memset(buf2, 0, 32); + memset(buf3, 0, 32); + if (q > 0) { + memcpy(buf0, blocks0, q); + memcpy(buf1, blocks1, q); + memcpy(buf2, blocks2, q); + memcpy(buf3, blocks3, q); + } + blocks0 = (const uint8_t*)buf0; + blocks1 = (const uint8_t*)buf1; + blocks2 = (const uint8_t*)buf2; + blocks3 = (const uint8_t*)buf3; + q = 32; + } + + uint64_t rot = size & 63; + uint64_t rot_diff = 64 - rot; + static const uint64_t const_mask_i = 01555555555555555555555ULL; + static const uint64_t const_mask_j = 01333333333333333333333ULL; + static const uint64_t const_mask_k = 00666666666666666666666ULL; + + __asm__ __volatile__ ( + /* Load seeds */ + "movq %[seed0], %%rax\n\t" + "vmovq (%%rax), %%xmm9\n\t" + "movq %[seed1], %%rax\n\t" + "vmovhps (%%rax), %%xmm9, %%xmm9\n\t" + "movq %[seed2], %%rax\n\t" + "vmovq (%%rax), %%xmm13\n\t" + "movq %[seed3], %%rax\n\t" + "vmovhps (%%rax), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm9, %%ymm9\n\t" /* ymm9 = seed_a */ + + "movq %[seed0], %%rax\n\t" + "vmovq 8(%%rax), %%xmm10\n\t" + "movq %[seed1], %%rax\n\t" + "vmovhps 8(%%rax), %%xmm10, %%xmm10\n\t" + "movq %[seed2], %%rax\n\t" + "vmovq 8(%%rax), %%xmm13\n\t" + "movq %[seed3], %%rax\n\t" + "vmovhps 8(%%rax), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm10, %%ymm10\n\t" /* ymm10 = seed_b */ + + /* Load masks */ + "vmovq %[mask_i], %%xmm12\n\t" + "vpunpcklqdq %%xmm12, %%xmm12, %%xmm12\n\t" + "vinserti128 $1, %%xmm12, %%ymm12, %%ymm12\n\t" + + "vmovq %[mask_j], %%xmm13\n\t" + "vpunpcklqdq %%xmm13, %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm13, %%ymm13\n\t" + + "vmovq %[mask_k], %%xmm14\n\t" + "vpunpcklqdq %%xmm14, %%xmm14, %%xmm14\n\t" + "vinserti128 $1, %%xmm14, %%ymm14, %%ymm14\n\t" + + /* Initialize states */ + "leaq %[const_ptr], %%rax\n\t" + + /* state0 */ + "vmovq 0*8(%%rax), %%xmm0\n\t" + "vpunpcklqdq %%xmm0, %%xmm0, %%xmm0\n\t" + "vinserti128 $1, %%xmm0, %%ymm0, %%ymm0\n\t" + "vpand %%ymm12, %%ymm9, %%ymm15\n\t" + "vpxor %%ymm15, %%ymm0, %%ymm0\n\t" + + /* state1 */ + "vmovq 1*8(%%rax), %%xmm1\n\t" + "vpunpcklqdq %%xmm1, %%xmm1, %%xmm1\n\t" + "vinserti128 $1, %%xmm1, %%ymm1, %%ymm1\n\t" + "vpand %%ymm13, %%ymm10, %%ymm15\n\t" + "vpxor %%ymm15, %%ymm1, %%ymm1\n\t" + + /* state2 */ + "vmovq 2*8(%%rax), %%xmm2\n\t" + "vpunpcklqdq %%xmm2, %%xmm2, %%xmm2\n\t" + "vinserti128 $1, %%xmm2, %%ymm2, %%ymm2\n\t" + "vpand %%ymm14, %%ymm9, %%ymm15\n\t" + "vpxor %%ymm15, %%ymm2, %%ymm2\n\t" + + /* state3 */ + "vmovq 3*8(%%rax), %%xmm3\n\t" + "vpunpcklqdq %%xmm3, %%xmm3, %%xmm3\n\t" + "vinserti128 $1, %%xmm3, %%ymm3, %%ymm3\n\t" + "vpand %%ymm12, %%ymm10, %%ymm15\n\t" + "vpxor %%ymm15, %%ymm3, %%ymm3\n\t" + + /* state4 */ + "vmovq 4*8(%%rax), %%xmm4\n\t" + "vpunpcklqdq %%xmm4, %%xmm4, %%xmm4\n\t" + "vinserti128 $1, %%xmm4, %%ymm4, %%ymm4\n\t" + "vpand %%ymm13, %%ymm9, %%ymm15\n\t" + "vpxor %%ymm15, %%ymm4, %%ymm4\n\t" + + /* state5 */ + "vmovq 5*8(%%rax), %%xmm5\n\t" + "vpunpcklqdq %%xmm5, %%xmm5, %%xmm5\n\t" + "vinserti128 $1, %%xmm5, %%ymm5, %%ymm5\n\t" + "vpand %%ymm14, %%ymm10, %%ymm15\n\t" + "vpxor %%ymm15, %%ymm5, %%ymm5\n\t" + + /* lo5 and hi5 */ + "vmovq 6*8(%%rax), %%xmm6\n\t" + "vpunpcklqdq %%xmm6, %%xmm6, %%xmm6\n\t" + "vinserti128 $1, %%xmm6, %%ymm6, %%ymm6\n\t" + "vmovdqa %%ymm6, %%ymm7\n\t" + + /* Conditional loop */ + "cmp $96, %[q]\n\t" + "jbe 3f\n\t" + + "1:\n\t" + /* Loop step idx = 0 */ + AVX_LOAD(0, "%%ymm15") + "vpxor %%ymm15, %%ymm0, %%ymm0\n\t" + AVX_LOAD(1, "%%ymm15") + "vpxor %%ymm15, %%ymm1, %%ymm1\n\t" + AVX_MULT("%%ymm0", "%%ymm1", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm6, %%ymm0\n\t" + "vmovdqa %%ymm8, %%ymm10\n\t" + + /* Loop step idx = 1 */ + AVX_LOAD(2, "%%ymm15") + "vpxor %%ymm15, %%ymm1, %%ymm1\n\t" + AVX_LOAD(3, "%%ymm15") + "vpxor %%ymm15, %%ymm2, %%ymm2\n\t" + AVX_MULT("%%ymm1", "%%ymm2", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm10, %%ymm1\n\t" + "vmovdqa %%ymm8, %%ymm10\n\t" + + /* Loop step idx = 2 */ + AVX_LOAD(4, "%%ymm15") + "vpxor %%ymm15, %%ymm2, %%ymm2\n\t" + AVX_LOAD(5, "%%ymm15") + "vpxor %%ymm15, %%ymm3, %%ymm3\n\t" + AVX_MULT("%%ymm2", "%%ymm3", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm10, %%ymm2\n\t" + "vmovdqa %%ymm8, %%ymm10\n\t" + + /* Loop step idx = 3 */ + AVX_LOAD(6, "%%ymm15") + "vpxor %%ymm15, %%ymm3, %%ymm3\n\t" + AVX_LOAD(7, "%%ymm15") + "vpxor %%ymm15, %%ymm4, %%ymm4\n\t" + AVX_MULT("%%ymm3", "%%ymm4", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm10, %%ymm3\n\t" + "vmovdqa %%ymm8, %%ymm10\n\t" + + /* Loop step idx = 4 */ + AVX_LOAD(8, "%%ymm15") + "vpxor %%ymm15, %%ymm4, %%ymm4\n\t" + AVX_LOAD(9, "%%ymm15") + "vpxor %%ymm15, %%ymm5, %%ymm5\n\t" + AVX_MULT("%%ymm4", "%%ymm5", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm10, %%ymm4\n\t" + "vmovdqa %%ymm8, %%ymm10\n\t" + + /* Loop step idx = 5 */ + AVX_LOAD(10, "%%ymm15") + "vpxor %%ymm15, %%ymm5, %%ymm5\n\t" + AVX_LOAD(11, "%%ymm15") + "vpxor %%ymm15, %%ymm0, %%ymm0\n\t" + AVX_MULT("%%ymm5", "%%ymm0", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm10, %%ymm5\n\t" + "vmovdqa %%ymm8, %%ymm6\n\t" + "vmovdqa %%ymm9, %%ymm7\n\t" + + "add $96, %[blocks0]\n\t" + "add $96, %[blocks1]\n\t" + "add $96, %[blocks2]\n\t" + "add $96, %[blocks3]\n\t" + "sub $96, %[q]\n\t" + "cmp $96, %[q]\n\t" + "ja 1b\n\t" + + "vpxor %%ymm6, %%ymm0, %%ymm0\n\t" + "3:\n\t" + + /* Tail */ + "vmovdqa %%ymm1, %%ymm10\n\t" /* xor0 = state1 */ + "vmovdqa %%ymm2, %%ymm11\n\t" /* xor1 = state2 */ + "vmovdqa %%ymm3, %%ymm12\n\t" /* xor2 = state3 */ + "vmovdqa %%ymm4, %%ymm13\n\t" /* xor3 = state4 */ + + "cmp $32, %[q]\n\t" + "jbe 4f\n\t" + + AVX_LOAD(0, "%%ymm15") + "vpxor %%ymm15, %%ymm0, %%ymm0\n\t" + AVX_LOAD(1, "%%ymm15") + "vpxor %%ymm15, %%ymm1, %%ymm1\n\t" + AVX_MULT("%%ymm0", "%%ymm1", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm8, %%ymm10\n\t" + + "cmp $48, %[q]\n\t" + "jbe 4f\n\t" + + AVX_LOAD(2, "%%ymm15") + "vpxor %%ymm15, %%ymm1, %%ymm1\n\t" + AVX_LOAD(3, "%%ymm15") + "vpxor %%ymm15, %%ymm2, %%ymm2\n\t" + AVX_MULT("%%ymm1", "%%ymm2", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm8, %%ymm11\n\t" + + "cmp $64, %[q]\n\t" + "jbe 4f\n\t" + + AVX_LOAD(4, "%%ymm15") + "vpxor %%ymm15, %%ymm2, %%ymm2\n\t" + AVX_LOAD(5, "%%ymm15") + "vpxor %%ymm15, %%ymm3, %%ymm3\n\t" + AVX_MULT("%%ymm2", "%%ymm3", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm8, %%ymm12\n\t" + + "cmp $80, %[q]\n\t" + "jbe 4f\n\t" + + AVX_LOAD(6, "%%ymm15") + "vpxor %%ymm15, %%ymm3, %%ymm3\n\t" + AVX_LOAD(7, "%%ymm15") + "vpxor %%ymm15, %%ymm4, %%ymm4\n\t" + AVX_MULT("%%ymm3", "%%ymm4", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm8, %%ymm13\n\t" + + "4:\n\t" + + /* Compute end pointers */ + "movq %[blocks0], %%r8\n\t" + "add %[q], %%r8\n\t" + "movq %[blocks1], %%r9\n\t" + "add %[q], %%r9\n\t" + "movq %[blocks2], %%r10\n\t" + "add %[q], %%r10\n\t" + "movq %[blocks3], %%r11\n\t" + "add %[q], %%r11\n\t" + + /* Unconditional tail step 1 */ + AVX_LOAD_NEG(32, "%%ymm15") + "vpxor %%ymm15, %%ymm4, %%ymm4\n\t" + AVX_LOAD_NEG(24, "%%ymm15") + "vpxor %%ymm15, %%ymm5, %%ymm5\n\t" + AVX_MULT("%%ymm4", "%%ymm5", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm8, %%ymm14\n\t" + + /* Unconditional tail step 2 */ + AVX_LOAD_NEG(16, "%%ymm15") + "vpxor %%ymm15, %%ymm5, %%ymm5\n\t" + AVX_LOAD_NEG(8, "%%ymm15") + "vpxor %%ymm15, %%ymm0, %%ymm0\n\t" + AVX_MULT("%%ymm5", "%%ymm0", "%%ymm8", "%%ymm9") + "vpxor %%ymm9, %%ymm8, %%ymm15\n\t" + + /* Subtract states to compute i, j, k */ + "vpsubq %%ymm1, %%ymm0, %%ymm0\n\t" + "leaq %[const_ptr], %%rax\n\t" + "vmovq 7*8(%%rax), %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + "vpxor %%ymm8, %%ymm0, %%ymm0\n\t" /* ymm0 = i */ + + "vpsubq %%ymm3, %%ymm2, %%ymm2\n\t" + "vmovq 8*8(%%rax), %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + "vpxor %%ymm8, %%ymm2, %%ymm2\n\t" /* ymm2 = j */ + + "vpsubq %%ymm5, %%ymm4, %%ymm4\n\t" + "vmovq 9*8(%%rax), %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + "vpxor %%ymm8, %%ymm4, %%ymm4\n\t" /* ymm4 = k */ + + /* Rotations and size subtraction */ + "vmovq %[rot], %%xmm8\n\t" + "vmovq %[rot_diff], %%xmm9\n\t" + + "vpsllq %%xmm8, %%ymm0, %%ymm1\n\t" + "vpsrlq %%xmm9, %%ymm0, %%ymm0\n\t" + "vpor %%ymm0, %%ymm1, %%ymm0\n\t" + + "vpsrlq %%xmm8, %%ymm2, %%ymm3\n\t" + "vpsllq %%xmm9, %%ymm2, %%ymm2\n\t" + "vpor %%ymm2, %%ymm3, %%ymm2\n\t" + + "vmovq %[size], %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + "vpsubq %%ymm8, %%ymm4, %%ymm4\n\t" + + /* i, j, k corrections */ + "vpsubq %%ymm13, %%ymm0, %%ymm0\n\t" + "vpsubq %%ymm14, %%ymm0, %%ymm0\n\t" + + "vpsubq %%ymm15, %%ymm2, %%ymm2\n\t" + "vpsubq %%ymm10, %%ymm2, %%ymm2\n\t" + + "vpsubq %%ymm11, %%ymm4, %%ymm4\n\t" + "vpsubq %%ymm12, %%ymm4, %%ymm4\n\t" + + /* Final multiplications */ + AVX_MULT("%%ymm0", "%%ymm2", "%%ymm10", "%%ymm11") + AVX_MULT("%%ymm2", "%%ymm4", "%%ymm12", "%%ymm13") + AVX_MULT("%%ymm4", "%%ymm0", "%%ymm8", "%%ymm9") + + "vpxor %%ymm11, %%ymm8, %%ymm0\n\t" + "vpxor %%ymm13, %%ymm10, %%ymm2\n\t" + "vpxor %%ymm9, %%ymm12, %%ymm4\n\t" + + /* Multiply by constants 10, 11, 12 */ + "vmovq 10*8(%%rax), %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + AVX_MULT("%%ymm0", "%%ymm8", "%%ymm10", "%%ymm11") + + "vmovq 11*8(%%rax), %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + AVX_MULT("%%ymm2", "%%ymm8", "%%ymm12", "%%ymm13") + + "vmovq 12*8(%%rax), %%xmm8\n\t" + "vpunpcklqdq %%xmm8, %%xmm8, %%xmm8\n\t" + "vinserti128 $1, %%xmm8, %%ymm8, %%ymm8\n\t" + AVX_MULT("%%ymm4", "%%ymm8", "%%ymm6", "%%ymm7") + + /* Final digests XOR */ + "vpxor %%ymm13, %%ymm10, %%ymm10\n\t" + "vpxor %%ymm6, %%ymm10, %%ymm10\n\t" + + "vpxor %%ymm12, %%ymm11, %%ymm11\n\t" + "vpxor %%ymm7, %%ymm11, %%ymm11\n\t" + + /* Extract high parts of digests */ + "vextracti128 $1, %%ymm10, %%xmm12\n\t" + "vextracti128 $1, %%ymm11, %%xmm13\n\t" + + /* Store digest0 */ + "movq %[digest0], %%rax\n\t" + "vmovq %%xmm10, (%%rax)\n\t" + "vmovq %%xmm11, 8(%%rax)\n\t" + + /* Store digest1 */ + "movq %[digest1], %%rcx\n\t" + "vmovhps %%xmm10, (%%rcx)\n\t" + "vmovhps %%xmm11, 8(%%rcx)\n\t" + + /* Store digest2 */ + "movq %[digest2], %%rdx\n\t" + "vmovq %%xmm12, (%%rdx)\n\t" + "vmovq %%xmm13, 8(%%rdx)\n\t" + + /* Store digest3 */ + "movq %[digest3], %%rsi\n\t" + "vmovhps %%xmm12, (%%rsi)\n\t" + "vmovhps %%xmm13, 8(%%rsi)\n\t" + + : [blocks0] "+r" (blocks0), + [blocks1] "+r" (blocks1), + [blocks2] "+r" (blocks2), + [blocks3] "+r" (blocks3), + [q] "+r" (q) + : [seed0] "m" (seed0), + [seed1] "m" (seed1), + [seed2] "m" (seed2), + [seed3] "m" (seed3), + [digest0] "m" (digest0), + [digest1] "m" (digest1), + [digest2] "m" (digest2), + [digest3] "m" (digest3), + [const_ptr] "m" (MUSEAIR_CONSTANT), + [mask_i] "m" (const_mask_i), + [mask_j] "m" (const_mask_j), + [mask_k] "m" (const_mask_k), + [rot] "m" (rot), + [rot_diff] "m" (rot_diff), + [size] "m" (size) + : "rax", "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory" + ); +} + +#else /* !__x86_64__ */ + +void MuseAirLoongAVX( + const void* data0, const void* data1, const void* data2, const void* data3, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, + uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3 +) +{ + MuseAirLoong(data0, size, seed0, digest0); + MuseAirLoong(data1, size, seed1, digest1); + MuseAirLoong(data2, size, seed2, digest2); + MuseAirLoong(data3, size, seed3, digest3); +} + +#endif /* __x86_64__ */ + diff --git a/cmdline/museairsse.c b/cmdline/museairsse.c new file mode 100644 index 00000000..f623872f --- /dev/null +++ b/cmdline/museairsse.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#if defined(__x86_64__) + +#define SSE_LOAD(offset, dest) \ + "movq " #offset "*8(%[blocks0]), " dest "\n\t" \ + "movhps " #offset "*8(%[blocks1]), " dest "\n\t" + +#define SSE_LOAD_NEG(offset, dest) \ + "movq -" #offset "(%%r8), " dest "\n\t" \ + "movhps -" #offset "(%%r9), " dest "\n\t" + +#define SSE_MULT(a, b, lo, hi, s0, s1) \ + /* 1. Compute B_h in s0 */ \ + "movdqa " b ", " s0 "\n\t" \ + "psrlq $32, " s0 "\n\t" \ + /* 2. Compute P_lh in s1 */ \ + "movdqa " a ", " s1 "\n\t" \ + "pmuludq " s0 ", " s1 "\n\t" \ + /* 3. Compute A_h in hi */ \ + "movdqa " a ", " hi "\n\t" \ + "psrlq $32, " hi "\n\t" \ + /* 4. Compute P_hh in lo */ \ + "movdqa " hi ", " lo "\n\t" \ + "pmuludq " s0 ", " lo "\n\t" \ + /* 5. Compute P_hl in hi */ \ + "pmuludq " b ", " hi "\n\t" \ + /* 6. Compute P_ll in s0 */ \ + "movdqa " a ", " s0 "\n\t" \ + "pmuludq " b ", " s0 "\n\t" \ + /* 7. M = P_lh + P_hl */ \ + "paddq " hi ", " s1 "\n\t" /* s1 = M */ \ + /* 8. Move P_hh from lo to hi */ \ + "movdqa " lo ", " hi "\n\t" /* hi = P_hh */ \ + /* 9. Generate mask 0xffffffff in xmm15 */ \ + "pcmpeqd %%xmm15, %%xmm15\n\t" \ + "psrlq $32, %%xmm15\n\t" \ + /* 10. Split M into s1 = M_low, lo = M_high */ \ + "movdqa " s1 ", " lo "\n\t" \ + "pand %%xmm15, " s1 "\n\t" /* s1 = M_low */ \ + "psrlq $32, " lo "\n\t" /* lo = M_high */ \ + /* 11. Split P_ll (in s0) into s0 = P_lll, xmm15 = P_llh (overwriting mask) */ \ + "movdqa " s0 ", %%xmm15\n\t" \ + "psrlq $32, %%xmm15\n\t" /* xmm15 = P_llh */ \ + "pand %%xmm15, " s0 "\n\t" /* s0 = P_lll */ /* Wait! xmm15 contains P_llh which has 0 in high 32 bits, but low 32 bits is P_llh, not 0xffffffff! */ \ + /* Ah! We cannot pand with xmm15 because xmm15 is NOT the mask anymore! */ \ + /* To fix this, let's just use the shift method for s0! */ \ + /* "psllq $32, s0; psrlq $32, s0" is safer! */ \ + +#define SSE_MULT_FIXED(a, b, lo, hi, s0, s1) \ + /* 1. Compute B_h in s0 */ \ + "movdqa " b ", " s0 "\n\t" \ + "psrlq $32, " s0 "\n\t" \ + /* 2. Compute P_lh in s1 */ \ + "movdqa " a ", " s1 "\n\t" \ + "pmuludq " s0 ", " s1 "\n\t" \ + /* 3. Compute A_h in hi */ \ + "movdqa " a ", " hi "\n\t" \ + "psrlq $32, " hi "\n\t" \ + /* 4. Compute P_hh in lo */ \ + "movdqa " hi ", " lo "\n\t" \ + "pmuludq " s0 ", " lo "\n\t" \ + /* 5. Compute P_hl in hi */ \ + "pmuludq " b ", " hi "\n\t" \ + /* 6. Compute P_ll in s0 */ \ + "movdqa " a ", " s0 "\n\t" \ + "pmuludq " b ", " s0 "\n\t" \ + /* 7. M = P_lh + P_hl */ \ + "paddq " hi ", " s1 "\n\t" /* s1 = M */ \ + /* 8. Move P_hh from lo to hi */ \ + "movdqa " lo ", " hi "\n\t" /* hi = P_hh */ \ + /* 9. Generate mask 0xffffffff in xmm15 */ \ + "pcmpeqd %%xmm15, %%xmm15\n\t" \ + "psrlq $32, %%xmm15\n\t" \ + /* 10. Split M into s1 = M_low, lo = M_high */ \ + "movdqa " s1 ", " lo "\n\t" \ + "pand %%xmm15, " s1 "\n\t" /* s1 = M_low */ \ + "psrlq $32, " lo "\n\t" /* lo = M_high */ \ + /* 11. Split P_ll (in s0) into s0 = P_lll, xmm15 = P_llh (overwriting mask) */ \ + "movdqa " s0 ", %%xmm15\n\t" \ + "psrlq $32, %%xmm15\n\t" /* xmm15 = P_llh */ \ + "psllq $32, " s0 "\n\t" \ + "psrlq $32, " s0 "\n\t" /* s0 = P_lll */ \ + /* 12. Compute S1 = P_llh + M_low */ \ + "paddq " s1 ", %%xmm15\n\t" /* xmm15 = S1 */ \ + /* 13. Split S1 into s1 = C1, xmm15 = S1_l */ \ + "movdqa %%xmm15, " s1 "\n\t" \ + "psllq $32, %%xmm15\n\t" \ + "psrlq $32, %%xmm15\n\t" /* xmm15 = S1_l */ \ + "psrlq $32, " s1 "\n\t" /* s1 = C1 */ \ + /* 14. Assemble final low (in lo) */ \ + "psllq $32, %%xmm15\n\t" \ + "paddq " s0 ", %%xmm15\n\t" \ + "movdqa %%xmm15, " lo "\n\t" /* lo = final low */ \ + /* 15. Split P_hh (in hi) into s0 = P_hhl, hi = P_hhh */ \ + "movdqa " hi ", " s0 "\n\t" \ + "psllq $32, " s0 "\n\t" \ + "psrlq $32, " s0 "\n\t" /* s0 = P_hhl */ \ + "psrlq $32, " hi "\n\t" /* hi = P_hhh */ \ + /* 16. Compute final high (in hi) */ \ + "paddq " lo ", " s0 "\n\t" /* s0 = M_high + P_hhl */ \ + "paddq " s1 ", " s0 "\n\t" /* s0 = M_high + P_hhl + C1 */ \ + "psllq $32, " hi "\n\t" \ + "paddq " s0 ", " hi "\n\t" /* hi = final high */ + +void MuseAirLoongSSE( + const void* data0, const void* data1, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, + uint8_t* digest0, uint8_t* digest1 +) +{ + uint64_t buf0[12]; + uint64_t buf1[12]; + const uint8_t* blocks0 = data0; + const uint8_t* blocks1 = data1; + size_t q = size; + + if (q < 32) { + memset(buf0, 0, 32); + memset(buf1, 0, 32); + if (q > 0) { + memcpy(buf0, blocks0, q); + memcpy(buf1, blocks1, q); + } + blocks0 = (const uint8_t*)buf0; + blocks1 = (const uint8_t*)buf1; + q = 32; + } + + uint64_t rot = size & 63; + uint64_t rot_diff = 64 - rot; + static const uint64_t const_mask_i = 01555555555555555555555ULL; + static const uint64_t const_mask_j = 01333333333333333333333ULL; + static const uint64_t const_mask_k = 00666666666666666666666ULL; + + __asm__ __volatile__ ( + /* Load seed0 and seed1 */ + "movq %[seed0], %%rax\n\t" + "movq (%%rax), %%xmm9\n\t" + "movq %[seed1], %%rcx\n\t" + "movhps (%%rcx), %%xmm9\n\t" /* xmm9 = seed_a */ + + "movq %[seed0], %%rax\n\t" + "movq 8(%%rax), %%xmm10\n\t" + "movq %[seed1], %%rcx\n\t" + "movhps 8(%%rcx), %%xmm10\n\t" /* xmm10 = seed_b */ + + /* Load and duplicate masks */ + "movq %[mask_i], %%xmm12\n\t" + "punpcklqdq %%xmm12, %%xmm12\n\t" + + "movq %[mask_j], %%xmm13\n\t" + "punpcklqdq %%xmm13, %%xmm13\n\t" + + "movq %[mask_k], %%xmm14\n\t" + "punpcklqdq %%xmm14, %%xmm14\n\t" + + /* Initialize states */ + "leaq %[const_ptr], %%rax\n\t" + + /* state0 */ + "movq 0*8(%%rax), %%xmm0\n\t" + "punpcklqdq %%xmm0, %%xmm0\n\t" + "movdqa %%xmm9, %%xmm15\n\t" + "pand %%xmm12, %%xmm15\n\t" + "pxor %%xmm15, %%xmm0\n\t" + + /* state1 */ + "movq 1*8(%%rax), %%xmm1\n\t" + "punpcklqdq %%xmm1, %%xmm1\n\t" + "movdqa %%xmm10, %%xmm15\n\t" + "pand %%xmm13, %%xmm15\n\t" + "pxor %%xmm15, %%xmm1\n\t" + + /* state2 */ + "movq 2*8(%%rax), %%xmm2\n\t" + "punpcklqdq %%xmm2, %%xmm2\n\t" + "movdqa %%xmm9, %%xmm15\n\t" + "pand %%xmm14, %%xmm15\n\t" + "pxor %%xmm15, %%xmm2\n\t" + + /* state3 */ + "movq 3*8(%%rax), %%xmm3\n\t" + "punpcklqdq %%xmm3, %%xmm3\n\t" + "movdqa %%xmm10, %%xmm15\n\t" + "pand %%xmm12, %%xmm15\n\t" + "pxor %%xmm15, %%xmm3\n\t" + + /* state4 */ + "movq 4*8(%%rax), %%xmm4\n\t" + "punpcklqdq %%xmm4, %%xmm4\n\t" + "movdqa %%xmm9, %%xmm15\n\t" + "pand %%xmm13, %%xmm15\n\t" + "pxor %%xmm15, %%xmm4\n\t" + + /* state5 */ + "movq 5*8(%%rax), %%xmm5\n\t" + "punpcklqdq %%xmm5, %%xmm5\n\t" + "movdqa %%xmm10, %%xmm15\n\t" + "pand %%xmm14, %%xmm15\n\t" + "pxor %%xmm15, %%xmm5\n\t" + + /* lo5 and hi5 */ + "movq 6*8(%%rax), %%xmm6\n\t" + "punpcklqdq %%xmm6, %%xmm6\n\t" + "movdqa %%xmm6, %%xmm7\n\t" + + /* Conditional loop */ + "cmp $96, %[q]\n\t" + "jbe 3f\n\t" + + "1:\n\t" + /* Loop step idx = 0 */ + SSE_LOAD(0, "%%xmm15") + "pxor %%xmm15, %%xmm0\n\t" + SSE_LOAD(1, "%%xmm15") + "pxor %%xmm15, %%xmm1\n\t" + SSE_MULT_FIXED("%%xmm0", "%%xmm1", "%%xmm8", "%%xmm9", "%%xmm10", "%%xmm11") + "movdqa %%xmm6, %%xmm0\n\t" + "pxor %%xmm9, %%xmm0\n\t" + "movdqa %%xmm8, %%xmm10\n\t" + + /* Loop step idx = 1 */ + SSE_LOAD(2, "%%xmm15") + "pxor %%xmm15, %%xmm1\n\t" + SSE_LOAD(3, "%%xmm15") + "pxor %%xmm15, %%xmm2\n\t" + SSE_MULT_FIXED("%%xmm1", "%%xmm2", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15") + "movdqa %%xmm10, %%xmm1\n\t" + "pxor %%xmm9, %%xmm1\n\t" + "movdqa %%xmm8, %%xmm10\n\t" + + /* Loop step idx = 2 */ + SSE_LOAD(4, "%%xmm15") + "pxor %%xmm15, %%xmm2\n\t" + SSE_LOAD(5, "%%xmm15") + "pxor %%xmm15, %%xmm3\n\t" + SSE_MULT_FIXED("%%xmm2", "%%xmm3", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15") + "movdqa %%xmm10, %%xmm2\n\t" + "pxor %%xmm9, %%xmm2\n\t" + "movdqa %%xmm8, %%xmm10\n\t" + + /* Loop step idx = 3 */ + SSE_LOAD(6, "%%xmm15") + "pxor %%xmm15, %%xmm3\n\t" + SSE_LOAD(7, "%%xmm15") + "pxor %%xmm15, %%xmm4\n\t" + SSE_MULT_FIXED("%%xmm3", "%%xmm4", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15") + "movdqa %%xmm10, %%xmm3\n\t" + "pxor %%xmm9, %%xmm3\n\t" + "movdqa %%xmm8, %%xmm10\n\t" + + /* Loop step idx = 4 */ + SSE_LOAD(8, "%%xmm15") + "pxor %%xmm15, %%xmm4\n\t" + SSE_LOAD(9, "%%xmm15") + "pxor %%xmm15, %%xmm5\n\t" + SSE_MULT_FIXED("%%xmm4", "%%xmm5", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15") + "movdqa %%xmm10, %%xmm4\n\t" + "pxor %%xmm9, %%xmm4\n\t" + "movdqa %%xmm8, %%xmm10\n\t" + + /* Loop step idx = 5 */ + SSE_LOAD(10, "%%xmm15") + "pxor %%xmm15, %%xmm5\n\t" + SSE_LOAD(11, "%%xmm15") + "pxor %%xmm15, %%xmm0\n\t" + SSE_MULT_FIXED("%%xmm5", "%%xmm0", "%%xmm8", "%%xmm9", "%%xmm11", "%%xmm15") + "movdqa %%xmm10, %%xmm5\n\t" + "pxor %%xmm9, %%xmm5\n\t" + "movdqa %%xmm8, %%xmm6\n\t" + "movdqa %%xmm9, %%xmm7\n\t" + + "add $96, %[blocks0]\n\t" + "add $96, %[blocks1]\n\t" + "sub $96, %[q]\n\t" + "cmp $96, %[q]\n\t" + "ja 1b\n\t" + + "pxor %%xmm6, %%xmm0\n\t" + "3:\n\t" + + /* Tail */ + "movdqa %%xmm1, %%xmm10\n\t" /* xor0 = state1 */ + "movdqa %%xmm2, %%xmm11\n\t" /* xor1 = state2 */ + "movdqa %%xmm3, %%xmm12\n\t" /* xor2 = state3 */ + "movdqa %%xmm4, %%xmm13\n\t" /* xor3 = state4 */ + + "cmp $32, %[q]\n\t" + "jbe 4f\n\t" + + SSE_LOAD(0, "%%xmm15") + "pxor %%xmm15, %%xmm0\n\t" + SSE_LOAD(1, "%%xmm15") + "pxor %%xmm15, %%xmm1\n\t" + SSE_MULT_FIXED("%%xmm0", "%%xmm1", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15") + "movdqa %%xmm8, %%xmm10\n\t" + "pxor %%xmm9, %%xmm10\n\t" + + "cmp $48, %[q]\n\t" + "jbe 4f\n\t" + + SSE_LOAD(2, "%%xmm15") + "pxor %%xmm15, %%xmm1\n\t" + SSE_LOAD(3, "%%xmm15") + "pxor %%xmm15, %%xmm2\n\t" + SSE_MULT_FIXED("%%xmm1", "%%xmm2", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15") + "movdqa %%xmm8, %%xmm11\n\t" + "pxor %%xmm9, %%xmm11\n\t" + + "cmp $64, %[q]\n\t" + "jbe 4f\n\t" + + SSE_LOAD(4, "%%xmm15") + "pxor %%xmm15, %%xmm2\n\t" + SSE_LOAD(5, "%%xmm15") + "pxor %%xmm15, %%xmm3\n\t" + SSE_MULT_FIXED("%%xmm2", "%%xmm3", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15") + "movdqa %%xmm8, %%xmm12\n\t" + "pxor %%xmm9, %%xmm12\n\t" + + "cmp $80, %[q]\n\t" + "jbe 4f\n\t" + + SSE_LOAD(6, "%%xmm15") + "pxor %%xmm15, %%xmm3\n\t" + SSE_LOAD(7, "%%xmm15") + "pxor %%xmm15, %%xmm4\n\t" + SSE_MULT_FIXED("%%xmm3", "%%xmm4", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15") + "movdqa %%xmm8, %%xmm13\n\t" + "pxor %%xmm9, %%xmm13\n\t" + + "4:\n\t" + + /* Compute end pointers */ + "movq %[blocks0], %%r8\n\t" + "add %[q], %%r8\n\t" + "movq %[blocks1], %%r9\n\t" + "add %[q], %%r9\n\t" + + /* Unconditional tail step 1 */ + SSE_LOAD_NEG(32, "%%xmm15") + "pxor %%xmm15, %%xmm4\n\t" + SSE_LOAD_NEG(24, "%%xmm15") + "pxor %%xmm15, %%xmm5\n\t" + SSE_MULT_FIXED("%%xmm4", "%%xmm5", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15") + "movdqa %%xmm8, %%xmm14\n\t" + "pxor %%xmm9, %%xmm14\n\t" + + /* Unconditional tail step 2 */ + SSE_LOAD_NEG(16, "%%xmm15") + "pxor %%xmm15, %%xmm5\n\t" + SSE_LOAD_NEG(8, "%%xmm15") + "pxor %%xmm15, %%xmm0\n\t" + SSE_MULT_FIXED("%%xmm5", "%%xmm0", "%%xmm8", "%%xmm9", "%%xmm12", "%%xmm15") + "movdqa %%xmm8, %%xmm15\n\t" + "pxor %%xmm9, %%xmm15\n\t" + + /* Subtract states to compute i, j, k */ + "psubq %%xmm1, %%xmm0\n\t" + "leaq %[const_ptr], %%rax\n\t" + "movq 7*8(%%rax), %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + "pxor %%xmm8, %%xmm0\n\t" /* xmm0 = i */ + + "psubq %%xmm3, %%xmm2\n\t" + "movq 8*8(%%rax), %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + "pxor %%xmm8, %%xmm2\n\t" /* xmm2 = j */ + + "psubq %%xmm5, %%xmm4\n\t" + "movq 9*8(%%rax), %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + "pxor %%xmm8, %%xmm4\n\t" /* xmm4 = k */ + + /* Rotations and size subtraction */ + "movq %[rot], %%xmm8\n\t" + "movq %[rot_diff], %%xmm9\n\t" + + "movdqa %%xmm0, %%xmm1\n\t" + "psllq %%xmm8, %%xmm0\n\t" + "psrlq %%xmm9, %%xmm1\n\t" + "por %%xmm1, %%xmm0\n\t" + + "movdqa %%xmm2, %%xmm3\n\t" + "psrlq %%xmm8, %%xmm2\n\t" + "psllq %%xmm9, %%xmm3\n\t" + "por %%xmm3, %%xmm2\n\t" + + "movq %[size], %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + "psubq %%xmm8, %%xmm4\n\t" + + /* i, j, k corrections */ + "psubq %%xmm13, %%xmm0\n\t" + "psubq %%xmm14, %%xmm0\n\t" + + "psubq %%xmm15, %%xmm2\n\t" + "psubq %%xmm10, %%xmm2\n\t" + + "psubq %%xmm11, %%xmm4\n\t" + "psubq %%xmm12, %%xmm4\n\t" + + /* Final multiplications */ + SSE_MULT_FIXED("%%xmm0", "%%xmm2", "%%xmm10", "%%xmm11", "%%xmm8", "%%xmm9") + SSE_MULT_FIXED("%%xmm2", "%%xmm4", "%%xmm12", "%%xmm13", "%%xmm8", "%%xmm9") + SSE_MULT_FIXED("%%xmm4", "%%xmm0", "%%xmm8", "%%xmm9", "%%xmm14", "%%xmm15") + + "movdqa %%xmm8, %%xmm0\n\t" + "pxor %%xmm11, %%xmm0\n\t" + + "movdqa %%xmm10, %%xmm2\n\t" + "pxor %%xmm13, %%xmm2\n\t" + + "movdqa %%xmm12, %%xmm4\n\t" + "pxor %%xmm9, %%xmm4\n\t" + + /* Multiply by constants 10, 11, 12 */ + "leaq %[const_ptr], %%rax\n\t" + "movq 10*8(%%rax), %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + SSE_MULT_FIXED("%%xmm0", "%%xmm8", "%%xmm10", "%%xmm11", "%%xmm12", "%%xmm13") + + "leaq %[const_ptr], %%rax\n\t" + "movq 11*8(%%rax), %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + SSE_MULT_FIXED("%%xmm2", "%%xmm8", "%%xmm12", "%%xmm13", "%%xmm10", "%%xmm11") + + "leaq %[const_ptr], %%rax\n\t" + "movq 12*8(%%rax), %%xmm8\n\t" + "punpcklqdq %%xmm8, %%xmm8\n\t" + SSE_MULT_FIXED("%%xmm4", "%%xmm8", "%%xmm6", "%%xmm7", "%%xmm10", "%%xmm11") + + /* Final digests XOR */ + "pxor %%xmm13, %%xmm10\n\t" + "pxor %%xmm6, %%xmm10\n\t" + + "pxor %%xmm12, %%xmm11\n\t" + "pxor %%xmm7, %%xmm11\n\t" + + /* Store back digests */ + "movq %[digest0], %%rax\n\t" + "movq %%xmm10, (%%rax)\n\t" + "movq %%xmm11, 8(%%rax)\n\t" + + "movq %[digest1], %%rcx\n\t" + "movhps %%xmm10, (%%rcx)\n\t" + "movhps %%xmm11, 8(%%rcx)\n\t" + + : [blocks0] "+r" (blocks0), + [blocks1] "+r" (blocks1), + [q] "+r" (q) + : [seed0] "m" (seed0), + [seed1] "m" (seed1), + [digest0] "m" (digest0), + [digest1] "m" (digest1), + [const_ptr] "m" (MUSEAIR_CONSTANT), + [mask_i] "m" (const_mask_i), + [mask_j] "m" (const_mask_j), + [mask_k] "m" (const_mask_k), + [rot] "m" (rot), + [rot_diff] "m" (rot_diff), + [size] "m" (size) + : "rax", "rcx", "rdx", "r8", "r9", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory" + ); +} + +#else /* !__x86_64__ */ + +void MuseAirLoongSSE( + const void* data0, const void* data1, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, + uint8_t* digest0, uint8_t* digest1 +) +{ + MuseAirLoong(data0, size, seed0, digest0); + MuseAirLoong(data1, size, seed1, digest1); +} + +#endif /* __x86_64__ */ + diff --git a/cmdline/selftest.c b/cmdline/selftest.c index 1481dce4..d3c7213b 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -235,6 +235,132 @@ static void test_hash(void) } } + for (i = 0; TEST_SPOOKY2[i].data; ++i) { + unsigned char digest0[HASH_MAX]; + unsigned char digest1[HASH_MAX]; + unsigned char buffer0[4096]; + unsigned char buffer1[4096]; + unsigned char seed0[HASH_MAX]; + unsigned char seed1[HASH_MAX]; + int len = TEST_SPOOKY2[i].len; + unsigned k; + + if (len > 4096) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Test vector too long\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + for (k = 0; TEST_SPOOKY2[k].data; ++k) { + if (TEST_SPOOKY2[k].len == len && k != i) { + break; + } + } + if (!TEST_SPOOKY2[k].data) { + k = i; + } + + memcpy(buffer0, TEST_SPOOKY2[i].data, len); + memcpy(seed0, TEST_SPOOKY2[i].seed, HASH_MAX); + + memcpy(buffer1, TEST_SPOOKY2[k].data, len); + memcpy(seed1, TEST_SPOOKY2[k].seed, HASH_MAX); + + SpookyHash128SSE(buffer0, buffer1, len, seed0, seed1, digest0, digest1); + + if (memcmp(digest0, TEST_SPOOKY2[i].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed SpookyHash128SSE test 0 on vector %d\n", i); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest1, TEST_SPOOKY2[k].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed SpookyHash128SSE test 1 on vector %d and %d\n", i, k); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + + /* SpookyHash128AVX self-test */ + for (i = 0; TEST_SPOOKY2[i].data; ++i) { + unsigned char buffer0[4096]; + unsigned char buffer1[4096]; + unsigned char buffer2[4096]; + unsigned char buffer3[4096]; + unsigned char seed0[HASH_MAX]; + unsigned char seed1[HASH_MAX]; + unsigned char seed2[HASH_MAX]; + unsigned char seed3[HASH_MAX]; + unsigned char digest0[HASH_MAX]; + unsigned char digest1[HASH_MAX]; + unsigned char digest2[HASH_MAX]; + unsigned char digest3[HASH_MAX]; + int len = TEST_SPOOKY2[i].len; + unsigned k, l, m; + + for (k = 0; TEST_SPOOKY2[k].data; ++k) { + if (TEST_SPOOKY2[k].len == len && k != i) { + break; + } + } + if (!TEST_SPOOKY2[k].data) k = i; + + for (l = 0; TEST_SPOOKY2[l].data; ++l) { + if (TEST_SPOOKY2[l].len == len && l != i && l != k) { + break; + } + } + if (!TEST_SPOOKY2[l].data) l = i; + + for (m = 0; TEST_SPOOKY2[m].data; ++m) { + if (TEST_SPOOKY2[m].len == len && m != i && m != k && m != l) { + break; + } + } + if (!TEST_SPOOKY2[m].data) m = i; + + memcpy(buffer0, TEST_SPOOKY2[i].data, len); + memcpy(seed0, TEST_SPOOKY2[i].seed, HASH_MAX); + + memcpy(buffer1, TEST_SPOOKY2[k].data, len); + memcpy(seed1, TEST_SPOOKY2[k].seed, HASH_MAX); + + memcpy(buffer2, TEST_SPOOKY2[l].data, len); + memcpy(seed2, TEST_SPOOKY2[l].seed, HASH_MAX); + + memcpy(buffer3, TEST_SPOOKY2[m].data, len); + memcpy(seed3, TEST_SPOOKY2[m].seed, HASH_MAX); + + SpookyHash128AVX(buffer0, buffer1, buffer2, buffer3, len, seed0, seed1, seed2, seed3, digest0, digest1, digest2, digest3); + + if (memcmp(digest0, TEST_SPOOKY2[i].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 0 on vector %d\n", i); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest1, TEST_SPOOKY2[k].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 1 on vector %d and %d\n", i, k); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest2, TEST_SPOOKY2[l].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 2 on vector %d and %d\n", i, l); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest3, TEST_SPOOKY2[m].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed SpookyHash128AVX test 3 on vector %d and %d\n", i, m); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + for (i = 0; TEST_METRO[i].data; ++i) { unsigned char digest[HASH_MAX]; memcpy(buffer_aligned, TEST_METRO[i].data, TEST_METRO[i].len); @@ -261,6 +387,140 @@ static void test_hash(void) } } + /* MuseAir SSE self-test */ + for (i = 0; TEST_MUSEAIR[i].data; ++i) { + unsigned char digest0[HASH_MAX]; + unsigned char digest1[HASH_MAX]; + unsigned char buffer0[4096]; + unsigned char buffer1[4096]; + unsigned char seed0[HASH_MAX]; + unsigned char seed1[HASH_MAX]; + int len = TEST_MUSEAIR[i].len; + unsigned k; + + if (len > 4096) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Test vector too long\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + for (k = 0; TEST_MUSEAIR[k].data; ++k) { + if (TEST_MUSEAIR[k].len == len && k != i) { + break; + } + } + if (!TEST_MUSEAIR[k].data) { + k = i; + } + + memcpy(buffer0, TEST_MUSEAIR[i].data, len); + memcpy(seed0, TEST_MUSEAIR[i].seed, HASH_MAX); + + memcpy(buffer1, TEST_MUSEAIR[k].data, len); + memcpy(seed1, TEST_MUSEAIR[k].seed, HASH_MAX); + + MuseAirLoongSSE(buffer0, buffer1, len, seed0, seed1, digest0, digest1); + + if (memcmp(digest0, TEST_MUSEAIR[i].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed MuseAirLoongSSE test 0 on vector %d\n", i); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest1, TEST_MUSEAIR[k].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed MuseAirLoongSSE test 1 on vector %d and %d\n", i, k); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + + /* MuseAir AVX self-test */ + for (i = 0; TEST_MUSEAIR[i].data; ++i) { + unsigned char buffer0[4096]; + unsigned char buffer1[4096]; + unsigned char buffer2[4096]; + unsigned char buffer3[4096]; + unsigned char seed0[HASH_MAX]; + unsigned char seed1[HASH_MAX]; + unsigned char seed2[HASH_MAX]; + unsigned char seed3[HASH_MAX]; + unsigned char digest0[HASH_MAX]; + unsigned char digest1[HASH_MAX]; + unsigned char digest2[HASH_MAX]; + unsigned char digest3[HASH_MAX]; + int len = TEST_MUSEAIR[i].len; + unsigned k, l, m; + + if (len > 4096) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Test vector too long\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + for (k = 0; TEST_MUSEAIR[k].data; ++k) { + if (TEST_MUSEAIR[k].len == len && k != i) { + break; + } + } + if (!TEST_MUSEAIR[k].data) k = i; + + for (l = 0; TEST_MUSEAIR[l].data; ++l) { + if (TEST_MUSEAIR[l].len == len && l != i && l != k) { + break; + } + } + if (!TEST_MUSEAIR[l].data) l = i; + + for (m = 0; TEST_MUSEAIR[m].data; ++m) { + if (TEST_MUSEAIR[m].len == len && m != i && m != k && m != l) { + break; + } + } + if (!TEST_MUSEAIR[m].data) m = i; + + memcpy(buffer0, TEST_MUSEAIR[i].data, len); + memcpy(seed0, TEST_MUSEAIR[i].seed, HASH_MAX); + + memcpy(buffer1, TEST_MUSEAIR[k].data, len); + memcpy(seed1, TEST_MUSEAIR[k].seed, HASH_MAX); + + memcpy(buffer2, TEST_MUSEAIR[l].data, len); + memcpy(seed2, TEST_MUSEAIR[l].seed, HASH_MAX); + + memcpy(buffer3, TEST_MUSEAIR[m].data, len); + memcpy(seed3, TEST_MUSEAIR[m].seed, HASH_MAX); + + MuseAirLoongAVX(buffer0, buffer1, buffer2, buffer3, len, seed0, seed1, seed2, seed3, digest0, digest1, digest2, digest3); + + if (memcmp(digest0, TEST_MUSEAIR[i].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 0 on vector %d\n", i); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest1, TEST_MUSEAIR[k].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 1 on vector %d and %d\n", i, k); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest2, TEST_MUSEAIR[l].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 2 on vector %d and %d\n", i, l); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (memcmp(digest3, TEST_MUSEAIR[m].digest, HASH_MAX) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed MuseAirLoongAVX test 3 on vector %d and %d\n", i, m); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + free(buffer_alloc); free(seed_alloc); } diff --git a/cmdline/speed.c b/cmdline/speed.c index de4dcfe1..cf9362b5 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -154,8 +154,12 @@ void speed_hash(int nd, void** v, int size, int delta, int period) printf("%8s", "best"); printf("%8s", "murmur3"); printf("%8s", "spooky2"); + printf("%8s", "spk2sse"); + printf("%8s", "spk2avx"); printf("%8s", "metro"); printf("%8s", "museair"); + printf("%8s", "musesse"); + printf("%8s", "museavx"); printf("\n"); printf("%8s", "hash"); @@ -178,6 +182,38 @@ void speed_hash(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); + if (raid_cpu_has_sse2()) { + SPEED_START { + for (j = 0; j < nd - 1; j += 2) { + SpookyHash128SSE(v[j], v[j + 1], size, seed, seed, digest, digest); + } + if (j < nd) { + SpookyHash128(v[j], size, seed, digest); + } + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + } else { + printf("%8s", "-"); + } + fflush(stdout); + + if (raid_cpu_has_avx2()) { + SPEED_START { + for (j = 0; j < nd - 3; j += 4) { + SpookyHash128AVX(v[j], v[j + 1], v[j + 2], v[j + 3], size, seed, seed, seed, seed, digest, digest, digest, digest); + } + for (; j < nd; ++j) { + SpookyHash128(v[j], size, seed, digest); + } + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + } else { + printf("%8s", "-"); + } + fflush(stdout); + SPEED_START { for (j = 0; j < nd; ++j) memhash(HASH_METRO, seed, digest, v[j], size); @@ -192,6 +228,38 @@ void speed_hash(int nd, void** v, int size, int delta, int period) } SPEED_STOP printf("%8" PRIu64, ds / dt); + fflush(stdout); + + if (raid_cpu_has_sse2()) { + SPEED_START { + for (j = 0; j < nd - 1; j += 2) { + MuseAirLoongSSE(v[j], v[j + 1], size, seed, seed, digest, digest); + } + if (j < nd) { + MuseAirLoong(v[j], size, seed, digest); + } + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + } else { + printf("%8s", "-"); + } + fflush(stdout); + + if (raid_cpu_has_avx2()) { + SPEED_START { + for (j = 0; j < nd - 3; j += 4) { + MuseAirLoongAVX(v[j], v[j + 1], v[j + 2], v[j + 3], size, seed, seed, seed, seed, digest, digest, digest, digest); + } + for (; j < nd; ++j) { + MuseAirLoong(v[j], size, seed, digest); + } + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + } else { + printf("%8s", "-"); + } printf("\n"); printf("\n"); } diff --git a/cmdline/spooky2avx.c b/cmdline/spooky2avx.c new file mode 100644 index 00000000..110db1e5 --- /dev/null +++ b/cmdline/spooky2avx.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2013 Andrea Mazzoleni + +#if defined(__x86_64__) + +#ifndef sc_numVars +#define sc_numVars 12 +#endif +#ifndef sc_blockSize +#define sc_blockSize (sc_numVars * 8) +#endif +#ifndef sc_const +#define sc_const 0xdeadbeefdeadbeefLL +#endif + +#define ASM_STEP_AVX2(idx, s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, r, r_diff) \ + "vmovq " #idx "*8(%[blocks0]), %%xmm12\n\t" \ + "vmovhps " #idx "*8(%[blocks1]), %%xmm12, %%xmm12\n\t" \ + "vmovq " #idx "*8(%[blocks2]), %%xmm13\n\t" \ + "vmovhps " #idx "*8(%[blocks3]), %%xmm13, %%xmm13\n\t" \ + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" \ + "vpaddq %%ymm12, " s0 ", " s0 "\n\t" \ + "vpxor " s10 ", " s2 ", " s2 "\n\t" \ + "vpxor " s0 ", " s11 ", " s11 "\n\t" \ + "vmovdqa " s0 ", %%ymm13\n\t" \ + "vpsllq $" #r ", " s0 ", " s0 "\n\t" \ + "vpsrlq $" #r_diff ", %%ymm13, %%ymm13\n\t" \ + "vpor %%ymm13, " s0 ", " s0 "\n\t" \ + "vpaddq " s1 ", " s11 ", " s11 "\n\t" + +#define ASM_MIX_AVX2 \ + ASM_STEP_AVX2(0, "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", 11, 53) \ + ASM_STEP_AVX2(1, "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", 32, 32) \ + ASM_STEP_AVX2(2, "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", 43, 21) \ + ASM_STEP_AVX2(3, "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", 31, 33) \ + ASM_STEP_AVX2(4, "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", 17, 47) \ + ASM_STEP_AVX2(5, "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", 28, 36) \ + ASM_STEP_AVX2(6, "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", 39, 25) \ + ASM_STEP_AVX2(7, "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", 57, 7) \ + ASM_STEP_AVX2(8, "%%ymm8", "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", 55, 9) \ + ASM_STEP_AVX2(9, "%%ymm9", "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", 54, 10) \ + ASM_STEP_AVX2(10, "%%ymm10", "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", 22, 42) \ + ASM_STEP_AVX2(11, "%%ymm11", "%%ymm0", "%%ymm1", "%%ymm2", "%%ymm3", "%%ymm4", "%%ymm5", "%%ymm6", "%%ymm7", "%%ymm8", "%%ymm9", "%%ymm10", 46, 18) + +#define ASM_END_STEP_AVX2(s1, s11, s2, r, r_diff) \ + "vpaddq " s1 ", " s11 ", " s11 "\n\t" \ + "vpxor " s11 ", " s2 ", " s2 "\n\t" \ + "vmovdqa " s1 ", %%ymm13\n\t" \ + "vpsllq $" #r ", " s1 ", " s1 "\n\t" \ + "vpsrlq $" #r_diff ", %%ymm13, %%ymm13\n\t" \ + "vpor %%ymm13, " s1 ", " s1 "\n\t" + +#define ASM_END_PARTIAL_AVX2 \ + ASM_END_STEP_AVX2("%%ymm1", "%%ymm11", "%%ymm2", 44, 20) \ + ASM_END_STEP_AVX2("%%ymm2", "%%ymm0", "%%ymm3", 15, 49) \ + ASM_END_STEP_AVX2("%%ymm3", "%%ymm1", "%%ymm4", 34, 30) \ + ASM_END_STEP_AVX2("%%ymm4", "%%ymm2", "%%ymm5", 21, 43) \ + ASM_END_STEP_AVX2("%%ymm5", "%%ymm3", "%%ymm6", 38, 26) \ + ASM_END_STEP_AVX2("%%ymm6", "%%ymm4", "%%ymm7", 33, 31) \ + ASM_END_STEP_AVX2("%%ymm7", "%%ymm5", "%%ymm8", 10, 54) \ + ASM_END_STEP_AVX2("%%ymm8", "%%ymm6", "%%ymm9", 13, 51) \ + ASM_END_STEP_AVX2("%%ymm9", "%%ymm7", "%%ymm10", 38, 26) \ + ASM_END_STEP_AVX2("%%ymm10", "%%ymm8", "%%ymm11", 53, 11) \ + ASM_END_STEP_AVX2("%%ymm11", "%%ymm9", "%%ymm0", 42, 22) \ + ASM_END_STEP_AVX2("%%ymm0", "%%ymm10", "%%ymm1", 54, 10) + +void SpookyHash128AVX( + const void* data0, const void* data1, const void* data2, const void* data3, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, + uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3 +) +{ + uint64_t buf0[sc_numVars]; + uint64_t buf1[sc_numVars]; + uint64_t buf2[sc_numVars]; + uint64_t buf3[sc_numVars]; + size_t nblocks; + const uint8_t* blocks0; + const uint8_t* blocks1; + const uint8_t* blocks2; + const uint8_t* blocks3; + const uint8_t* end0; + size_t size_remainder; + static const uint64_t sc_const_val = sc_const; + + nblocks = size / sc_blockSize; + blocks0 = (const uint8_t*)data0; + blocks1 = (const uint8_t*)data1; + blocks2 = (const uint8_t*)data2; + blocks3 = (const uint8_t*)data3; + end0 = blocks0 + nblocks * sc_blockSize; + + /* tail data is computed beforehand */ + size_remainder = size - nblocks * sc_blockSize; + memcpy(buf0, blocks0 + nblocks * sc_blockSize, size_remainder); + memset(((uint8_t*)buf0) + size_remainder, 0, sc_blockSize - size_remainder); + ((uint8_t*)buf0)[sc_blockSize - 1] = size_remainder; + + memcpy(buf1, blocks1 + nblocks * sc_blockSize, size_remainder); + memset(((uint8_t*)buf1) + size_remainder, 0, sc_blockSize - size_remainder); + ((uint8_t*)buf1)[sc_blockSize - 1] = size_remainder; + + memcpy(buf2, blocks2 + nblocks * sc_blockSize, size_remainder); + memset(((uint8_t*)buf2) + size_remainder, 0, sc_blockSize - size_remainder); + ((uint8_t*)buf2)[sc_blockSize - 1] = size_remainder; + + memcpy(buf3, blocks3 + nblocks * sc_blockSize, size_remainder); + memset(((uint8_t*)buf3) + size_remainder, 0, sc_blockSize - size_remainder); + ((uint8_t*)buf3)[sc_blockSize - 1] = size_remainder; + + __asm__ __volatile__ ( + /* Load seed0 into XMM registers */ + "movq %[seed0], %%rax\n\t" + "vmovq (%%rax), %%xmm9\n\t" + "movq %[seed1], %%rax\n\t" + "vmovhps (%%rax), %%xmm9, %%xmm9\n\t" + "movq %[seed2], %%rax\n\t" + "vmovq (%%rax), %%xmm13\n\t" + "movq %[seed3], %%rax\n\t" + "vmovhps (%%rax), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm9, %%ymm9\n\t" + + "movq %[seed0], %%rax\n\t" + "vmovq 8(%%rax), %%xmm10\n\t" + "movq %[seed1], %%rax\n\t" + "vmovhps 8(%%rax), %%xmm10, %%xmm10\n\t" + "movq %[seed2], %%rax\n\t" + "vmovq 8(%%rax), %%xmm13\n\t" + "movq %[seed3], %%rax\n\t" + "vmovhps 8(%%rax), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm10, %%ymm10\n\t" + + /* Load sc_const into XMM15 and duplicate it across YMM15 */ + "vmovq %[sc_const_val], %%xmm15\n\t" + "vpunpcklqdq %%xmm15, %%xmm15, %%xmm15\n\t" + "vinserti128 $1, %%xmm15, %%ymm15, %%ymm15\n\t" + + /* Initialize h0..h11 */ + "vmovdqa %%ymm9, %%ymm0\n\t" + "vmovdqa %%ymm9, %%ymm3\n\t" + "vmovdqa %%ymm9, %%ymm6\n\t" + "vmovdqa %%ymm10, %%ymm1\n\t" + "vmovdqa %%ymm10, %%ymm4\n\t" + "vmovdqa %%ymm10, %%ymm7\n\t" + "vmovdqa %%ymm15, %%ymm2\n\t" + "vmovdqa %%ymm15, %%ymm5\n\t" + "vmovdqa %%ymm15, %%ymm8\n\t" + "vmovdqa %%ymm15, %%ymm11\n\t" + + /* Main loop */ + "1:\n\t" + "cmp %[end0], %[blocks0]\n\t" + "jae 2f\n\t" + + ASM_MIX_AVX2 + + "add $96, %[blocks0]\n\t" + "add $96, %[blocks1]\n\t" + "add $96, %[blocks2]\n\t" + "add $96, %[blocks3]\n\t" + "jmp 1b\n\t" + + "2:\n\t" + + /* Copy buf0..buf3 pointers into blocks0..blocks3 to reuse registers */ + "leaq %[buf0], %[blocks0]\n\t" + "leaq %[buf1], %[blocks1]\n\t" + "leaq %[buf2], %[blocks2]\n\t" + "leaq %[buf3], %[blocks3]\n\t" + + /* Load and add buf0..buf3 elements */ + "vmovq 0*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 0*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 0*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 0*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm0, %%ymm0\n\t" + + "vmovq 1*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 1*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 1*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 1*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm1, %%ymm1\n\t" + + "vmovq 2*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 2*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 2*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 2*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm2, %%ymm2\n\t" + + "vmovq 3*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 3*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 3*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 3*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm3, %%ymm3\n\t" + + "vmovq 4*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 4*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 4*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 4*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm4, %%ymm4\n\t" + + "vmovq 5*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 5*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 5*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 5*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm5, %%ymm5\n\t" + + "vmovq 6*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 6*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 6*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 6*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm6, %%ymm6\n\t" + + "vmovq 7*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 7*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 7*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 7*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm7, %%ymm7\n\t" + + "vmovq 8*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 8*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 8*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 8*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm8, %%ymm8\n\t" + + "vmovq 9*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 9*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 9*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 9*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm9, %%ymm9\n\t" + + "vmovq 10*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 10*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 10*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 10*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm10, %%ymm10\n\t" + + "vmovq 11*8(%[blocks0]), %%xmm12\n\t" + "vmovhps 11*8(%[blocks1]), %%xmm12, %%xmm12\n\t" + "vmovq 11*8(%[blocks2]), %%xmm13\n\t" + "vmovhps 11*8(%[blocks3]), %%xmm13, %%xmm13\n\t" + "vinserti128 $1, %%xmm13, %%ymm12, %%ymm12\n\t" + "vpaddq %%ymm12, %%ymm11, %%ymm11\n\t" + + /* EndPartial 3 times */ + ASM_END_PARTIAL_AVX2 + ASM_END_PARTIAL_AVX2 + ASM_END_PARTIAL_AVX2 + + /* Extract upper halves of h0 and h1 */ + "vextracti128 $1, %%ymm0, %%xmm12\n\t" + "vextracti128 $1, %%ymm1, %%xmm13\n\t" + + /* Store digest0 and digest1 using XMM registers */ + "movq %[digest0], %%rax\n\t" + "vmovq %%xmm0, (%%rax)\n\t" + "vmovq %%xmm1, 8(%%rax)\n\t" + + "movq %[digest1], %%rcx\n\t" + "vmovhps %%xmm0, (%%rcx)\n\t" + "vmovhps %%xmm1, 8(%%rcx)\n\t" + + /* Store digest2 and digest3 using extracted XMM registers */ + "movq %[digest2], %%rdx\n\t" + "vmovq %%xmm12, (%%rdx)\n\t" + "vmovq %%xmm13, 8(%%rdx)\n\t" + + "movq %[digest3], %%r8\n\t" + "vmovhps %%xmm12, (%%r8)\n\t" + "vmovhps %%xmm13, 8(%%r8)\n\t" + + : [blocks0] "+r" (blocks0), + [blocks1] "+r" (blocks1), + [blocks2] "+r" (blocks2), + [blocks3] "+r" (blocks3) + : [end0] "r" (end0), + [seed0] "m" (seed0), + [seed1] "m" (seed1), + [seed2] "m" (seed2), + [seed3] "m" (seed3), + [digest0] "m" (digest0), + [digest1] "m" (digest1), + [digest2] "m" (digest2), + [digest3] "m" (digest3), + [sc_const_val] "m" (sc_const_val), + [buf0] "m" (buf0), + [buf1] "m" (buf1), + [buf2] "m" (buf2), + [buf3] "m" (buf3) + : "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory" + ); +} + +#else /* !__x86_64__ */ + +void SpookyHash128AVX( + const void* data0, const void* data1, const void* data2, const void* data3, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, + uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3 +) +{ + SpookyHash128(data0, size, seed0, digest0); + SpookyHash128(data1, size, seed1, digest1); + SpookyHash128(data2, size, seed2, digest2); + SpookyHash128(data3, size, seed3, digest3); +} + +#endif /* __x86_64__ */ + diff --git a/cmdline/spooky2sse.c b/cmdline/spooky2sse.c new file mode 100644 index 00000000..48e584bb --- /dev/null +++ b/cmdline/spooky2sse.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2013 Andrea Mazzoleni + +#if defined(__x86_64__) + +#ifndef sc_numVars +#define sc_numVars 12 +#endif +#ifndef sc_blockSize +#define sc_blockSize (sc_numVars * 8) +#endif +#ifndef sc_const +#define sc_const 0xdeadbeefdeadbeefLL +#endif + +#define ASM_END_STEP(s1, s11, s2, r, r_diff) \ + "paddq " s1 ", " s11 "\n\t" \ + "pxor " s11 ", " s2 "\n\t" \ + "movdqa " s1 ", %%xmm13\n\t" \ + "psllq $" #r ", " s1 "\n\t" \ + "psrlq $" #r_diff ", %%xmm13\n\t" \ + "por %%xmm13, " s1 "\n\t" + +#define ASM_END_PARTIAL \ + ASM_END_STEP("%%xmm1", "%%xmm11", "%%xmm2", 44, 20) \ + ASM_END_STEP("%%xmm2", "%%xmm0", "%%xmm3", 15, 49) \ + ASM_END_STEP("%%xmm3", "%%xmm1", "%%xmm4", 34, 30) \ + ASM_END_STEP("%%xmm4", "%%xmm2", "%%xmm5", 21, 43) \ + ASM_END_STEP("%%xmm5", "%%xmm3", "%%xmm6", 38, 26) \ + ASM_END_STEP("%%xmm6", "%%xmm4", "%%xmm7", 33, 31) \ + ASM_END_STEP("%%xmm7", "%%xmm5", "%%xmm8", 10, 54) \ + ASM_END_STEP("%%xmm8", "%%xmm6", "%%xmm9", 13, 51) \ + ASM_END_STEP("%%xmm9", "%%xmm7", "%%xmm10", 38, 26) \ + ASM_END_STEP("%%xmm10", "%%xmm8", "%%xmm11", 53, 11) \ + ASM_END_STEP("%%xmm11", "%%xmm9", "%%xmm0", 42, 22) \ + ASM_END_STEP("%%xmm0", "%%xmm10", "%%xmm1", 54, 10) + +void SpookyHash128SSE( + const void* data0, const void* data1, + size_t size, + const uint8_t* seed0, const uint8_t* seed1, + uint8_t* digest0, uint8_t* digest1 +) +{ + uint64_t buf0[sc_numVars]; + uint64_t buf1[sc_numVars]; + size_t nblocks; + const uint8_t* blocks0; + const uint8_t* blocks1; + const uint8_t* end0; + size_t size_remainder; + static const uint64_t sc_const_val = sc_const; + + nblocks = size / sc_blockSize; + blocks0 = (const uint8_t*)data0; + blocks1 = (const uint8_t*)data1; + end0 = blocks0 + nblocks * sc_blockSize; + + /* tail data is computed beforehand */ + size_remainder = size - nblocks * sc_blockSize; + memcpy(buf0, blocks0 + nblocks * sc_blockSize, size_remainder); + memset(((uint8_t*)buf0) + size_remainder, 0, sc_blockSize - size_remainder); + ((uint8_t*)buf0)[sc_blockSize - 1] = size_remainder; + + memcpy(buf1, blocks1 + nblocks * sc_blockSize, size_remainder); + memset(((uint8_t*)buf1) + size_remainder, 0, sc_blockSize - size_remainder); + ((uint8_t*)buf1)[sc_blockSize - 1] = size_remainder; + + __asm__ __volatile__ ( + /* Load seed0 into XMM registers */ + "movq (%[seed0]), %%xmm9\n\t" + "movhps (%[seed1]), %%xmm9\n\t" + "movq 8(%[seed0]), %%xmm10\n\t" + "movhps 8(%[seed1]), %%xmm10\n\t" + + /* Load sc_const into XMM15 and duplicate it */ + "movq (%[sc_const_ptr]), %%xmm15\n\t" + "punpcklqdq %%xmm15, %%xmm15\n\t" + + /* Initialize h0..h11 */ + "movdqa %%xmm9, %%xmm0\n\t" + "movdqa %%xmm9, %%xmm3\n\t" + "movdqa %%xmm9, %%xmm6\n\t" + "movdqa %%xmm10, %%xmm1\n\t" + "movdqa %%xmm10, %%xmm4\n\t" + "movdqa %%xmm10, %%xmm7\n\t" + "movdqa %%xmm15, %%xmm2\n\t" + "movdqa %%xmm15, %%xmm5\n\t" + "movdqa %%xmm15, %%xmm8\n\t" + "movdqa %%xmm15, %%xmm11\n\t" + + /* Main loop */ + "1:\n\t" + "cmp %[end0], %[blocks0]\n\t" + "jae 2f\n\t" + + /* Step 0 */ + "movq 0*8(%[blocks0]), %%xmm12\n\t" + "movhps 0*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm0\n\t" + "pxor %%xmm10, %%xmm2\n\t" + "pxor %%xmm0, %%xmm11\n\t" + "movdqa %%xmm0, %%xmm13\n\t" + "psllq $11, %%xmm0\n\t" + "psrlq $53, %%xmm13\n\t" + "por %%xmm13, %%xmm0\n\t" + "paddq %%xmm1, %%xmm11\n\t" + + /* Step 1 */ + "movq 1*8(%[blocks0]), %%xmm12\n\t" + "movhps 1*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm1\n\t" + "pxor %%xmm11, %%xmm3\n\t" + "pxor %%xmm1, %%xmm0\n\t" + "movdqa %%xmm1, %%xmm13\n\t" + "psllq $32, %%xmm1\n\t" + "psrlq $32, %%xmm13\n\t" + "por %%xmm13, %%xmm1\n\t" + "paddq %%xmm2, %%xmm0\n\t" + + /* Step 2 */ + "movq 2*8(%[blocks0]), %%xmm12\n\t" + "movhps 2*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm2\n\t" + "pxor %%xmm0, %%xmm4\n\t" + "pxor %%xmm2, %%xmm1\n\t" + "movdqa %%xmm2, %%xmm13\n\t" + "psllq $43, %%xmm2\n\t" + "psrlq $21, %%xmm13\n\t" + "por %%xmm13, %%xmm2\n\t" + "paddq %%xmm3, %%xmm1\n\t" + + /* Step 3 */ + "movq 3*8(%[blocks0]), %%xmm12\n\t" + "movhps 3*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm3\n\t" + "pxor %%xmm1, %%xmm5\n\t" + "pxor %%xmm3, %%xmm2\n\t" + "movdqa %%xmm3, %%xmm13\n\t" + "psllq $31, %%xmm3\n\t" + "psrlq $33, %%xmm13\n\t" + "por %%xmm13, %%xmm3\n\t" + "paddq %%xmm4, %%xmm2\n\t" + + /* Step 4 */ + "movq 4*8(%[blocks0]), %%xmm12\n\t" + "movhps 4*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm4\n\t" + "pxor %%xmm2, %%xmm6\n\t" + "pxor %%xmm4, %%xmm3\n\t" + "movdqa %%xmm4, %%xmm13\n\t" + "psllq $17, %%xmm4\n\t" + "psrlq $47, %%xmm13\n\t" + "por %%xmm13, %%xmm4\n\t" + "paddq %%xmm5, %%xmm3\n\t" + + /* Step 5 */ + "movq 5*8(%[blocks0]), %%xmm12\n\t" + "movhps 5*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm5\n\t" + "pxor %%xmm3, %%xmm7\n\t" + "pxor %%xmm5, %%xmm4\n\t" + "movdqa %%xmm5, %%xmm13\n\t" + "psllq $28, %%xmm5\n\t" + "psrlq $36, %%xmm13\n\t" + "por %%xmm13, %%xmm5\n\t" + "paddq %%xmm6, %%xmm4\n\t" + + /* Step 6 */ + "movq 6*8(%[blocks0]), %%xmm12\n\t" + "movhps 6*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm6\n\t" + "pxor %%xmm4, %%xmm8\n\t" + "pxor %%xmm6, %%xmm5\n\t" + "movdqa %%xmm6, %%xmm13\n\t" + "psllq $39, %%xmm6\n\t" + "psrlq $25, %%xmm13\n\t" + "por %%xmm13, %%xmm6\n\t" + "paddq %%xmm7, %%xmm5\n\t" + + /* Step 7 */ + "movq 7*8(%[blocks0]), %%xmm12\n\t" + "movhps 7*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm7\n\t" + "pxor %%xmm5, %%xmm9\n\t" + "pxor %%xmm7, %%xmm6\n\t" + "movdqa %%xmm7, %%xmm13\n\t" + "psllq $57, %%xmm7\n\t" + "psrlq $7, %%xmm13\n\t" + "por %%xmm13, %%xmm7\n\t" + "paddq %%xmm8, %%xmm6\n\t" + + /* Step 8 */ + "movq 8*8(%[blocks0]), %%xmm12\n\t" + "movhps 8*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm8\n\t" + "pxor %%xmm6, %%xmm10\n\t" + "pxor %%xmm8, %%xmm7\n\t" + "movdqa %%xmm8, %%xmm13\n\t" + "psllq $55, %%xmm8\n\t" + "psrlq $9, %%xmm13\n\t" + "por %%xmm13, %%xmm8\n\t" + "paddq %%xmm9, %%xmm7\n\t" + + /* Step 9 */ + "movq 9*8(%[blocks0]), %%xmm12\n\t" + "movhps 9*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm9\n\t" + "pxor %%xmm7, %%xmm11\n\t" + "pxor %%xmm9, %%xmm8\n\t" + "movdqa %%xmm9, %%xmm13\n\t" + "psllq $54, %%xmm9\n\t" + "psrlq $10, %%xmm13\n\t" + "por %%xmm13, %%xmm9\n\t" + "paddq %%xmm10, %%xmm8\n\t" + + /* Step 10 */ + "movq 10*8(%[blocks0]), %%xmm12\n\t" + "movhps 10*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm10\n\t" + "pxor %%xmm8, %%xmm0\n\t" + "pxor %%xmm10, %%xmm9\n\t" + "movdqa %%xmm10, %%xmm13\n\t" + "psllq $22, %%xmm10\n\t" + "psrlq $42, %%xmm13\n\t" + "por %%xmm13, %%xmm10\n\t" + "paddq %%xmm11, %%xmm9\n\t" + + /* Step 11 */ + "movq 11*8(%[blocks0]), %%xmm12\n\t" + "movhps 11*8(%[blocks1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm11\n\t" + "pxor %%xmm9, %%xmm1\n\t" + "pxor %%xmm11, %%xmm10\n\t" + "movdqa %%xmm11, %%xmm13\n\t" + "psllq $46, %%xmm11\n\t" + "psrlq $18, %%xmm13\n\t" + "por %%xmm13, %%xmm11\n\t" + "paddq %%xmm0, %%xmm10\n\t" + + "add $96, %[blocks0]\n\t" + "add $96, %[blocks1]\n\t" + "jmp 1b\n\t" + + "2:\n\t" + + /* Load and add buf0 and buf1 elements */ + "movq 0*8(%[buf0]), %%xmm12\n\t" + "movhps 0*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm0\n\t" + + "movq 1*8(%[buf0]), %%xmm12\n\t" + "movhps 1*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm1\n\t" + + "movq 2*8(%[buf0]), %%xmm12\n\t" + "movhps 2*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm2\n\t" + + "movq 3*8(%[buf0]), %%xmm12\n\t" + "movhps 3*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm3\n\t" + + "movq 4*8(%[buf0]), %%xmm12\n\t" + "movhps 4*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm4\n\t" + + "movq 5*8(%[buf0]), %%xmm12\n\t" + "movhps 5*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm5\n\t" + + "movq 6*8(%[buf0]), %%xmm12\n\t" + "movhps 6*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm6\n\t" + + "movq 7*8(%[buf0]), %%xmm12\n\t" + "movhps 7*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm7\n\t" + + "movq 8*8(%[buf0]), %%xmm12\n\t" + "movhps 8*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm8\n\t" + + "movq 9*8(%[buf0]), %%xmm12\n\t" + "movhps 9*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm9\n\t" + + "movq 10*8(%[buf0]), %%xmm12\n\t" + "movhps 10*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm10\n\t" + + "movq 11*8(%[buf0]), %%xmm12\n\t" + "movhps 11*8(%[buf1]), %%xmm12\n\t" + "paddq %%xmm12, %%xmm11\n\t" + + /* EndPartial 3 times */ + ASM_END_PARTIAL + ASM_END_PARTIAL + ASM_END_PARTIAL + + /* Store back h0 and h1 to output digests */ + "movq %%xmm0, (%[digest0])\n\t" + "movhps %%xmm0, (%[digest1])\n\t" + "movq %%xmm1, 8(%[digest0])\n\t" + "movhps %%xmm1, 8(%[digest1])\n\t" + + : [blocks0] "+r" (blocks0), + [blocks1] "+r" (blocks1) + : [end0] "r" (end0), + [seed0] "r" (seed0), + [seed1] "r" (seed1), + [digest0] "r" (digest0), + [digest1] "r" (digest1), + [sc_const_ptr] "r" (&sc_const_val), + [buf0] "r" (buf0), + [buf1] "r" (buf1) + : "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "cc", "memory" + ); +} + +#endif /* __x86_64__ */ + diff --git a/cmdline/util.c b/cmdline/util.c index 4f9cb2ef..1eccccac 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -760,8 +760,12 @@ unsigned membesthash(void) #include "murmur3.c" #include "spooky2.c" +#include "spooky2sse.c" +#include "spooky2avx.c" #include "metro.c" #include "museair.c" +#include "museairsse.c" +#include "museairavx.c" void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size) { diff --git a/cmdline/util.h b/cmdline/util.h index 0731bd15..383b4610 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -230,6 +230,12 @@ const char* memhashname(unsigned kind); * Compute the HASH of a memory block. * Seed is a 128 bit vector. */ +void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest); +void SpookyHash128SSE(const void* data0, const void* data1, size_t size, const uint8_t* seed0, const uint8_t* seed1, uint8_t* digest0, uint8_t* digest1); +void SpookyHash128AVX(const void* data0, const void* data1, const void* data2, const void* data3, size_t size, const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3); +void MuseAirLoongSSE(const void* data0, const void* data1, size_t size, const uint8_t* seed0, const uint8_t* seed1, uint8_t* digest0, uint8_t* digest1); +void MuseAirLoongAVX(const void* data0, const void* data1, const void* data2, const void* data3, size_t size, const uint8_t* seed0, const uint8_t* seed1, const uint8_t* seed2, const uint8_t* seed3, uint8_t* digest0, uint8_t* digest1, uint8_t* digest2, uint8_t* digest3); +void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uint8_t* out); void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size); /**