From b725333a490986327110f6aed6504fb2fde3cc93 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 23 Nov 2019 08:26:57 +0100 Subject: [PATCH] Add MetroHash128 for speed measurement --- Makefile.am | 1 + cmdline/metro.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++ cmdline/speed.c | 9 ++++ cmdline/state.c | 10 +++++ cmdline/util.c | 33 +++++++++++++++ cmdline/util.h | 1 + 6 files changed, 162 insertions(+) create mode 100644 cmdline/metro.c diff --git a/Makefile.am b/Makefile.am index 4c0c167a..9cd51a2b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -83,6 +83,7 @@ noinst_HEADERS = \ cmdline/murmur3test.c \ cmdline/spooky2.c \ cmdline/spooky2test.c \ + cmdline/metro.c \ cmdline/fnmatch.h \ cmdline/import.h \ cmdline/search.h \ diff --git a/cmdline/metro.c b/cmdline/metro.c new file mode 100644 index 00000000..18f40277 --- /dev/null +++ b/cmdline/metro.c @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2019 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Derivative work from metrohash128.cpp + * + * metrohash128.cpp + * + * Copyright 2015-2018 J. Andrew Rogers + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +static const uint64_t k0 = 0xC83A91E1; +static const uint64_t k1 = 0x8648DBDB; +static const uint64_t k2 = 0x7BDEC03B; +static const uint64_t k3 = 0x2F5870A5; + +void MetroHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) +{ + const uint8_t* ptr = data; + uint64_t v[4]; + + v[0] = (util_read64(seed) - k0) * k3; + v[1] = (util_read64(seed + 8) + k1) * k2; + + if (size >= 32) { + v[2] = (util_read64(seed) + k0) * k2; + v[3] = (util_read64(seed + 8) - k1) * k3; + + do { + v[0] += util_read64(ptr) * k0; ptr += 8; v[0] = util_rotr64(v[0], 29) + v[2]; + v[1] += util_read64(ptr) * k1; ptr += 8; v[1] = util_rotr64(v[1], 29) + v[3]; + v[2] += util_read64(ptr) * k2; ptr += 8; v[2] = util_rotr64(v[2], 29) + v[0]; + v[3] += util_read64(ptr) * k3; ptr += 8; v[3] = util_rotr64(v[3], 29) + v[1]; + size -= 32; + } while (size >= 32); + + v[2] ^= util_rotr64(((v[0] + v[3]) * k0) + v[1], 21) * k1; + v[3] ^= util_rotr64(((v[1] + v[2]) * k1) + v[0], 21) * k0; + v[0] ^= util_rotr64(((v[0] + v[2]) * k0) + v[3], 21) * k1; + v[1] ^= util_rotr64(((v[1] + v[3]) * k1) + v[2], 21) * k0; + } + + if (size >= 16) { + v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3; + v[1] += util_read64(ptr) * k2; ptr += 8; v[1] = util_rotr64(v[1], 33) * k3; + v[0] ^= util_rotr64((v[0] * k2) + v[1], 45) * k1; + v[1] ^= util_rotr64((v[1] * k3) + v[0], 45) * k0; + size -= 16; + } + + if (size >= 8) { + v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3; + v[0] ^= util_rotr64((v[0] * k2) + v[1], 27) * k1; + size -= 8; + } + + if (size >= 4) { + v[1] += util_read32(ptr) * k2; ptr += 4; v[1] = util_rotr64(v[1], 33) * k3; + v[1] ^= util_rotr64((v[1] * k3) + v[0], 46) * k0; + size -= 4; + } + + if (size >= 2) { + v[0] += util_read16(ptr) * k2; ptr += 2; v[0] = util_rotr64(v[0], 33) * k3; + v[0] ^= util_rotr64((v[0] * k2) + v[1], 22) * k1; + size -= 2; + } + + if (size >= 1) { + v[1] += util_read8(ptr) * k2; v[1] = util_rotr64(v[1], 33) * k3; + v[1] ^= util_rotr64((v[1] * k3) + v[0], 58) * k0; + } + + v[0] += util_rotr64((v[0] * k0) + v[1], 13); + v[1] += util_rotr64((v[1] * k1) + v[0], 37); + v[0] += util_rotr64((v[0] * k2) + v[1], 13); + v[1] += util_rotr64((v[1] * k3) + v[0], 37); + + util_write64(digest, v[0]); + util_write64(digest + 8, v[0]); +} + diff --git a/cmdline/speed.c b/cmdline/speed.c index f3ecdf8f..73d2722d 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -216,6 +216,7 @@ void speed(int period) printf("%8s", "best"); printf("%8s", "murmur3"); printf("%8s", "spooky2"); + printf("%8s", "metro"); printf("\n"); printf("%8s", "hash"); @@ -245,6 +246,14 @@ void speed(int period) memhash(HASH_SPOOKY2, seed, digest, v[j], size); } SPEED_STOP + printf("%8" PRIu64, ds / dt); + fflush(stdout); + + SPEED_START { + for (j = 0; j < nd; ++j) + memhash(HASH_METRO, seed, digest, v[j], size); + } SPEED_STOP + printf("%8" PRIu64, ds / dt); printf("\n"); printf("\n"); diff --git a/cmdline/state.c b/cmdline/state.c index 667167ff..b32c5f0c 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2406,6 +2406,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S case 'k' : state->hash = HASH_SPOOKY2; break; + case 'm' : + state->hash = HASH_METRO; + break; default : /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2433,6 +2436,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S case 'k' : state->prevhash = HASH_SPOOKY2; break; + case 'm' : + state->prevhash = HASH_METRO; + break; default : /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2955,6 +2961,8 @@ static void* state_write_thread(void* arg) sputc('u', f); } else if (state->hash == HASH_SPOOKY2) { sputc('k', f); + } else if (state->hash == HASH_METRO) { + sputc('m', f); } else { /* LCOV_EXCL_START */ log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f)); @@ -2978,6 +2986,8 @@ static void* state_write_thread(void* arg) sputc('u', f); } else if (state->prevhash == HASH_SPOOKY2) { sputc('k', f); + } else if (state->prevhash == HASH_METRO) { + sputc('m', f); } else { /* LCOV_EXCL_START */ log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); diff --git a/cmdline/util.c b/cmdline/util.c index 1a35b857..c4b36409 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -470,6 +470,22 @@ static inline uint64_t util_rotl64(uint64_t x, int8_t r) return (x << r) | (x >> (64 - r)); } +/* + * Rotate right. + * In x86/x64 they are optimized with a single assembler instruction. + */ +#if 0 /* unused */ +static inline uint32_t util_rotr32(uint32_t x, int8_t r) +{ + return (x >> r) | (x << (32 - r)); +} +#endif +static inline uint64_t util_rotr64(uint64_t x, int8_t r) +{ + return (x >> r) | (x << (64 - r)); +} + + /** * Swap endianess. * They are needed only if BigEndian. @@ -502,6 +518,18 @@ static inline uint64_t util_swap64(uint64_t v) } #endif +static inline uint8_t util_read8(const void* void_ptr) +{ + const uint8_t* ptr = void_ptr; + return ptr[0]; +} + +static inline uint16_t util_read16(const void* void_ptr) +{ + const uint8_t* ptr = void_ptr; + return ptr[0] + (ptr[1] << 8); +} + static inline uint32_t util_read32(const void* ptr) { uint32_t v; @@ -543,6 +571,7 @@ static inline void util_write64(void* ptr, uint64_t v) #include "murmur3.c" #include "spooky2.c" +#include "metro.c" void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size) { @@ -553,6 +582,9 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* case HASH_SPOOKY2 : SpookyHash128(src, size, seed, digest); break; + case HASH_METRO : + MetroHash128(src, size, seed, digest); + break; default : /* LCOV_EXCL_START */ log_fatal("Internal inconsistency in hash function %u\n", kind); @@ -568,6 +600,7 @@ const char* hash_config_name(unsigned kind) case HASH_UNDEFINED : return "undefined"; case HASH_MURMUR3 : return "murmur3"; case HASH_SPOOKY2 : return "spooky2"; + case HASH_METRO : return "metro"; default : /* LCOV_EXCL_START */ return "unknown"; diff --git a/cmdline/util.h b/cmdline/util.h index 1661a9df..3d7e3941 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -188,6 +188,7 @@ void crc32c_init(void); #define HASH_UNDEFINED 0 #define HASH_MURMUR3 1 #define HASH_SPOOKY2 2 +#define HASH_METRO 3 /** * Compute the HASH of a memory block.