From f4f308e32f1544b92b2bcb8363ed62f3eec3a22b Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 1 Mar 2011 22:21:30 +0100 Subject: [PATCH] Initial commit --- Makefile.am | 83 +++ README | 5 + acinclude.m4 | 20 + autogen.sh | 17 + configure.ac | 47 ++ getopt.c | 113 ++++ md5.c | 246 +++++++ md5.h | 46 ++ portable.h | 114 ++++ snapraid.c | 1681 ++++++++++++++++++++++++++++++++++++++++++++++++ test.conf | 7 + testgen.c | 76 +++ tommy.c | 31 + tommyarray.c | 72 +++ tommyarray.h | 158 +++++ tommychain.h | 224 +++++++ tommyhash.c | 167 +++++ tommyhash.h | 104 +++ tommyhashdyn.c | 174 +++++ tommyhashdyn.h | 250 +++++++ tommylist.c | 87 +++ tommylist.h | 309 +++++++++ tommytypes.h | 313 +++++++++ util.h | 170 +++++ 24 files changed, 4514 insertions(+) create mode 100644 Makefile.am create mode 100644 README create mode 100644 acinclude.m4 create mode 100755 autogen.sh create mode 100644 configure.ac create mode 100644 getopt.c create mode 100644 md5.c create mode 100644 md5.h create mode 100644 portable.h create mode 100644 snapraid.c create mode 100644 test.conf create mode 100644 testgen.c create mode 100644 tommy.c create mode 100644 tommyarray.c create mode 100644 tommyarray.h create mode 100644 tommychain.h create mode 100644 tommyhash.c create mode 100644 tommyhash.h create mode 100644 tommyhashdyn.c create mode 100644 tommyhashdyn.h create mode 100644 tommylist.c create mode 100644 tommylist.h create mode 100644 tommytypes.h create mode 100644 util.h diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 00000000..4bbaa0bb --- /dev/null +++ b/Makefile.am @@ -0,0 +1,83 @@ +bin_PROGRAMS = snapraid +check_PROGRAMS = testgen + +snapraid_SOURCES = \ + snapraid.c \ + getopt.c \ + md5.c \ + tommy.c + + +noinst_HEADERS = \ + portable.h \ + md5.h \ + tommyarray.h \ + tommychain.h \ + tommyhash.h \ + tommyhashdyn.h \ + tommylist.h \ + tommytypes.h \ + util.h \ + tommyarray.c \ + tommyhash.c \ + tommyhashdyn.c \ + tommylist.c + +testgen_SOURCES = \ + testgen.c + +EXTRA_DIST = \ + autogen.sh \ + noautomake.sh \ + README AUTHORS HISTORY INSTALL COPYING \ + test.conf \ + acinclude.m4 + +clean-local: + rm -f valgrind.log callgrind.log + rm -rf test + +CHECK = ./snapraid -v -c test.conf sync + +check-local: + rm -rf test + mkdir test + mkdir test/disk1 + mkdir test/disk2 + mkdir test/disk3 + ./testgen 1 3 2000 + ./snapraid -v -c $(srcdir)/test.conf sync + ./snapraid -v -c $(srcdir)/test.conf check + ./testgen 2 3 2000 + ./snapraid -v -c $(srcdir)/test.conf sync + ./snapraid -v -c $(srcdir)/test.conf check + rm test/disk1/a* + ./snapraid -v -c $(srcdir)/test.conf check + echo Success! + +valgrind: + valgrind \ + --tool=memcheck \ + --track-origins=yes \ + --read-var-info=yes \ + --leak-check=full \ + -v $(CHECK) \ + 2> valgrind.log + tail valgrind.log + +callgrind: + valgrind \ + --tool=callgrind \ + --dump-instr=yes \ + --trace-jump=yes \ + -v $(CHECK) \ + 2> callgrind.log + tail callgrind.log + +cachegrind: + valgrind \ + --tool=cachegrind \ + -v $(CHECK) \ + 2> cachegrind.log + tail cachegrind.log + diff --git a/README b/README new file mode 100644 index 00000000..0c6a1a6d --- /dev/null +++ b/README @@ -0,0 +1,5 @@ +SNAPRAID +======== + + + diff --git a/acinclude.m4 b/acinclude.m4 new file mode 100644 index 00000000..ceed823a --- /dev/null +++ b/acinclude.m4 @@ -0,0 +1,20 @@ +dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno) +dnl +dnl Shows a message as like "checking wether gcc accepts flag ... no" +dnl and executess ifyes or ifno. + +AC_DEFUN([AC_CHECK_CC_OPT], +[ +AC_MSG_CHECKING([whether ${CC-cc} accepts $1]) +echo 'void f(){}' > conftest.c +if test -z "`${CC-cc} -c $1 conftest.c 2>&1`"; then + AC_MSG_RESULT([yes]) + $2 +else + AC_MSG_RESULT([no]) + $3 +fi +rm -f conftest* +]) + + diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000..68fcf2ef --- /dev/null +++ b/autogen.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# +echo "Generating build information using aclocal, automake and autoconf" +echo "This may take a while ..." + +# Touch the timestamps on all the files since CVS messes them up +touch configure.ac + +# Regenerate configuration files +aclocal +automake --add-missing --force-missing +autoconf +autoheader && touch config.h + +# Run configure for this platform +echo "Now you are ready to run ./configure" + diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..8b44cd3e --- /dev/null +++ b/configure.ac @@ -0,0 +1,47 @@ +dnl Process this file with autoconf to produce a configure script. +AC_INIT([snapraid], [0.1]) +AM_INIT_AUTOMAKE([foreign no-dependencies]) +AC_CONFIG_SRCDIR([snapraid.c]) +AC_CONFIG_HEADER([config.h]) +AC_CANONICAL_HOST +AC_COPYRIGHT([Copyright 2011 Andrea Mazzoleni]) + +dnl Checks for programs. +AC_PROG_CC + +dnl Checks for libraries. +AC_SYS_LARGEFILE + +dnl Checks for header files. +AC_HEADER_STDC +AC_HEADER_DIRENT +AC_HEADER_TIME +AC_FUNC_UTIME_NULL +AC_CHECK_HEADERS([unistd.h getopt.h stdint.h]) + +dnl Checks for typedefs, structures, and compiler characteristics. +AC_C_CONST +AC_C_INLINE +AC_C_RESTRICT +AC_C_VOLATILE +AC_TYPE_SIZE_T + +dnl Checks for library functions. +AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf]) +AC_CHECK_FUNCS([fsync fallocate]) + +dnl Configure for 64 bit files +CFLAGS="$CFLAGS -D_FILE_OFFSET_BITS=64" + +dnl Enable GLIBC symbols +AC_DEFINE(_GNU_SOURCE,1,[Enable GLIBC symbols]) + +dnl Checks for architecture +AC_C_BIGENDIAN + +dnl Checks for compiler +AC_CHECK_CC_OPT([-Wall], CFLAGS="$CFLAGS -Wall", []) + +AC_CONFIG_FILES([Makefile]) +AC_OUTPUT + diff --git a/getopt.c b/getopt.c new file mode 100644 index 00000000..b8c070aa --- /dev/null +++ b/getopt.c @@ -0,0 +1,113 @@ +/* + * This file is part of the Advance project. + * + * Copyright (C) 2001, 2002 Andrea Mazzoleni + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#if HAVE_CONFIG_H +#include +#endif + +#if !HAVE_GETOPT + +/* This source is extracted from the DJGPP LIBC library */ + +#define unconst(var, type) ((type)(var)) + +/* Copyright (C) 1995 DJ Delorie, see COPYING.DJ for details */ +#include +#include + +int opterr = 1, optind = 1, optopt = 0; +char *optarg = 0; + +#define BADCH (int)'?' +#define EMSG "" + +int getopt(int nargc, char *const nargv[], const char *ostr) +{ + static const char *place = EMSG; /* option letter processing */ + char *oli; /* option letter list index */ + char *p; + + if (!*place) + { + if (optind >= nargc || *(place = nargv[optind]) != '-') + { + place = EMSG; + return(EOF); + } + if (place[1] && *++place == '-') + { + ++optind; + place = EMSG; + return(EOF); + } + } + + if ((optopt = (int)*place++) == (int)':' + || !(oli = strchr(ostr, optopt))) + { + /* + * if the user didn't specify '-' as an option, + * assume it means EOF. + */ + if (optopt == (int)'-') + return EOF; + if (!*place) + ++optind; + if (opterr) + { + if (!(p = strrchr(*nargv, '/'))) + p = *nargv; + else + ++p; + fprintf(stderr, "%s: illegal option -- %c\n", p, optopt); + } + return BADCH; + } + if (*++oli != ':') + { /* don't need argument */ + optarg = NULL; + if (!*place) + ++optind; + } + else + { /* need an argument */ + if (*place) /* no white space */ + optarg = unconst(place, char *); + else if (nargc <= ++optind) + { /* no arg */ + place = EMSG; + if (!(p = strrchr(*nargv, '/'))) + p = *nargv; + else + ++p; + if (opterr) + fprintf(stderr, "%s: option requires an argument -- %c\n", p, optopt); + return BADCH; + } + else /* white space */ + optarg = nargv[optind]; + place = EMSG; + ++optind; + } + return optopt; /* dump back option letter */ +} + +#endif + diff --git a/md5.c b/md5.c new file mode 100644 index 00000000..fcad6ae1 --- /dev/null +++ b/md5.c @@ -0,0 +1,246 @@ +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + * + * Changed so as no longer to depend on Colin Plumb's `usual.h' header + * definitions; now uses stuff from dpkg's config.h. + * - Ian Jackson . + * Still in the public domain. + * + * Added convenience MD5(buffer,length,digest) function. + * - Gordon Mohr + * Still in the public domain. + */ + +#include "portable.h" + +#include "md5.h" + +#if WORDS_BIGENDIAN +void byteSwap(uint32_t *buf, unsigned words) +{ + md5byte *p = (md5byte *)buf; + + do { + *buf++ = (uint32_t)((unsigned)p[3] << 8 | p[2]) << 16 | + ((unsigned)p[1] << 8 | p[0]); + p += 4; + } while (--words); +} +#else +#define byteSwap(buf,words) +#endif + +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f,w,x,y,z,in,s) \ + (w += f(x,y,z) + in, w = (w<>(32-s)) + x) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. md5_update blocks + * the data and converts bytes into longwords for this routine. + */ +static void +md5_transform(uint32_t buf[4], uint32_t const in[16]) +{ + uint32_t a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +void md5_init(struct md5_t *ctx) +{ + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + + ctx->bytes[0] = 0; + ctx->bytes[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +void md5_update(struct md5_t* ctx, const unsigned char* buf, unsigned len) +{ + uint32_t t; + + /* Update byte count */ + + t = ctx->bytes[0]; + if ((ctx->bytes[0] = t + len) < t) + ctx->bytes[1]++; /* Carry from low to high */ + + t = 64 - (t & 0x3f); /* Space available in ctx->in (at least 1) */ + if (t > len) { + memcpy((uint8_t *)ctx->in + 64 - t, buf, len); + return; + } + /* First chunk is an odd size */ + memcpy((uint8_t *)ctx->in + 64 - t, buf, t); + byteSwap(ctx->in, 16); + md5_transform(ctx->buf, ctx->in); + buf += t; + len -= t; + + /* Process data in 64-byte chunks */ + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteSwap(ctx->in, 16); + md5_transform(ctx->buf, ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +void md5_final(struct md5_t* ctx, unsigned char* digest) +{ + int count = ctx->bytes[0] & 0x3f; /* Number of bytes in ctx->in */ + uint8_t *p = (uint8_t *)ctx->in + count; + + /* Set the first char of padding to 0x80. There is always room. */ + *p++ = 0x80; + + /* Bytes of padding needed to make 56 bytes (-8..55) */ + count = 56 - 1 - count; + + if (count < 0) { /* Padding forces an extra block */ + memset(p, 0, count + 8); + byteSwap(ctx->in, 16); + md5_transform(ctx->buf, ctx->in); + p = (uint8_t *)ctx->in; + count = 56; + } + memset(p, 0, count); + byteSwap(ctx->in, 14); + + /* Append length in bits and transform */ + ctx->in[14] = ctx->bytes[0] << 3; + ctx->in[15] = ctx->bytes[1] << 3 | ctx->bytes[0] >> 29; + md5_transform(ctx->buf, ctx->in); + + byteSwap(ctx->buf, 4); + memcpy(digest, ctx->buf, 16); +} + +/* + * Convenience function for when what you need to MD5 is + * in a single contiguous buffer. + */ +void MD5(uint8_t *buffer, unsigned int length, uint8_t *digest) +{ + struct md5_t md5ctx; + md5_init(&md5ctx); + md5_update(&md5ctx,buffer,length); + md5_final(&md5ctx,digest); +} + diff --git a/md5.h b/md5.h new file mode 100644 index 00000000..70d33b41 --- /dev/null +++ b/md5.h @@ -0,0 +1,46 @@ +/* + * This is the header file for the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + * + * Changed so as no longer to depend on Colin Plumb's `usual.h' + * header definitions; now uses stuff from dpkg's config.h + * - Ian Jackson . + * Still in the public domain. + */ + +#ifndef MD5_H +#define MD5_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define MD5_SIZE 16 + +struct md5_t { + uint32_t buf[4]; + uint32_t bytes[2]; + uint32_t in[16]; +}; + +void md5_init(struct md5_t* context); +void md5_update(struct md5_t* context, const unsigned char* buf, unsigned len); +void md5_final(struct md5_t* context, unsigned char* digest); + +#ifdef __cplusplus +} +#endif + +#endif /* !MD5_H */ diff --git a/portable.h b/portable.h new file mode 100644 index 00000000..a8738cf0 --- /dev/null +++ b/portable.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __PORTABLE_H +#define __PORTABLE_H + +#if HAVE_CONFIG_H +#include "config.h" /* Use " to include first in the same directory of this file */ +#endif + +/***************************************************************************/ +/* Config */ + +#ifdef __WIN32__ +/* Customize for Windows Mingw/Cygwin */ +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_STAT_H 1 +#define HAVE_DIRENT_H 1 +#define TIME_WITH_SYS_TIME 1 +#define HAVE_SYS_TIME_H 1 +#define HAVE_UNISTD_H 1 +#endif + +/* Include some standard headers */ +#include +#include /* On many systems (e.g., Darwin), `stdio.h' is a prerequisite. */ +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_STDINT_H +#include +#endif + +#if HAVE_UNISTD_H +#include +#endif + +#if TIME_WITH_SYS_TIME +#include +#include +#else +#if HAVE_SYS_TIME_H +#include +#else +#include +#endif +#endif + +#if HAVE_DIRENT_H +#include +#define NAMLEN(dirent) strlen((dirent)->d_name) +#else +#define dirent direct +#define NAMLEN(dirent) (dirent)->d_namlen +#if HAVE_SYS_NDIR_H +#include +#endif +#if HAVE_SYS_DIR_H +#include +#endif +#if HAVE_NDIR_H +#include +#endif +#endif + +#if HAVE_SYS_TYPES_H +#include +#endif + +#if HAVE_SYS_STAT_H +#include +#endif + +#if HAVE_GETOPT_H +#include +#endif + +#if !HAVE_GETOPT +int getopt(int argc, char * const *argv, const char *options); +extern char *optarg; +extern int optind, opterr, optopt; +#endif + +#if HAVE_GETOPT_LONG +#define SWITCH_GETOPT_LONG(a,b) a +#else +#define SWITCH_GETOPT_LONG(a,b) b +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#endif + diff --git a/snapraid.c b/snapraid.c new file mode 100644 index 00000000..a9f69ebd --- /dev/null +++ b/snapraid.c @@ -0,0 +1,1681 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "util.h" +#include "md5.h" +#include "tommylist.h" +#include "tommyhash.h" +#include "tommyhashdyn.h" +#include "tommyarray.h" + +/****************************************************************************/ +/* generic */ + +/** + * Max length of a line. + */ +#define LINE_MAX 1024 + +/** + * If Ctrl+C was pressed. + */ +static volatile int global_interrupt = 0; + +/** + * Safe malloc. + */ +void* malloc_nofail(size_t size) +{ + void* ptr = malloc(size); + + if (!ptr) { + fprintf(stderr, "Low memory\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} + +/****************************************************************************/ +/* snapraid */ + +/** + * Size of the hash used as a checksum. + */ +#define HASH_MAX MD5_SIZE + +/** + * Invalid position. + */ +#define POS_INVALID -1 + +/** + * Basic position type. + * With a 32 bit and 128k block you can address 256 TB. + */ +typedef unsigned pos_t; + +struct snapraid_file; + +/** + * Block of a file. + */ +struct snapraid_block { + pos_t parity_pos; /**< Position of the block in the parity. */ + int is_hashed; /**< If the hash of the block is valid. */ + struct snapraid_file* file; /**< Back pointer to the file owning this block. */ + unsigned char hash[HASH_MAX]; /**< Hash of the block. */ +}; + +/** + * File. + */ +struct snapraid_file { + char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */ + uint64_t size; /**< Size of the file. */ + struct snapraid_block* blockvec; /**< All the blocks of the file. */ + unsigned blockmax; /**< Number of blocks. */ + time_t mtime; /**< Modification time. */ + int is_present; /**< If it's seen as present. */ + + /* nodes for data structures */ + tommy_node nodelist; + tommy_hashdyn_node nodeset; +}; + +/** + * Disk. + */ +struct snapraid_disk { + char name[PATH_MAX]; /**< Name of the disk. */ + char dir[PATH_MAX]; /**< Mount point of the disk. */ + pos_t first_free_block; /**< First free searching block. */ + tommy_list filelist; /**< List of all the files. */ + tommy_hashdyn fileset; /**< Hashtable by sub of all the files. */ + tommy_array blockarr; /**< Block array of the disk. */ +}; + +/****************************************************************************/ +/* snapraid */ + +/** + * Get the relative position of a block inside the file. + */ +pos_t block_file_pos(struct snapraid_block* block) +{ + struct snapraid_file* file = block->file; + + if (block < file->blockvec || block >= file->blockvec + file->blockmax) { + fprintf(stderr, "Internal inconsistency in block %u ownership\n", block->parity_pos); + exit(EXIT_FAILURE); + } + + return block - file->blockvec; +} + +/** + * Get the size in bytes of the block. + * If it's the last block of a file it could less than block_size. + */ +unsigned block_file_size(struct snapraid_block* block, unsigned block_size) +{ + pos_t pos = block_file_pos(block); + + /* if it's the last block */ + if (pos + 1 == block->file->blockmax) { + unsigned remainder; + if (block->file->size == 0) + return 0; + remainder = block->file->size % block_size; + if (remainder == 0) + remainder = block_size; + return remainder; + } + + return block_size; +} + +struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, time_t mtime) +{ + struct snapraid_file* file; + pos_t i; + + file = malloc_nofail(sizeof(struct snapraid_file)); + snprintf(file->sub, sizeof(file->sub), "%s", sub); + file->size = size; + file->blockmax = (size + block_size - 1) / block_size; + file->mtime = mtime; + file->is_present = 0; + file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); + + /* set the back pointer */ + for(i=0;iblockmax;++i) { + file->blockvec[i].parity_pos = POS_INVALID; + file->blockvec[i].is_hashed = 0; + file->blockvec[i].file = file; + } + + return file; +} + +void file_free(struct snapraid_file* file) +{ + free(file->blockvec); + free(file); +} + +int file_compare(const void* void_arg, const void* void_data) +{ + const char* arg = void_arg; + const struct snapraid_file* file = void_data; + return strcmp(arg, file->sub); +} + +tommy_uint32_t file_hash(const char* sub) +{ + return tommy_hash_u32(0, sub, strlen(sub)); +} + +struct snapraid_disk* disk_alloc(const char* name, const char* dir) +{ + struct snapraid_disk* disk; + + disk = malloc_nofail(sizeof(struct snapraid_disk)); + snprintf(disk->name, sizeof(disk->name), "%s", name); + snprintf(disk->dir, sizeof(disk->dir), "%s", dir); + disk->first_free_block = 0; + tommy_list_init(&disk->filelist); + tommy_hashdyn_init(&disk->fileset); + tommy_array_init(&disk->blockarr); + + return disk; +} + +void disk_free(struct snapraid_disk* disk) +{ + tommy_node* node = disk->filelist; + while (node) { + struct snapraid_file* file = node->data; + node = node->next; + file_free(file); + } + tommy_hashdyn_done(&disk->fileset); + tommy_array_done(&disk->blockarr); + free(disk); +} + +static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, pos_t pos) +{ + if (pos < tommy_array_size(&disk->blockarr)) + return tommy_array_get(&disk->blockarr, pos); + else + return 0; +} + +/****************************************************************************/ +/* state */ + +struct snapraid_state { + int verbose; /**< Verbose output. */ + unsigned block_size; /**< Block size in bytes. */ + char content[PATH_MAX]; /**< Path of the content file. */ + char parity[PATH_MAX]; /**< Path of the parity file. */ + tommy_array diskarr; /**< Disk array. */ +}; + +void state_init(struct snapraid_state* state) +{ + state->verbose = 0; + state->block_size = 128*1024; /* default 128 kB */ + state->content[0] = 0; + state->parity[0] = 0; + tommy_array_init(&state->diskarr); + + if (sizeof(off_t) < sizeof(uint64_t)) { + fprintf(stderr, "Internal inconsistency in off_t type size\n"); + exit(EXIT_FAILURE); + } +} + +void state_done(struct snapraid_state* state) +{ + unsigned i; + for(i=0;idiskarr);++i) + disk_free(tommy_array_get(&state->diskarr, i)); + tommy_array_done(&state->diskarr); +} + +/****************************************************************************/ +/* config */ + +void state_config(struct snapraid_state* state, const char* path, int verbose) +{ + FILE* f; + unsigned line; + + state->verbose = verbose; + + f = fopen(path, "rt"); + if (!f) { + if (errno == ENOENT) { + fprintf(stderr, "No configuration file found at '%s'\n", path); + } else if (errno == EACCES) { + fprintf(stderr, "You do not have rights to access the configuration file '%s'\n", path); + } else { + fprintf(stderr, "Error opening the configuration file '%s'. %s.\n", path, strerror(errno)); + } + exit(EXIT_FAILURE); + } + + line = 0; + while (1) { + char buffer[LINE_MAX]; + char* tag; + char* s; + int ret; + + ++line; + + ret = strgets(buffer, LINE_MAX, f); + if (ret < 0) { + fprintf(stderr, "Error reading the configuration file '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (ret == 0) + break; + + /* start */ + s = buffer; + s = strskip(s); + + /* ignore commens and empty lines */ + if (*s == '#' || *s == 0) + continue; + + tag = s; + s = strtoken(s); + + if (strcmp(tag, "block_size") == 0) { + ret = stru(s, &state->block_size); + if (ret != 0) { + fprintf(stderr, "Invalid 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (state->block_size < 1) { + fprintf(stderr, "Too small 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (state->block_size > 16*1024) { + fprintf(stderr, "Too big 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + /* check if it's a power of 2 */ + if ((state->block_size & (state->block_size - 1)) != 0) { + fprintf(stderr, "Not power of 2 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + state->block_size *= 1024; + } else if (strcmp(tag, "parity") == 0) { + if (*state->parity) { + fprintf(stderr, "Multiple 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + snprintf(state->parity, sizeof(state->parity), "%s", s); + } else if (strcmp(tag, "content") == 0) { + if (*state->content) { + fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + snprintf(state->content, sizeof(state->content), "%s", s); + } else if (strcmp(tag, "disk") == 0) { + char* name = s; + s = strtoken(s); + tommy_array_insert(&state->diskarr, disk_alloc(name, s)); + } else { + fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path); + exit(EXIT_FAILURE); + } + } + + fclose(f); + + if (!state->parity) { + fprintf(stderr, "No 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (!state->content) { + fprintf(stderr, "No 'content' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } +} + +/****************************************************************************/ +/* read */ + +/** + * Read the state. + */ +void state_read(struct snapraid_state* state) +{ + FILE* f; + char path[PATH_MAX]; + struct snapraid_disk* disk; + struct snapraid_file* file; + unsigned block_index; + unsigned line; + unsigned count_file; + unsigned count_block; + + count_file = 0; + count_block = 0; + + snprintf(path, sizeof(path), "%s", state->content); + f = fopen(state->content, "r"); + if (!f) { + /* if not found, assume empty */ + if (errno == ENOENT) + return; + + fprintf(stderr, "Error opening the state file '%s'\n", path); + exit(EXIT_FAILURE); + } + + printf("Loading state...\n"); + + disk = 0; + file = 0; + line = 0; + block_index = 0; + while (1) { + char buffer[LINE_MAX]; + char* tag; + char* s; + int ret; + + ++line; + + ret = strgets(buffer, LINE_MAX, f); + if (ret < 0) { + fprintf(stderr, "Error reading the state file '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (ret == 0) + break; + + /* start */ + s = buffer; + s = strskip(s); + + /* ignore commens and empty lines */ + if (*s == '#' || *s == 0) + continue; + + tag = s; + s = strtoken(s); + + if (strcmp(tag, "file") == 0) { + char* name; + char* size; + char* mtime; + char* sub; + uint64_t v_size; + uint64_t v_mtime; + unsigned i; + + if (file) { + fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + name = s; + s = strtoken(s); + size = s; + s = strtoken(s); + mtime = s; + s = strtoken(s); + sub = s; + + ret = stru64(size, &v_size); + if (ret != 0) { + fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + ret = stru64(mtime, &v_mtime); + if (ret != 0) { + fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + if (!*sub) { + fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + /* find the disk */ + for(i=0;idiskarr);++i) { + disk = tommy_array_get(&state->diskarr, i); + if (strcmp(disk->name, name) == 0) + break; + } + if (i == tommy_array_size(&state->diskarr)) { + fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", name, path, line); + exit(EXIT_FAILURE); + } + + /* allocate the file */ + file = file_alloc(state->block_size, sub, v_size, v_mtime); + + /* insert the file in the file containers */ + tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub)); + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); + + /* start the block allocation of the file */ + block_index = 0; + + /* check for empty file */ + if (block_index == file->blockmax) { + file = 0; + disk = 0; + } + + /* stat */ + ++count_file; + } else if (strcmp(tag, "blk") == 0) { + char* pos; + char* hash; + pos_t v_pos; + struct snapraid_block* block; + char* e; + + if (!file) { + fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + pos = s; + s = strtoken(s); + hash = s; + + ret = stru(pos, &v_pos); + if (ret != 0) { + fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + if (block_index >= file->blockmax) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + block = &file->blockvec[block_index]; + + if (block->parity_pos != POS_INVALID) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + block->parity_pos = v_pos; + + /* set the hash only if present */ + if (*hash != 0) { + e = strdechex(block->hash, HASH_MAX, hash); + if (e) { + fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + block->is_hashed = 1; + } + + /* insert the block in the block array */ + tommy_array_grow(&disk->blockarr, block->parity_pos + 1); + tommy_array_set(&disk->blockarr, block->parity_pos, block); + + /* check for termination of the block list */ + ++block_index; + if (block_index == file->blockmax) { + file = 0; + disk = 0; + } + + /* stat */ + ++count_block; + } else { + fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line); + exit(EXIT_FAILURE); + } + } + + if (file) { + fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + fclose(f); + + if (state->verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + } +} + +/****************************************************************************/ +/* write */ + +/** + * Write the new state. + */ +void state_write(struct snapraid_state* state) +{ + FILE* f; + char path[PATH_MAX]; + unsigned i; + unsigned count_file; + unsigned count_block; + + count_file = 0; + count_block = 0; + + printf("Saving state...\n"); + + snprintf(path, sizeof(path), "%s.tmp", state->content); + f = fopen(path, "w"); + if (!f) { + fprintf(stderr, "Error opening for writing the state file '%s'\n", path); + exit(EXIT_FAILURE); + } + + /* for each disk */ + for(i=0;idiskarr);++i) { + tommy_node* j; + struct snapraid_disk* disk = tommy_array_get(&state->diskarr, i); + + /* for each file */ + for(j=disk->filelist;j!=0;j=j->next) { + pos_t k; + struct snapraid_file* file = j->data; + int ret; + + ret = fprintf(f,"file %s %lld %ld %s\n", disk->name, file->size, file->mtime, file->sub); + if (ret < 0) { + fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + /* for each block */ + for(k=0;kblockmax;++k) { + struct snapraid_block* block = &file->blockvec[k]; + + if (block->is_hashed) { + char s_hash[HASH_MAX*2+1]; + strenchex(s_hash, block->hash, HASH_MAX); + s_hash[HASH_MAX*2] = 0; + ret = fprintf(f, "blk %u %s\n", block->parity_pos, s_hash); + } else { + ret = fprintf(f, "blk %u\n", block->parity_pos); + } + if (ret < 0) { + fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ++count_block; + } + + ++count_file; + } + } + + /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */ + /* than even in a system crash event we have one valid copy of the file. */ + + if (fflush(f) != 0) { + fprintf(stderr, "Error writing the state file '%s', in fflush(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (fsync(fileno(f)) != 0) { + fprintf(stderr, "Error writing the state file '%s' in fsync(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (fclose(f) != 0) { + fprintf(stderr, "Error writing the state file '%s' in close(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (rename(path, state->content) != 0) { + fprintf(stderr, "Error renaming the state file '%s' to '%s' in rename(). %s.\n", path, state->content, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (state->verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + } +} + +/****************************************************************************/ +/* scan */ + +struct snapraid_scan { + unsigned count_equal; + unsigned count_change; + unsigned count_remove; + unsigned count_insert; +}; + +/** + * Remove the specified file from the data set. + */ +static void state_scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + pos_t i; + + /* free all the blocks of the file */ + for(i=0;iblockmax;++i) { + pos_t block_pos = file->blockvec[i].parity_pos; + + /* adjust the first free position */ + if (disk->first_free_block > block_pos) + disk->first_free_block = block_pos; + + tommy_array_set(&disk->blockarr, block_pos, 0); + } + + /* remove the file from the file containers */ + tommy_hashdyn_remove_existing(&disk->fileset, &file->nodeset); + tommy_list_remove_existing(&disk->filelist, &file->nodelist); + + /* deallocate */ + file_free(file); +} + +/** + * Insert the specified file in the data set. + */ +static void state_scan_file_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + pos_t i; + pos_t block_max; + pos_t block_pos; + + /* allocate the blocks of the file */ + block_pos = disk->first_free_block; + block_max = tommy_array_size(&disk->blockarr); + for(i=0;iblockmax;++i) { + /* find a free block */ + while (block_pos < block_max && tommy_array_get(&disk->blockarr, block_pos) != 0) + ++block_pos; + + /* if not found, allocate a new one */ + if (block_pos == block_max) { + ++block_max; + tommy_array_grow(&disk->blockarr, block_max); + } + + /* set it */ + file->blockvec[i].parity_pos = block_pos; + tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]); + } + if (file->blockmax) { + /* set the new free position, but only if allocated something */ + disk->first_free_block = block_pos + 1; + } + + /* insert the file in the file containers */ + tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub)); + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); +} + +static void state_scan_file(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* path, const char* sub) +{ + struct stat st; + struct snapraid_file* file; + + if (stat(path, &st) != 0) { + fprintf(stderr, "Error in stat file '%s'\n", path); + exit(EXIT_FAILURE); + } + + /* check if the file already exists */ + file = tommy_hashdyn_search(&disk->fileset, file_compare, sub, file_hash(sub)); + if (file) { + /* check if the file is the same */ + if (file->size == st.st_size && file->mtime == st.st_mtime) { + /* mark as present */ + ++scan->count_equal; + file->is_present = 1; + return; + } else { + /* remove and reinsert it */ + ++scan->count_change; + --scan->count_insert; + state_scan_file_remove(state, disk, file); + /* continue to insert it */ + } + } + + /* create the new file */ + ++scan->count_insert; + file = file_alloc(state->block_size, sub, st.st_size, st.st_mtime); + file->is_present = 1; + + /* insert it */ + state_scan_file_insert(state, disk, file); +} + +static void state_scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* dir, const char* sub) +{ + DIR* d; + struct dirent* dd; + + d = opendir(dir); + if (!d) { + fprintf(stderr, "Error accessing directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + } + + while ((dd = readdir(d)) != 0) { + if (dd->d_type == DT_REG) { + char path_next[PATH_MAX]; + char sub_next[PATH_MAX]; + snprintf(path_next, sizeof(path_next), "%s%s", dir, dd->d_name); + snprintf(sub_next, sizeof(sub_next), "%s%s", sub, dd->d_name); + state_scan_file(scan, state, disk, path_next, sub_next); + } else if (dd->d_type == DT_DIR) { + if (strcmp(dd->d_name, ".") != 0 && strcmp(dd->d_name, "..") != 0 && strcmp(dd->d_name, "lost+found") != 0 + ) { + char dir_next[PATH_MAX]; + char sub_next[PATH_MAX]; + snprintf(dir_next, sizeof(dir_next), "%s%s/", dir, dd->d_name); + snprintf(sub_next, sizeof(sub_next), "%s%s/", sub, dd->d_name); + state_scan_dir(scan, state, disk, dir_next, sub_next); + } + } else { + if (state->verbose) { + printf("warning: File '%s/%s' not processed\n", dir, dd->d_name); + } + } + } + + closedir(d); +} + +/** + * Update the internal state to represent the new state. + */ +void state_scan(struct snapraid_state* state) +{ + unsigned disk_count = tommy_array_size(&state->diskarr); + unsigned i; + struct snapraid_scan scan; + + scan.count_equal = 0; + scan.count_change = 0; + scan.count_remove = 0; + scan.count_insert = 0; + + for(i=0;idiskarr, i); + tommy_node* node; + + printf("Scanning disk %s...\n", disk->name); + + state_scan_dir(&scan, state, disk, disk->dir, ""); + + /* check for removed files */ + node = disk->filelist; + while (node) { + struct snapraid_file* file = node->data; + + /* next node */ + node = node->next; + + /* remove if not present */ + if (!file->is_present) { + state_scan_file_remove(state, disk, file); + ++scan.count_remove; + } + } + } + + if (state->verbose) { + printf("\tequal %d\n", scan.count_equal); + printf("\tchanged %d\n", scan.count_change); + printf("\tremoved %d\n", scan.count_remove); + printf("\tadded %d\n", scan.count_insert); + } +} + +/****************************************************************************/ +/* common */ + +/** + * Computes the size of the parity data in number of blocks. + */ +static pos_t state_sync_resize(struct snapraid_state* state) +{ + unsigned disk_count = tommy_array_size(&state->diskarr); + pos_t parity_block; + unsigned i; + + /* compute the size of the parity file */ + parity_block = 0; + for(i=0;idiskarr, i); + + /* start from the declared size */ + pos_t block = tommy_array_size(&disk->blockarr); + + /* decrease the block until an allocated block */ + while (block > 0 && tommy_array_get(&disk->blockarr, block - 1) == 0) + --block; + + if (i == 0 || parity_block < block) + parity_block = block; + } + + return parity_block; +} + +static void state_xor_block(unsigned char* xor, const unsigned char* block, unsigned size) +{ + while (size >= 16) { + uint32_t* xor32 = (uint32_t*)xor; + const uint32_t* block32 = (const uint32_t*)block; + xor32[0] ^= block32[0]; + xor32[1] ^= block32[1]; + xor32[2] ^= block32[2]; + xor32[3] ^= block32[3]; + + xor += 16; + block += 16; + size -= 16; + } + + while (size >= 4) { + uint32_t* xor32 = (uint32_t*)xor; + const uint32_t* block32 = (const uint32_t*)block; + xor32[0] ^= block32[0]; + + xor += 4; + block += 4; + size -= 4; + } + + while (size != 0) { + xor[0] ^= block[0]; + + xor += 1; + block += 1; + size -= 1; + } +} + +struct snapraid_sync_block { + char path[PATH_MAX]; /**< Path of the file. */ + struct snapraid_disk* disk; /**< Disk of the file. */ + struct snapraid_file* file; /**< File opened. */ + int f; /**< Handle of the file. */ + off_t offset; /**< Position in the file. */ +}; + +static int state_sync_read_block(int ret_on_error, struct snapraid_sync_block* sync, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size) +{ + ssize_t read_ret; + off_t offset; + unsigned read_size; + + if (sync->file != block->file) { + if (sync->f != -1) { + int ret; + ret = close(sync->f); + if (ret != 0) { + /* here we always abort, as close is supposed to never fail */ + fprintf(stderr, "Error closing file '%s'. %s.\n", sync->file->sub, strerror(errno)); + exit(EXIT_FAILURE); + } + } + + /* invalidate for error */ + sync->file = 0; + sync->f = -1; + + snprintf(sync->path, sizeof(sync->path), "%s%s", sync->disk->dir, block->file->sub); + sync->f = open(sync->path, O_RDONLY | O_BINARY); + if (sync->f == -1) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error opening file '%s'. %s.\n", sync->path, strerror(errno)); + exit(EXIT_FAILURE); + } + + sync->file = block->file; + sync->offset = 0; + } + + offset = block_file_pos(block) * (off_t)block_size; + + if (sync->offset != offset) { + off_t ret; + + /* invalidate for error */ + sync->offset = -1; + + ret = lseek(sync->f, offset, SEEK_SET); + if (ret != offset) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error seeking file '%s'. %s.\n", sync->path, strerror(errno)); + exit(EXIT_FAILURE); + } + sync->offset = offset; + } + + read_size = block_file_size(block, block_size); + + read_ret = read(sync->f, block_buffer, read_size); + if (read_ret != read_size) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error reading file '%s'. %s.\n", sync->path, strerror(errno)); + exit(EXIT_FAILURE); + } + + sync->offset += read_size; + + return read_size; +} + +static void state_sync_write_parity(const char* parity_path, int parity_f, off_t* parity_offset, pos_t pos, unsigned char* block_buffer, unsigned block_size) +{ + ssize_t write_ret; + off_t offset; + + offset = pos * (off_t)block_size; + + if (*parity_offset != offset) { + off_t ret; + ret = lseek(parity_f, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "Error seeking file '%s'. %s.\n", parity_path, strerror(errno)); + exit(EXIT_FAILURE); + } + *parity_offset = offset; + } + + write_ret = write(parity_f, block_buffer, block_size); + if (write_ret != block_size) { + if (errno == ENOSPC) { + fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", parity_path); + } else { + fprintf(stderr, "Error writing file '%s'. %s.\n", parity_path, strerror(errno)); + } + exit(EXIT_FAILURE); + } + + *parity_offset += block_size; +} + +static int state_sync_read_parity(int ret_on_error, const char* parity_path, int parity_f, off_t* parity_offset, pos_t pos, unsigned char* block_buffer, unsigned block_size) +{ + ssize_t read_ret; + off_t offset; + + offset = pos * (off_t)block_size; + + if (*parity_offset != offset) { + off_t ret; + + /* invalidate for error */ + *parity_offset = -1; + + ret = lseek(parity_f, offset, SEEK_SET); + if (ret != offset) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error seeking file '%s'. %s.\n", parity_path, strerror(errno)); + exit(EXIT_FAILURE); + } + *parity_offset = offset; + } + + read_ret = read(parity_f, block_buffer, block_size); + if (read_ret != block_size) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error reading file '%s'. %s.\n", parity_path, strerror(errno)); + exit(EXIT_FAILURE); + } + + *parity_offset += block_size; + + return block_size; +} + +/****************************************************************************/ +/* sync */ + +static void state_sync_process(struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax) +{ + struct snapraid_sync_block* sync; + off_t parity_offset; + unsigned diskmax = tommy_array_size(&state->diskarr); + pos_t i; + unsigned char* block_buffer; + unsigned char* xor_buffer; + int ret; + uint64_t count_size; + time_t start; + time_t last; + + block_buffer = malloc_nofail(state->block_size); + xor_buffer = malloc_nofail(state->block_size); + + sync = malloc_nofail(diskmax * sizeof(struct snapraid_sync_block)); + for(i=0;idiskarr, i); + sync[i].file = 0; + sync[i].f = -1; + } + parity_offset = -1; + + start = time(0); + last = start; + count_size = 0; + + for(i=blockstart;iis_hashed) { + unhashed = 1; + break; + } + } + + if (!unhashed) + continue; + + /* start with 0 */ + memset(xor_buffer, 0, state->block_size); + + /* for each disk, process the block */ + for(j=0;jblock_size); + + /* now compute the hash */ + md5_init(&md5); + md5_update(&md5, block_buffer, read_size); + md5_final(&md5, hash); + + if (block->is_hashed) { + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_MAX) != 0) { + fprintf(stderr, "Data error for file %s at position %u\n", block->file->sub, block_file_pos(block)); + fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u fix'\n", block_file_pos(block)); + exit(EXIT_FAILURE); + } + } else { + /* copy the hash */ + memcpy(block->hash, hash, HASH_MAX); + + /* mark the block as hashed */ + block->is_hashed = 1; + } + + /* compute the parity */ + state_xor_block(xor_buffer, block_buffer, read_size); + + count_size += read_size; + } + + /* write the parity */ + state_sync_write_parity(state->parity, parity_f, &parity_offset, i, xor_buffer, state->block_size); + + /* progress */ + now = time(0); + if (last != now) { + printf("%u%%, %u MB", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + if (now != start) { + printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / (now - start))); + } + printf("\r"); + fflush(stdout); + last = now; + } + + /* stop if requested */ + if (global_interrupt) { + printf("\rStopping at block %u\n", i); + break; + } + } + + printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + + printf("Done\n"); + + for(i=0;isub, strerror(errno)); + exit(EXIT_FAILURE); + } + } + } + + free(sync); + free(block_buffer); + free(xor_buffer); +} + +/** + * Update the parity file. + */ +void state_sync(struct snapraid_state* state, pos_t blockstart) +{ + char path[PATH_MAX]; + pos_t blockmax; + off_t size; + int ret; + int f; + struct stat st; + + printf("Syncing...\n"); + + blockmax = state_sync_resize(state); + size = blockmax * (off_t)state->block_size; + + if (blockstart >= blockmax) { + fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax); + exit(EXIT_FAILURE); + } + + snprintf(path, sizeof(path), "%s", state->parity); + f = open(path, O_RDWR | O_CREAT, 0600); + if (f == -1) { + fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ret = fstat(f, &st); + if (ret != 0) { + fprintf(stderr, "Error accessing parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (st.st_size < size) { +#if HAVE_FALLOCATE + /* allocate real space. This is Linux specific. */ + ret = fallocate(f, 0, 0, size); +#else + /* allocate using a sparse file */ + ret = ftruncate(f, size); +#endif + if (ret != 0) { + if (errno == ENOSPC) { + fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", path); + } else { + fprintf(stderr, "Error growing parity file '%s'. %s.\n", path, strerror(errno)); + } + exit(EXIT_FAILURE); + } + } else { + ret = ftruncate(f, size); + if (ret != 0) { + fprintf(stderr, "Error truncating parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + } + + /* ensure that the file size change is written to disk */ + /* this is not really required, but just to avoid to result in a wrong sized file */ + /* in case of a system crash */ + ret = fsync(f); + if (ret != 0) { + fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + state_sync_process(state, f, blockstart, blockmax); + + /* ensure that data changes are written to disk */ + /* this is required to ensure that parity is more updated than content */ + /* in case of a system crash */ + ret = fsync(f); + if (ret != 0) { + fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ret = close(f); + if (ret != 0) { + fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } +} + +/****************************************************************************/ +/* check */ + + +static int state_check_process(int ret_on_error, struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax) +{ + struct snapraid_sync_block* sync; + off_t parity_offset; + unsigned diskmax = tommy_array_size(&state->diskarr); + pos_t i; + unsigned char* block_buffer; + unsigned char* xor_buffer; + int ret; + uint64_t count_size; + time_t start; + time_t last; + unsigned error; + unsigned unrecoverable_error; + + block_buffer = malloc_nofail(state->block_size); + xor_buffer = malloc_nofail(state->block_size); + + sync = malloc_nofail(diskmax * sizeof(struct snapraid_sync_block)); + for(i=0;idiskarr, i); + sync[i].file = 0; + sync[i].f = -1; + } + parity_offset = -1; + error = 0; + unrecoverable_error = 0; + + start = time(0); + last = start; + count_size = 0; + + for(i=blockstart;iis_hashed) { + hashed = 1; + break; + } + } + + /* if there is at least an hashed block we have to check */ + if (!hashed) + continue; + + /* start with 0 */ + memset(xor_buffer, 0, state->block_size); + + /* for each disk, process the block */ + unhashed = 0; + failed = 0; + for(j=0;jis_hashed) { + unhashed = 1; + continue; + } + + read_size = state_sync_read_block(-1, &sync[j], block, block_buffer, state->block_size); + + if (read_size == -1) { + ++failed; + + /* save the hash for the parity check */ + memcpy(failed_hash, block->hash, HASH_MAX); + failed_size = block_file_size(block, state->block_size); + + fprintf(stderr, "Read error for file %s at position %u\n", block->file->sub, block_file_pos(block)); + ++error; + } else { + /* now compute the hash */ + md5_init(&md5); + md5_update(&md5, block_buffer, read_size); + md5_final(&md5, hash); + + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_MAX) != 0) { + ++failed; + fprintf(stderr, "Data error for file %s at position %u\n", block->file->sub, block_file_pos(block)); + ++error; + } + + /* compute the parity */ + state_xor_block(xor_buffer, block_buffer, read_size); + + count_size += read_size; + } + } + + /* only if no unhashed block, we can check for parity */ + if (unhashed) + continue; + + if (failed == 0) { + int ret; + + /* read the parity */ + ret = state_sync_read_parity(-1, state->parity, parity_f, &parity_offset, i, block_buffer, state->block_size); + if (ret == -1) { + fprintf(stderr, "Parity read error for position %u\n", i); + ++error; + } else { + /* compare it */ + if (memcmp(xor_buffer, block_buffer, state->block_size) != 0) { + fprintf(stderr, "Parity data error for position %u\n", i); + ++error; + } + } + } else if (failed == 1) { + struct md5_t md5; + unsigned char hash[HASH_MAX]; + int ret; + + /* read the parity */ + ret = state_sync_read_parity(-1, state->parity, parity_f, &parity_offset, i, block_buffer, state->block_size); + if (ret == -1) { + fprintf(stderr, "UNRECOVERABLE parity read error for position %u\n", i); + ++unrecoverable_error; + } + + /* compute the failed block */ + state_xor_block(block_buffer, xor_buffer, state->block_size); + + /* now compute the hash */ + md5_init(&md5); + md5_update(&md5, block_buffer, failed_size); + md5_final(&md5, hash); + + /* compare the hash */ + if (memcmp(hash, failed_hash, HASH_MAX) != 0) { + fprintf(stderr, "UNRECOVERABLE parity data error for position %u\n", i); + ++unrecoverable_error; + } + } else { + fprintf(stderr, "UNRECOVERABLE parity missing for position %u\n", i); + ++unrecoverable_error; + } + + /* progress */ + now = time(0); + if (last != now) { + printf("%u%%, %u MB", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + if (now != start) { + printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / (now - start))); + } + printf("\r"); + fflush(stdout); + last = now; + } + + /* stop if requested */ + if (global_interrupt) { + printf("\rStopping at block %u\n", i); + break; + } + } + + printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + + if (error || unrecoverable_error) { + if (error) + printf("%u recoverable errors\n", error); + else + printf("No recoverable errors\n"); + if (unrecoverable_error) + printf("%u UNRECOVERABLE errors\n", unrecoverable_error); + else + printf("No unrecoverable errors\n"); + } else { + printf("No error\n"); + } + + for(i=0;isub, strerror(errno)); + exit(EXIT_FAILURE); + } + } + } + + free(sync); + free(block_buffer); + free(xor_buffer); + + return unrecoverable_error != 0 ? ret_on_error : 0; +} + + +/** + * Check. + */ +void state_check(struct snapraid_state* state, pos_t blockstart) +{ + char path[PATH_MAX]; + pos_t blockmax; + off_t size; + int ret; + int f; + + printf("Checking...\n"); + + blockmax = state_sync_resize(state); + size = blockmax * (off_t)state->block_size; + + if (blockstart >= blockmax) { + fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax); + exit(EXIT_FAILURE); + } + + snprintf(path, sizeof(path), "%s", state->parity); + f = open(path, O_RDONLY); + if (f == -1) { + fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ret = state_check_process(-1, state, f, blockstart, blockmax); + if (ret == -1) { + /* no extra message */ + exit(EXIT_FAILURE); + } + + ret = close(f); + if (ret != 0) { + fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } +} + +/****************************************************************************/ +/* main */ + +void version(void) { + printf(PACKAGE " v" VERSION " by Andrea Mazzoleni\n"); +} + +void usage(void) { + version(); + + printf("Usage: snapraid [options]\n"); + printf("\n"); + printf("Options:\n"); + printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file (default /etc/snapraid.conf)\n"); + printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); + printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n"); + printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n"); +} + +#if HAVE_GETOPT_LONG +struct option long_options[] = { + { "conf", 1, 0, 'c' }, + { "start", 1, 0, 's' }, + { "verbose", 0, 0, 'v' }, + { "help", 0, 0, 'h' }, + { "version", 0, 0, 'V' }, + { 0, 0, 0, 0 } +}; +#endif + +#define OPTIONS "c:s:vhV" + +void signal_handler(int signal) +{ + switch (signal) { + case SIGINT : + global_interrupt = 1; + break; + } +} + +#define OPERATION_SYNC 0 +#define OPERATION_CHECK 1 +#define OPERATION_FIX 2 + +int main(int argc, char* argv[]) +{ + int c; + int verbose = 0; + const char* conf; + struct snapraid_state state; + int operation; + pos_t blockstart; + + /* defaults */ + conf = "/etc/snapraid.conf"; + verbose = 0; + blockstart = 0; + + opterr = 0; + while ((c = +#if HAVE_GETOPT_LONG + getopt_long(argc, argv, OPTIONS, long_options, 0)) +#else + getopt(argc, argv, OPTIONS)) +#endif + != EOF) { + switch (c) { + case 'c' : + conf = optarg; + break; + case 's' : + if (stru(optarg, &blockstart) != 0) { + fprintf(stderr, "Invalid start position '%s'\n", optarg); + exit(EXIT_FAILURE); + } + break; + case 'v' : + verbose = 1; + break; + case 'h' : + usage(); + exit(EXIT_SUCCESS); + case 'V' : + version(); + exit(EXIT_SUCCESS); + default: + fprintf(stderr, "Unknown option '%c'\n", (char)c); + exit(EXIT_FAILURE); + } + } + + if (optind + 1 != argc) { + usage(); + exit(EXIT_FAILURE); + } + + if (strcmp(argv[optind], "sync") == 0) { + operation = OPERATION_SYNC; + } else if (strcmp(argv[optind], "check") == 0) { + operation = OPERATION_CHECK; + } else if (strcmp(argv[optind], "fix") == 0) { + operation = OPERATION_FIX; + } else { + fprintf(stderr, "Unknown command '%s'\n", argv[optind]); + exit(EXIT_FAILURE); + } + + struct sigaction sig; + sig.sa_handler = signal_handler; + sigemptyset(&sig.sa_mask); + sig.sa_flags = 0; + sigaction(SIGINT, &sig, 0); + + state_init(&state); + + state_config(&state, conf, verbose); + + if (operation == OPERATION_SYNC) { + state_read(&state); + + state_scan(&state); + + state_sync(&state, blockstart); + + state_write(&state); + } else if (operation == OPERATION_CHECK) { + state_read(&state); + + state_check(&state, blockstart); + } + + state_done(&state); + + return EXIT_SUCCESS; +} + diff --git a/test.conf b/test.conf new file mode 100644 index 00000000..8c02358f --- /dev/null +++ b/test.conf @@ -0,0 +1,7 @@ +block_size 1 +parity test/parity +content test/content +disk d1 test/disk1/ +disk d2 test/disk2/ +disk d3 test/disk3/ + diff --git a/testgen.c b/testgen.c new file mode 100644 index 00000000..e7b6b790 --- /dev/null +++ b/testgen.c @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +void gen(int disk, int size) +{ + char path[PATH_MAX]; + char name[PATH_MAX]; + FILE* f; + int i; + int l; + + l = 1 + rand() % 32; + + for(i=0;i. + */ + +#include "portable.h" + +/* redefine a safe malloc for tommy use */ +#define tommy_malloc malloc_nofail +#define tommy_free free + +void* malloc_nofail(size_t size); + +/* direct include tommy source */ +#include "tommylist.c" +#include "tommyhash.c" +#include "tommyhashdyn.c" +#include "tommyarray.c" + diff --git a/tommyarray.c b/tommyarray.c new file mode 100644 index 00000000..9fac6e9f --- /dev/null +++ b/tommyarray.c @@ -0,0 +1,72 @@ +/* + * Copyright 2011 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommyarray.h" + +#include /* for memset */ + +/******************************************************************************/ +/* array */ + +void tommy_array_init(tommy_array* array) +{ + /* fixed initial size */ + array->bucket_bit = TOMMY_ARRAY_BIT; + array->bucket_max = 1 << array->bucket_bit; + array->bucket[0] = tommy_cast(void**, tommy_malloc(array->bucket_max * sizeof(void*))); + memset(array->bucket[0], 0, array->bucket_max * sizeof(void*)); + array->bucket_mac = 1; + array->size = 0; +} + +void tommy_array_done(tommy_array* array) +{ + unsigned i; + for(i=0;ibucket_mac;++i) + tommy_free(array->bucket[i]); +} + +void tommy_array_grow(tommy_array* array, unsigned size) +{ + while (size > array->bucket_max) { + /* grow the hash size and allocate */ + array->bucket[array->bucket_mac] = tommy_cast(void**, tommy_malloc(array->bucket_max * sizeof(void*))); + memset(array->bucket[array->bucket_mac], 0, array->bucket_max * sizeof(void*)); + ++array->bucket_mac; + ++array->bucket_bit; + array->bucket_max = 1 << array->bucket_bit; + } + + if (array->size < size) + array->size = size; +} + +tommy_size_t tommy_array_memory_usage(tommy_array* array) +{ + return array->bucket_max * (tommy_size_t)sizeof(array->bucket[0][0]); +} + diff --git a/tommyarray.h b/tommyarray.h new file mode 100644 index 00000000..82b2e288 --- /dev/null +++ b/tommyarray.h @@ -0,0 +1,158 @@ +/* + * Copyright 2011 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Dynamic array. + * + * This array is able to grow dynamically upon request. + * + * The resize involve an allocation of a new array segment, without reallocating + * the already allocated memory, and then not increasing the heap fragmentation. + * + * This means that the address of the allocated segments never change. + */ + +#ifndef __TOMMYARRAY_H +#define __TOMMYARRAY_H + +#include "tommytypes.h" + +#include /* for assert */ + +/******************************************************************************/ +/* array */ + +/** + * Initial and minimal size of the array expressed as a power of 2. + * The initial size is 2^TOMMY_ARRAY_BIT. + */ +#define TOMMY_ARRAY_BIT 6 + +/** \internal + * Max number of elements as a power of 2. + */ +#define TOMMY_ARRAY_BIT_MAX 32 + +/** + * Array. + */ +typedef struct tommy_array_struct { + void** bucket[TOMMY_ARRAY_BIT_MAX]; /**< Dynamic array of buckets. */ + unsigned bucket_bit; /**< Bits used in the bit mask. */ + unsigned bucket_max; /**< Number of buckets. */ + unsigned bucket_mac; /**< Number of vectors allocated. */ + unsigned size; /**< Currently allocated and initialized size. */ +} tommy_array; + +/** + * Initializes the array. + */ +void tommy_array_init(tommy_array* array); + +/** + * Deinitializes the array. + */ +void tommy_array_done(tommy_array* array); + +/** + * Grow the size up to the specified value. + * All the new elements in the array are initialized with the 0 value. + */ +void tommy_array_grow(tommy_array* array, unsigned size); + +/** + * Gets a reference of the element at the specified position. + * You must be sure that space for this position is already + * allocated calling tommy_array_grow(). + */ +tommy_inline void** tommy_array_ref(tommy_array* array, unsigned pos) +{ + unsigned bsr; + + assert(pos < array->size); + + /* special case for the first bucket */ + if (pos < (1 << TOMMY_ARRAY_BIT)) { + return &array->bucket[0][pos]; + } + + /* get the highest bit set */ + bsr = tommy_ilog2_u32(pos); + + /* clear the highest bit */ + pos -= 1 << bsr; + + return &array->bucket[bsr - TOMMY_ARRAY_BIT + 1][pos]; +} + +/** + * Sets the element at the specified position. + * You must be sure that space for this position is already + * allocated calling tommy_array_grow(). + */ +tommy_inline void tommy_array_set(tommy_array* array, unsigned pos, void* node) +{ + *tommy_array_ref(array, pos) = node; +} + +/** + * Gets the element at the specified position. + * You must be sure that space for this position is already + * allocated calling tommy_array_grow(). + */ +tommy_inline void* tommy_array_get(tommy_array* array, unsigned pos) +{ + return *tommy_array_ref(array, pos); +} + +/** + * Grows and inserts a new element at the end of the array. + */ +tommy_inline void tommy_array_insert(tommy_array* array, void* node) +{ + unsigned pos = array->size; + + tommy_array_grow(array, pos + 1); + + tommy_array_set(array, pos, node); +} + +/** + * Gets the initialized size of the array. + */ +tommy_inline unsigned tommy_array_size(tommy_array* array) +{ + return array->size; +} + +/** + * Gets the size of allocated memory. + */ +tommy_size_t tommy_array_memory_usage(tommy_array* array); + +#endif + diff --git a/tommychain.h b/tommychain.h new file mode 100644 index 00000000..bc4a436d --- /dev/null +++ b/tommychain.h @@ -0,0 +1,224 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Chain of nodes. + * A chain of nodes is an abstraction used to implements complex list operations + * like sorting. + * + * Do not use this directly. Use lists instead. + */ + +#ifndef __TOMMYCHAIN_H +#define __TOMMYCHAIN_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* chain */ + +/** + * Chain of nodes. + * A chain of nodes is a sequence of nodes with the following properties: + * - It contains at least one node. A chains of zero nodes cannot exist. + * - The next field of the tail is of *undefined* value. + * - The prev field of the head is of *undefined* value. + * - All the other inner prev and next fields are correctly set. + */ +typedef struct tommy_chain_struct { + tommy_node* head; /**< Pointer at the head of the chain. */ + tommy_node* tail; /**< Pointer at the tail of the chain. */ +} tommy_chain; + +/** + * Splices a chain in the middle of another chain. + */ +tommy_inline void tommy_chain_splice(tommy_node* first_before, tommy_node* first_after, tommy_node* second_head, tommy_node* second_tail) +{ + /* set the prev list */ + first_after->prev = second_tail; + second_head->prev = first_before; + + /* set the next list */ + first_before->next = second_head; + second_tail->next = first_after; +} + +/** + * Concats two chains. + */ +tommy_inline void tommy_chain_concat(tommy_node* first_tail, tommy_node* second_head) +{ + /* set the prev list */ + second_head->prev = first_tail; + + /* set the next list */ + first_tail->next = second_head; +} + +/** + * Merges two chains. + */ +tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) +{ + tommy_node* first_i = first->head; + tommy_node* second_i = second->head; + + /* merge */ + while (1) { + if (cmp(first_i->data, second_i->data) > 0) { + tommy_node* next = second_i->next; + if (first_i == first->head) { + tommy_chain_concat(second_i, first_i); + first->head = second_i; + } else { + tommy_chain_splice(first_i->prev, first_i, second_i, second_i); + } + if (second_i == second->tail) + break; + second_i = next; + } else { + if (first_i == first->tail) { + tommy_chain_concat(first_i, second_i); + first->tail = second->tail; + break; + } + first_i = first_i->next; + } + } +} + +/** + * Merges two chains managing special degenerated cases. + * It's funtionally equivalent at tommy_chain_merge() but faster with already ordered chains. + */ +tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) +{ + /* identify the condition first <= second */ + if (cmp(first->tail->data, second->head->data) <= 0) { + tommy_chain_concat(first->tail, second->head); + first->tail = second->tail; + return; + } + + /* identify the condition second < first */ + /* here we must be strict on comparison to keep the sort stable */ + if (cmp(second->tail->data, first->head->data) < 0) { + tommy_chain_concat(second->tail, first->head); + first->head = second->head; + return; + } + + tommy_chain_merge(first, second, cmp); +} + +/** + * Max number of elements as a power of 2. + */ +#define TOMMY_CHAIN_BIT_MAX 32 + +/** + * Sorts a chain. + * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, + * similar at the one used in the SGI STL libraries and in the Linux Kernel, + * but faster on degenerated cases like already ordered lists. + * + * SGI STL stl_list.h + * http://www.sgi.com/tech/stl/stl_list.h + * + * Linux Kernel lib/list_sort.c + * http://lxr.linux.no/#linux+v2.6.36/lib/list_sort.c + */ +tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* cmp) +{ + /* + * Bit buckets of chains. + * Each bucket contains 2^i nodes or it's empty. + */ + tommy_chain bit[TOMMY_CHAIN_BIT_MAX]; + + /** + * Value stored inside the bit bucket. + * It's used to know which bucket is empty of full. + */ + unsigned counter; + tommy_node* node = chain->head; + tommy_node* tail = chain->tail; + unsigned mask; + unsigned i; + + counter = 0; + while (1) { + tommy_node* next; + tommy_chain carry; + tommy_chain* last; + + /* carry bit to add */ + last = &carry; + carry.head = node; + carry.tail = node; + next = node->next; + + /* add the bit, propagating the carry */ + i = 0; + mask = counter; + while ((mask & 1) != 0) { + tommy_chain_merge_degenerated(&bit[i], last, cmp); + mask >>= 1; + last = &bit[i]; + ++i; + } + + /* copy the carry in the first empty bit */ + bit[i] = *last; + + /* add the carry in the counter */ + ++counter; + + if (node == tail) + break; + node = next; + } + + /* merge the buckets */ + i = tommy_ctz_u32(counter); + mask = counter >> i; + while (mask != 1) { + mask >>= 1; + if (mask & 1) { + tommy_chain_merge_degenerated(&bit[i+1], &bit[i], cmp); + } else { + bit[i+1] = bit[i]; + } + ++i; + } + + *chain = bit[i]; +} + +#endif + diff --git a/tommyhash.c b/tommyhash.c new file mode 100644 index 00000000..77431417 --- /dev/null +++ b/tommyhash.c @@ -0,0 +1,167 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommyhash.h" + +/******************************************************************************/ +/* hash */ + +tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr) +{ + /* allow unaligned read on Intel x86 and x86_64 platforms */ +#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || defined(__x86_64__) || defined(_M_X64) + /* defines from http://predef.sourceforge.net/ */ + return *(tommy_uint32_t*)ptr; +#else + const unsigned char* ptr8 = tommy_cast(const unsigned char*, ptr); + return ptr8[0] + ((tommy_uint32_t)ptr8[1] << 8) + ((tommy_uint32_t)ptr8[2] << 16) + ((tommy_uint32_t)ptr8[3] << 24); +#endif +} + +#define tommy_rot(x,k) \ + (((x)<<(k)) | ((x)>>(32-(k)))) + +#define tommy_mix(a,b,c) \ + do { \ + a -= c; a ^= tommy_rot(c, 4); c += b; \ + b -= a; b ^= tommy_rot(a, 6); a += c; \ + c -= b; c ^= tommy_rot(b, 8); b += a; \ + a -= c; a ^= tommy_rot(c,16); c += b; \ + b -= a; b ^= tommy_rot(a,19); a += c; \ + c -= b; c ^= tommy_rot(b, 4); b += a; \ + } while (0) + +#define tommy_final(a,b,c) \ + do { \ + c ^= b; c -= tommy_rot(b,14); \ + a ^= c; a -= tommy_rot(c,11); \ + b ^= a; b -= tommy_rot(a,25); \ + c ^= b; c -= tommy_rot(b,16); \ + a ^= c; a -= tommy_rot(c,4); \ + b ^= a; b -= tommy_rot(a,14); \ + c ^= b; c -= tommy_rot(b,24); \ + } while (0) + +tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len) +{ + const unsigned char* key = tommy_cast(const unsigned char*,void_key); + tommy_uint32_t a, b, c; + + a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val; + + while (key_len > 12) { + a += tommy_le_uint32_read(key + 0); + b += tommy_le_uint32_read(key + 4); + c += tommy_le_uint32_read(key + 8); + + tommy_mix(a,b,c); + + key_len -= 12; + key += 12; + } + + switch (key_len) { + case 0 : + return c; /* used only when called with a zero length */ + case 12 : + c += tommy_le_uint32_read(key + 8); + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; + case 10 : c += ((tommy_uint32_t)key[9]) << 8; + case 9 : c += key[8]; + case 8 : + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; + case 6 : b += ((tommy_uint32_t)key[5]) << 8; + case 5 : b += key[4]; + case 4 : + a += tommy_le_uint32_read(key + 0); + break; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; + case 2 : a += ((tommy_uint32_t)key[1]) << 8; + case 1 : a += key[0]; + } + + tommy_final(a,b,c); + + return c; +} + +tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len) +{ + const unsigned char* key = tommy_cast(const unsigned char*,void_key); + tommy_uint32_t a, b, c; + + a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff); + c += init_val >> 32; + + while (key_len > 12) { + a += tommy_le_uint32_read(key + 0); + b += tommy_le_uint32_read(key + 4); + c += tommy_le_uint32_read(key + 8); + + tommy_mix(a,b,c); + + key_len -= 12; + key += 12; + } + + switch (key_len) { + case 0 : + return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */ + case 12 : + c += tommy_le_uint32_read(key + 8); + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; + case 10 : c += ((tommy_uint32_t)key[9]) << 8; + case 9 : c += key[8]; + case 8 : + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; + case 6 : b += ((tommy_uint32_t)key[5]) << 8; + case 5 : b += key[4]; + case 4 : + a += tommy_le_uint32_read(key + 0); + break; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; + case 2 : a += ((tommy_uint32_t)key[1]) << 8; + case 1 : a += key[0]; + } + + tommy_final(a,b,c); + + return c + ((tommy_uint64_t)b << 32); +} + diff --git a/tommyhash.h b/tommyhash.h new file mode 100644 index 00000000..d584923e --- /dev/null +++ b/tommyhash.h @@ -0,0 +1,104 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. + */ + +#ifndef __TOMMYHASH_H +#define __TOMMYHASH_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* hash */ + +/** + * Hash type used in hashtables. + */ +typedef tommy_key_t tommy_hash_t; + +/** + * Hash function. + * Robert Jenkins "lookup3" hash, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). + * \note + * This function is platform independent as it read always in little endian format. + * \param init_val Initial value. You can use this to concatenate hash. Otherwise set it at 0. + * \param void_key Key to hash. + * \param key_len Length of the key. + * \return The 32 bit hash value. + */ +tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); + +/** + * Hash function for 64 bits. + * Robert Jenkins "lookup3" hash, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2(). + * \note + * This function is platform independent as it read always in little endian format. + */ +tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); + +/** + * Integer hash of 32 bits. + * Robert Jenkins "4-byte Integer Hashing", + * from http://burtleburtle.net/bob/hash/integer.html + */ +tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) +{ + key -= key << 6; + key ^= key >> 17; + key -= key << 9; + key ^= key << 4; + key -= key << 3; + key ^= key << 10; + key ^= key >> 15; + + return key; +} + +/** + * Integer hash of 64 bits. + * Thomas Wang "Integer Hash Function", + * from http://www.cris.com/~Ttwang/tech/inthash.htm + */ +tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) +{ + key = ~key + (key << 21); + key = key ^ (key >> 24); + key = key + (key << 3) + (key << 8); + key = key ^ (key >> 14); + key = key + (key << 2) + (key << 4); + key = key ^ (key >> 28); + key = key + (key << 31); + + return key; +} + +#endif + diff --git a/tommyhashdyn.c b/tommyhashdyn.c new file mode 100644 index 00000000..3e6bee06 --- /dev/null +++ b/tommyhashdyn.c @@ -0,0 +1,174 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommyhashdyn.h" +#include "tommylist.h" + +#include /* for memset */ + +/******************************************************************************/ +/* hashdyn */ + +void tommy_hashdyn_init(tommy_hashdyn* hashdyn) +{ + /* fixed initial size */ + hashdyn->bucket_bit = TOMMY_HASHDYN_BIT; + hashdyn->bucket_max = 1 << hashdyn->bucket_bit; + hashdyn->bucket_mask = hashdyn->bucket_max - 1; + hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(hashdyn->bucket_max * sizeof(tommy_hashdyn_node*))); + memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); + + hashdyn->count = 0; +} + +void tommy_hashdyn_done(tommy_hashdyn* hashdyn) +{ + tommy_free(hashdyn->bucket); +} + +/** + * Resize the bucket vector. + */ +static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, unsigned new_bucket_bit) +{ + unsigned bucket_bit; + unsigned bucket_max; + unsigned new_bucket_max; + unsigned new_bucket_mask; + tommy_hashdyn_node** new_bucket; + + bucket_bit = hashdyn->bucket_bit; + bucket_max = hashdyn->bucket_max; + + new_bucket_max = 1 << new_bucket_bit; + new_bucket_mask = new_bucket_max - 1; + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); + + /* reinsert all the elements */ + if (new_bucket_bit > hashdyn->bucket_bit) { + unsigned i; + + /* grow */ + for(i=0;ibucket[i]; + while (j) { + tommy_hashdyn_node* j_next = j->next; + unsigned index = j->key & new_bucket_mask; + if (new_bucket[index]) + tommy_list_insert_tail_not_empty(new_bucket[index], j); + else + tommy_list_insert_first(&new_bucket[index], j); + j = j_next; + } + } + } else { + unsigned i; + + /* shrink */ + for(i=0;ibucket[i]; + + /* concat the upper bucket */ + tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); + } + } + + tommy_free(hashdyn->bucket); + + /* setup */ + hashdyn->bucket_bit = new_bucket_bit; + hashdyn->bucket_max = new_bucket_max; + hashdyn->bucket_mask = new_bucket_mask; + hashdyn->bucket = new_bucket; +} + +void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) +{ + unsigned pos = hash & hashdyn->bucket_mask; + + tommy_list_insert_tail(&hashdyn->bucket[pos], node, data); + + node->key = hash; + + ++hashdyn->count; + + /* grow if more than 50% full */ + if (hashdyn->count >= hashdyn->bucket_max / 2) { + tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); + } +} + +void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) +{ + unsigned pos = node->key & hashdyn->bucket_mask; + + tommy_list_remove_existing(&hashdyn->bucket[pos], node); + + --hashdyn->count; + + /* shrink if less than 12.5% full */ + if (hashdyn->count <= hashdyn->bucket_max / 8 && hashdyn->bucket_bit > TOMMY_HASHDYN_BIT) { + tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); + } + + return node->data; +} + +void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_compare_func* cmp, const void* cmp_arg, tommy_hash_t hash) +{ + unsigned pos = hash % hashdyn->bucket_max; + tommy_hashdyn_node* i = hashdyn->bucket[pos]; + + while (i) { + /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ + if (i->key == hash && cmp(cmp_arg, i->data) == 0) { + tommy_list_remove_existing(&hashdyn->bucket[pos], i); + + --hashdyn->count; + + return i->data; + } + i = i->next; + } + + return 0; +} + +tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn) +{ + return hashdyn->bucket_max * (tommy_size_t)sizeof(hashdyn->bucket[0]) + + tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node); +} + diff --git a/tommyhashdyn.h b/tommyhashdyn.h new file mode 100644 index 00000000..b22175e9 --- /dev/null +++ b/tommyhashdyn.h @@ -0,0 +1,250 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Dynamic chained hashtable. + * + * This hashtable resizes dynamically. It starts with the minimal size of 16 buckets, it doubles + * the size then it reaches a load factor greater than 0.5 and it halves the size with a load + * factor lower than 0.125. + * + * All the elements are reallocated in a single resize operation done inside + * tommy_hashdyn_insert() or tommy_hashdyn_remove(). + * + * Note that the resize operation takes approximatively 100 [ms] with 1 million of elements, + * and 1 [second] with 10 millions. This could be a problem in real-time applications. + * + * The resize also fragment the heap, as it involves allocating a double-sized table, copy elements, + * and deallocating the older table. Leaving a big hole in the heap. + * + * The ::tommy_hashlin hashtable fixes both problems. + * + * To initialize the hashtable you have to call tommy_hashdyn_init(). + * + * \code + * tommy_hashslin hashdyn; + * + * tommy_hashdyn_init(&hashdyn); + * \endcode + * + * To insert elements in the hashtable you have to call tommy_hashdyn_insert() for + * each element. + * In the insertion call you have to specify the address of the node, the + * address of the object, and the hash value of the key to use. + * The address of the object is used to initialize the tommy_node::data field + * of the node, and the hash to initialize the tommy_node::key field. + * + * \code + * struct object { + * tommy_node node; + * // other fields + * int value; + * }; + * + * struct object* obj = malloc(sizeof(struct object)); // creates the object + * + * obj->value = ...; // initializes the object + * + * tommy_hashdyn_insert(&hashdyn, &obj->node, obj, tommy_inthash_u32(obj->value)); // inserts the object + * \endcode + * + * To find and element in the hashtable you have to call tommy_hashtable_search() + * providing a comparison function, its argument, and the hash of the key to search. + * + * \code + * int compare(const void* arg, const void* obj) + * { + * return (*(const unsigned*)arg != ((const struct object*)obj)->value; + * } + * + * struct object* obj = tommy_hashdyn_search(&hashdyn, compare, &value_to_find, tommy_inthash_u32(value_to_find)); + * if (!obj) { + * // not found + * } else { + * // found + * } + * \endcode + * + * To iterate over all the elements in the hashtable with the same key, you have to + * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL. + * You have also to check explicitely for the key, as the bucket may contains + * different keys. + * + * \code + * tommy_node* i = tommy_hashdyn_bucket(&hashdyn, tommy_inthash_u32(value_to_find)); + * while (i) { + * struct object* obj = i->data; // gets the object pointer + * + * if (obj->value == value_to_find) { + * printf("%d\n", obj->value); // process the object + * } + * + * i = i->next; // goes to the next element + * } + * \endcode + * + * To remove an element from the hashtable you have to call tommy_hashdyn_remove() + * providing a comparison function, its argument, and the hash of the key to search + * and remove. + * + * \code + * struct object* obj = tommy_trie_remove(&hashtable, compare, &value_to_remove, tommy_inthash_u32(value_to_remove)); + * if (obj) { + * free(obj); // frees the object allocated memory + * } + * \endcode + * + * To destroy the hashtable you have to remove all the elements, and deinitialize + * the hashtable calling tommy_hashdyn_done(). + * + * \code + * tommy_hashdyn_done(&hashdyn); + * \endcode + * + * Note that you cannot iterates over all the elements in the hashtable using the + * hashtable itself. You have to insert all the elements also in a ::tommy_list, + * and use the list to iterate. See the \ref multiindex example for more detail. + */ + +#ifndef __TOMMYHASHDYN_H +#define __TOMMYHASHDYN_H + +#include "tommyhash.h" + +/******************************************************************************/ +/* hashdyn */ + +/** \internal + * Initial and minimal size of the hashtable expressed as a power of 2. + * The initial size is 2^TOMMY_HASHDYN_BIT. + */ +#define TOMMY_HASHDYN_BIT 4 + +/** + * Dynamic hashtable node. + * This is the node that you have to include inside your objects. + */ +typedef tommy_node tommy_hashdyn_node; + +/** + * Dynamic chained hashtable. + */ +typedef struct tommy_hashdyn_struct { + tommy_hashdyn_node** bucket; /**< Hash buckets. One list for each hash modulus. */ + unsigned bucket_bit; /**< Bits used in the bit mask. */ + unsigned bucket_max; /**< Number of buckets. */ + unsigned bucket_mask; /**< Bit mask to access the buckets. */ + unsigned count; /**< Number of elements. */ +} tommy_hashdyn; + +/** + * Initializes the hashtable. + */ +void tommy_hashdyn_init(tommy_hashdyn* hashdyn); + +/** + * Deinitializes the hashtable. + */ +void tommy_hashdyn_done(tommy_hashdyn* hashdyn); + +/** + * Inserts an element in the hashtable. + */ +void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash); + +/** + * Searches and removes an element from the hashtable. + * You have to provide a compare function and the hash of the element you want to remove. + * If the element is not found, 0 is returned. + * If more equal elements are present, the first one is removed. + * This operation is faster than calling tommy_hashdyn_bucket() and tommy_hashdyn_remove_existing() separately. + * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. + * The function should return 0 for equal elements, anything other for different elements. + * \param cmp_arg Compare argument passed as first argument of the compare function. + * \param hash Hash of the element to find and remove. + * \return The removed element, or 0 if not found. + */ +void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_compare_func* cmp, const void* cmp_arg, tommy_hash_t hash); + +/** + * Gets the bucket of the specified hash. + * The bucket is guaranteed to contain ALL the elements with the specified hash, + * but it can contain also others. + * You can access elements in the bucket following the ::next pointer until 0. + * \param hash Hash of the element to find. + * \return The head of the bucket, or 0 if empty. + */ +tommy_inline tommy_hashdyn_node* tommy_hashdyn_bucket(tommy_hashdyn* hashdyn, tommy_hash_t hash) +{ + return hashdyn->bucket[hash & hashdyn->bucket_mask]; +} + +/** + * Searches an element in the hashtable. + * You have to provide a compare function and the hash of the element you want to find. + * If more equal elements are present, the first one is returned. + * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. + * The function should return 0 for equal elements, anything other for different elements. + * \param cmp_arg Compare argument passed as first argument of the compare function. + * \param hash Hash of the element to find. + * \return The first element found, or 0 if none. + */ +tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_compare_func* cmp, const void* cmp_arg, tommy_hash_t hash) +{ + tommy_hashdyn_node* i = tommy_hashdyn_bucket(hashdyn, hash); + while (i) { + /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ + if (i->key == hash && cmp(cmp_arg, i->data) == 0) + return i->data; + i = i->next; + } + return 0; +} + +/** + * Removes an element from the hashtable. + * You must already have the address of the element to remove. + * \return The tommy_node::data field of the node removed. + */ +void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); + +/** + * Gets the number of elements. + */ +tommy_inline unsigned tommy_hashdyn_count(tommy_hashdyn* hashdyn) +{ + return hashdyn->count; +} + +/** + * Gets the size of allocated memory. + * It includes the size of the ::tommy_trie_node of the stored elements. + */ +tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn); + +#endif + diff --git a/tommylist.c b/tommylist.c new file mode 100644 index 00000000..0b0125b8 --- /dev/null +++ b/tommylist.c @@ -0,0 +1,87 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommylist.h" +#include "tommychain.h" + +void tommy_list_concat(tommy_list* first, tommy_list* second) +{ + tommy_node* first_head; + tommy_node* first_tail; + tommy_node* second_head; + + if (tommy_list_empty(second)) { + return; + } + + if (tommy_list_empty(first)) { + *first = *second; + return; + } + + first_head = tommy_list_head(first); + second_head = tommy_list_head(second); + first_tail = tommy_list_tail(first); + + /* set the "circular" prev list */ + first_head->prev = second_head->prev; + second_head->prev = first_tail; + + /* set the "0 terminated" next list */ + first_tail->next = second_head; +} + +/** \internal + * Setup a list. + */ +tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) +{ + head->prev = tail; + tail->next = 0; + *list = head; +} + +void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp) +{ + tommy_chain chain; + tommy_node* head; + + if (tommy_list_empty(list)) + return; + + head = tommy_list_head(list); + + /* create a chain from the list */ + chain.head = head; + chain.tail = head->prev; + + tommy_chain_mergesort(&chain, cmp); + + /* restore the list */ + tommy_list_set(list, chain.head, chain.tail); +} + diff --git a/tommylist.h b/tommylist.h new file mode 100644 index 00000000..a90483ff --- /dev/null +++ b/tommylist.h @@ -0,0 +1,309 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Double linked list for collisions into hashtables. + * + * This list is a double linked list mainly targetted for collisions into an hashtables, + * but useable also as a generic list. + * + * The main feature of this list is to require only one pointer to represent the + * list, compared to a classic implementation requiring an head an a tail pointers. + * This reduces the memory usage in hashtables. + * + * Another feature is to support the insertion at the end of the list. This allow to store + * collisions in a stable order. Where for stable order we mean that equal elements keep + * their insertion order. + * + * To initialize the list, you have to call tommy_list_init(), or simply assign + * to it NULL, as an empty list is represented by the NULL value. + * + * \code + * tommy_list list; + * + * tommy_list_init(&list); // initializes the list + * \endcode + * + * To insert elements in the list you have to call tommy_list_insert_tail() + * or tommy_list_insert_head() for each element. + * In the insertion call you have to specify the address of the node and the + * address of the object. + * The address of the object is used to initialize the tommy_node::data field + * of the node. + * + * \code + * struct object { + * tommy_node node; + * // other fields + * int value; + * }; + * + * struct object* obj = malloc(sizeof(struct object)); // creates the object + * + * obj->value = ...; // initializes the object + * + * tommy_list_insert_tail(&list, &obj->node, obj); // inserts the object + * \endcode + * + * To iterates over all the elements in the list you have to call + * tommy_list_head() to get the head of the list and follow the + * tommy_node::next pointer until NULL. + * + * \code + * tommy_node* i = tommy_list_head(&list); + * while (i) { + * struct object* obj = i->data; // gets the object pointer + * + * printf("%d\n", obj->value); // process the object + * + * i = i->next; // go to the next element + * } + * \endcode + * + * To destroy the list you have only to remove all the elements, as the list is + * completely inplace and it doesn't allocate memory. + * + * \code + * tommy_node* i = tommy_list_head(&list); + * while (i) { + * tommy_node* i_next = i->next; // saves the next element before freeing + * + * free(i->data); // frees the object allocated memory + * + * i = i_next; // goes to the next element + * } + * \endcode + */ + +#ifndef __TOMMYLIST_H +#define __TOMMYLIST_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* list */ + +/** + * Double linked list for collisions into hashtables. + */ +typedef tommy_node* tommy_list; + +/** + * Initializes the list. + * The list is completely inplace, so it doesn't need to be deinitialized. + */ +tommy_inline void tommy_list_init(tommy_list* list) +{ + *list = 0; +} + +/** + * Gets the head of the list. + * \return The head node. For empty lists 0 is returned. + */ +tommy_inline tommy_node* tommy_list_head(tommy_list* list) +{ + return *list; +} + +/** + * Gets the tail of the list. + * \return The tail node. For empty lists 0 is returned. + */ +tommy_inline tommy_node* tommy_list_tail(tommy_list* list) +{ + tommy_node* head = tommy_list_head(list); + + if (!head) + return 0; + + return head->prev; +} + +/** \internal + * Creates a new list with a single element. + * \param list The list to initialize. + * \param node The node to insert. + */ +tommy_inline void tommy_list_insert_first(tommy_list* list, tommy_node* node) +{ + /* one element "circular" prev list */ + node->prev = node; + + /* one element "0 terminated" next list */ + node->next = 0; + + *list = node; +} + +/** \internal + * Inserts an element at the head of a not empty list. + * The element is inserted at the head of the list. The list cannot be empty. + * \param list The list. The list cannot be empty. + * \param node The node to insert. + */ +tommy_inline void tommy_list_insert_head_not_empty(tommy_list* list, tommy_node* node) +{ + tommy_node* head = tommy_list_head(list); + + /* insert in the "circular" prev list */ + node->prev = head->prev; + head->prev = node; + + /* insert in the "0 terminated" next list */ + node->next = head; + + *list = node; +} + +/** \internal + * Inserts an element at the tail of a not empty list. + * The element is inserted at the tail of the list. The list cannot be empty. + * \param head The node at the list head. It cannot be 0. + * \param node The node to insert. + */ +tommy_inline void tommy_list_insert_tail_not_empty(tommy_node* head, tommy_node* node) +{ + /* insert in the "circular" prev list */ + node->prev = head->prev; + head->prev = node; + + /* insert in the "0 terminated" next list */ + node->next = 0; + node->prev->next = node; +} + +/** + * Inserts an element at the head of a list. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + */ +tommy_inline void tommy_list_insert_head(tommy_list* list, tommy_node* node, void* data) +{ + tommy_node* head = tommy_list_head(list); + + if (head) { + tommy_list_insert_head_not_empty(list, node); + } else { + tommy_list_insert_first(list, node); + } + + node->data = data; +} + +/** + * Inserts an element at the tail of a list. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + */ +tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, void* data) +{ + tommy_node* head = tommy_list_head(list); + + if (head) { + tommy_list_insert_tail_not_empty(head, node); + } else { + tommy_list_insert_first(list, node); + } + + node->data = data; +} + +/** \internal + * Removes an element from the head of a not empty list. + * \param list The list. The list cannot be empty. + * \return The node removed. + */ +tommy_inline tommy_node* tommy_list_remove_head_not_empty(tommy_list* list) +{ + tommy_node* head = tommy_list_head(list); + + /* remove from the "circular" prev list */ + head->next->prev = head->prev; + + /* remove from the "0 terminated" next list */ + *list = head->next; /* the new head, in case 0 */ + + return head; +} + +/** + * Removes an element from the list. + * You must already have the address of the element to remove. + * \note The node content is left unchanged, including the tommy_node::next + * and tommy_node::prev fields that still contain pointers at the list. + * \param node The node to remove. The node must be in the list. + * \return The tommy_node::data field of the node removed. + */ +tommy_inline void* tommy_list_remove_existing(tommy_list* list, tommy_node* node) +{ + tommy_node* head = tommy_list_head(list); + + /* remove from the "circular" prev list */ + if (node->next) { + node->next->prev = node->prev; + } else { + head->prev = node->prev; /* the last */ + } + + /* remove from the "0 terminated" next list */ + if (head == node) { + *list = node->next; /* the new head, in case 0 */ + } else { + node->prev->next = node->next; + } + + return node->data; +} + +/** + * Concats two lists. + * The second list is concatenated at the first list. + * \param first The first list. + * \param second The second list. After this call the list content is undefined, + * and you should not use it anymore. + */ +void tommy_list_concat(tommy_list* first, tommy_list* second); + +/** + * Sorts a list. + * It's a stable merge sort with O(N*log(N)) worst complexity. + * It's faster on degenerated cases like partially ordered lists. + */ +void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp); + +/** + * Checks if empty. + * \return If the list is empty. + */ +tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list) +{ + return tommy_list_head(list) == 0; +} + +#endif + diff --git a/tommytypes.h b/tommytypes.h new file mode 100644 index 00000000..79d6ef52 --- /dev/null +++ b/tommytypes.h @@ -0,0 +1,313 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Generic types. + */ + +#ifndef __TOMMYTYPES_H +#define __TOMMYTYPES_H + +/******************************************************************************/ +/* types */ + +#if defined(_MSC_VER) +#include +typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */ +typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */ +typedef size_t tommy_size_t; /**< Generic size_t type. */ +typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */ +#else +#include +typedef uint32_t tommy_uint32_t; /**< Generic uint32_t type. */ +typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */ +typedef uintptr_t tommy_size_t; /**< Generic size_t type. */ +typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */ +#endif +typedef int tommy_bool_t; /**< Generic boolean type. */ + +/** \internal + * Type cast required for the C++ compilation. + * When compiling in C++ we cannot convert a void* pointer to another pointer. + * In such case we need an explicit cast. + */ +#ifdef __cplusplus +#define tommy_cast(type, value) static_cast(value) +#else +#define tommy_cast(type, value) (value) +#endif + +/******************************************************************************/ +/* heap */ + +/* by default uses malloc/realloc/free */ + +/** + * Generic malloc(), realloc() and free() functions. + * Redefine them to what you need. By default they map to the C malloc(), realloc() and free(). + */ +#if !defined(tommy_malloc) && !defined(tommy_realloc) && !defined(tommy_free) +#include +#define tommy_malloc malloc +#define tommy_realloc realloc +#define tommy_free free +#endif + +/******************************************************************************/ +/* modificators */ + +/** \internal + * Definition of the inline keyword if available. + */ +#if !defined(tommy_inline) +#if __STDC_VERSION__ >= 199901L || defined(__cplusplus) +#define tommy_inline inline +#elif defined(_MSC_VER) || defined(__GNUC__) +#define tommy_inline static __inline +#else +#define tommy_inline static +#endif +#endif + +/** \internal + * Definition of the always_inline keyword if available. + */ +#if !defined(tommy_always_inline) +#if defined(__GNUC__) +#define tommy_always_inline static __attribute__((always_inline)) __inline +#else +#define tommy_always_inline tommy_inline +#endif +#endif + +/** \internal + * Definition of the restrict keyword if available. + */ +#if !defined(tommy_restrict) +#if __STDC_VERSION__ >= 199901L +#define tommy_restrict restrict +#elif defined(_MSC_VER) || defined(__GNUC__) +#define tommy_restrict __restrict +#else +#define tommy_restrict +#endif +#endif + +/** \internal + * Hints the compiler that a condition is likely true. + */ +#if !defined(tommy_likely) +#if defined(__GNUC__) +#define tommy_likely(x) __builtin_expect(!!(x),1) +#else +#define tommy_likely(x) (x) +#endif +#endif + +/** \internal + * Hints the compiler that a condition is likely false. + */ +#if !defined(tommy_unlikely) +#if defined(__GNUC__) +#define tommy_unlikely(x) __builtin_expect(!!(x),0) +#else +#define tommy_unlikely(x) (x) +#endif +#endif + +/******************************************************************************/ +/* key */ + +/** + * Key type used in indexed data structures to store the key or the hash value. + */ +typedef tommy_uint32_t tommy_key_t; + +/** + * Bits into the ::tommy_key_t type. + */ +#define TOMMY_KEY_BIT (sizeof(tommy_key_t) * 8) + +/******************************************************************************/ +/* node */ + +/** + * Data structure node. + * This node type is shared between all the data structures and used to store some + * info directly into the objects you want to store. + * + * A typical declaration is: + * \code + * struct object { + * tommy_node node; + * // other fields + * }; + * \endcode + */ +typedef struct tommy_node_struct { + /** + * Next node. + * The tail node has it at 0, like a 0 terminated list. + */ + struct tommy_node_struct* next; + + /** + * Previous node. + * The head node points to the tail node, like a circular list. + */ + struct tommy_node_struct* prev; + + /** + * Pointer at the object containing the node. + * This field is initialized when inserting nodes into a data structure. + */ + void* data; + + /** + * Key used to store the node. + * With hashtables this field is used to store the hash value. + * With lists this field is not used. + */ + tommy_key_t key; +} tommy_node; + +/******************************************************************************/ +/* compare */ + +/** + * Compare function for elements. + */ +typedef int tommy_compare_func(const void* void_a, const void* void_b); + +/******************************************************************************/ +/* bit hacks */ + +#if defined(_MSC_VER) && !defined(__cplusplus) +#include +#pragma intrinsic(_BitScanReverse) +#endif + +/** \internal + * Integer log2 for constants. + * You can use it only for exact power of 2 up to 256. + */ +#define TOMMY_ILOG2(value) ((value) == 256 ? 8 : (value) == 128 ? 7 :(value) == 64 ? 6 : (value) == 32 ? 5 : (value) == 16 ? 4 : (value) == 8 ? 3 : (value) == 4 ? 2 : (value) == 2 ? 1 : 0) + +/** + * Bit scan reverse or integer log2. + * Return the bit index of the most significant 1 bit. + * + * If no bit is set, the result is undefined. + * To force a return 0 in this case, you can use tommy_ilog2(value | 1). + * + * Other interesting ways for bitscan can be found at: + * + * Bit Twiddling Hacks + * http://graphics.stanford.edu/~seander/bithacks.html + * + * Chess Programming BitScan + * http://chessprogramming.wikispaces.com/BitScan + * + * \param value Value to scan. 0 is not allowed. + * \return The index of the most significan bit set. + */ +tommy_inline unsigned tommy_ilog2_u32(tommy_uint32_t value) +{ +#if defined(_MSC_VER) + unsigned long count; + _BitScanReverse(&count, value); + return count; +#elif defined(__GNUC__) + return 31 - __builtin_clz(value); +#else + /* Find the log base 2 of an N-bit integer in O(lg(N)) operations with multiply and lookup */ + /* from http://graphics.stanford.edu/~seander/bithacks.html */ + static const int TOMMY_DE_BRUIJN_INDEX_ILOG2[32] = { + 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 + }; + + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + + return TOMMY_DE_BRUIJN_INDEX_ILOG2[(tommy_uint32_t)(value * 0x07C4ACDDU) >> 27]; +#endif +} + +/** + * Bit scan forward or trailing zero count. + * Return the bit index of the least significant 1 bit. + * + * If no bit is set, the result is undefined. + * \param value Value to scan. 0 is not allowed. + * \return The index of the least significan bit set. + */ +tommy_inline unsigned tommy_ctz_u32(tommy_uint32_t value) +{ +#if defined(_MSC_VER) + unsigned long count; + _BitScanForward(&count, value); + return count; +#elif defined(__GNUC__) + return __builtin_ctz(value); +#else + /* Count the consecutive zero bits (trailing) on the right with multiply and lookup */ + /* from http://graphics.stanford.edu/~seander/bithacks.html */ + static const tommy_uint32_t TOMMY_DE_BRUIJN_INDEX_CTZ[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; + + return TOMMY_DE_BRUIJN_INDEX_CTZ[(tommy_uint32_t)(((value & -value) * 0x077CB531U)) >> 27]; +#endif +} + +/** + * Rounds up to the next power of 2. + * For the value 0, the result is undefined. + * \return The smallest power of 2 not less than the specified value. + */ +tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) +{ + /* Round up to the next highest power of 2 */ + /* from http://www-graphics.stanford.edu/~seander/bithacks.html */ + + --value; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + ++value; + + return value; +} +#endif + diff --git a/util.h b/util.h new file mode 100644 index 00000000..e59e14e0 --- /dev/null +++ b/util.h @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/****************************************************************************/ +/* string */ + +static inline char* strtoken(char* s) +{ + while (*s && *s != ' ') + ++s; + if (*s) + *s++ = 0; + while (*s == ' ') + ++s; + return s; +} + +static inline char* strskip(char* s) +{ + while (*s == ' ') + ++s; + return s; +} + +static inline int strgets(char* s, unsigned size, FILE* f) +{ + char* i = s; + char* send = s + size; + int c; + + while (1) { + c = fgetc_unlocked(f); + + if (c == EOF || c == '\n') + break; + + *i++ = c; + + if (i == send) { + return -1; + } + } + + if (c == EOF) { + if (ferror_unlocked(f)) { + return -1; + } + if (i == s) + return 0; + } + + /* remove ending spaces */ + while (i != s && isspace(i[-1])) + --i; + *i = 0; + + return 1; +} + +static inline int stru(const char* s, unsigned* value) +{ + unsigned v; + + if (!*s) + return -1; + + v = 0; + while (*s>='0' && *s<='9') { + v *= 10; + v += *s - '0'; + ++s; + } + + if (*s) + return -1; + + *value = v; + + return 0; +} + +static inline int stru64(const char* s, uint64_t* value) +{ + uint64_t v; + + if (!*s) + return -1; + + v = 0; + while (*s>='0' && *s<='9') { + v *= 10; + v += *s - '0'; + ++s; + } + + if (*s) + return -1; + + *value = v; + + return 0; +} + +static char strhexset[16] = "0123456789abcdef"; + +static inline void strenchex(char* str, const void* void_data, unsigned data_len) +{ + const unsigned char* data = void_data; + unsigned i; + + for(i=0;i> 4]; + *str++ = strhexset[b & 0xF]; + } +} + +static inline char* strdechex(void* void_data, unsigned data_len, char* str) +{ + unsigned char* data = void_data; + unsigned i; + + for(i=0;i= 'A' && c0 <= 'F') + b0 = c0 - 'A' + 10; + else if (c0 >= 'a' && c0 <= 'f') + b0 = c0 - 'a' + 10; + else if (c0 >= '0' && c0 <= '9') + b0 = c0 - '0'; + else + return str; + ++str; + + c1 = *str; + if (c1 >= 'A' && c1 <= 'F') + b1 = c1 - 'A' + 10; + else if (c1 >= 'a' && c1 <= 'f') + b1 = c1 - 'a' + 10; + else if (c1 >= '0' && c1 <= '9') + b1 = c1 - '0'; + else + return str; + ++str; + + data[i] = (b0 << 4) | b1; + } + + return 0; +} +