diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 00000000..4bbaa0bb --- /dev/null +++ b/Makefile.am @@ -0,0 +1,83 @@ +bin_PROGRAMS = snapraid +check_PROGRAMS = testgen + +snapraid_SOURCES = \ + snapraid.c \ + getopt.c \ + md5.c \ + tommy.c + + +noinst_HEADERS = \ + portable.h \ + md5.h \ + tommyarray.h \ + tommychain.h \ + tommyhash.h \ + tommyhashdyn.h \ + tommylist.h \ + tommytypes.h \ + util.h \ + tommyarray.c \ + tommyhash.c \ + tommyhashdyn.c \ + tommylist.c + +testgen_SOURCES = \ + testgen.c + +EXTRA_DIST = \ + autogen.sh \ + noautomake.sh \ + README AUTHORS HISTORY INSTALL COPYING \ + test.conf \ + acinclude.m4 + +clean-local: + rm -f valgrind.log callgrind.log + rm -rf test + +CHECK = ./snapraid -v -c test.conf sync + +check-local: + rm -rf test + mkdir test + mkdir test/disk1 + mkdir test/disk2 + mkdir test/disk3 + ./testgen 1 3 2000 + ./snapraid -v -c $(srcdir)/test.conf sync + ./snapraid -v -c $(srcdir)/test.conf check + ./testgen 2 3 2000 + ./snapraid -v -c $(srcdir)/test.conf sync + ./snapraid -v -c $(srcdir)/test.conf check + rm test/disk1/a* + ./snapraid -v -c $(srcdir)/test.conf check + echo Success! + +valgrind: + valgrind \ + --tool=memcheck \ + --track-origins=yes \ + --read-var-info=yes \ + --leak-check=full \ + -v $(CHECK) \ + 2> valgrind.log + tail valgrind.log + +callgrind: + valgrind \ + --tool=callgrind \ + --dump-instr=yes \ + --trace-jump=yes \ + -v $(CHECK) \ + 2> callgrind.log + tail callgrind.log + +cachegrind: + valgrind \ + --tool=cachegrind \ + -v $(CHECK) \ + 2> cachegrind.log + tail cachegrind.log + diff --git a/README b/README new file mode 100644 index 00000000..0c6a1a6d --- /dev/null +++ b/README @@ -0,0 +1,5 @@ +SNAPRAID +======== + + + diff --git a/acinclude.m4 b/acinclude.m4 new file mode 100644 index 00000000..ceed823a --- /dev/null +++ b/acinclude.m4 @@ -0,0 +1,20 @@ +dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno) +dnl +dnl Shows a message as like "checking wether gcc accepts flag ... no" +dnl and executess ifyes or ifno. + +AC_DEFUN([AC_CHECK_CC_OPT], +[ +AC_MSG_CHECKING([whether ${CC-cc} accepts $1]) +echo 'void f(){}' > conftest.c +if test -z "`${CC-cc} -c $1 conftest.c 2>&1`"; then + AC_MSG_RESULT([yes]) + $2 +else + AC_MSG_RESULT([no]) + $3 +fi +rm -f conftest* +]) + + diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000..68fcf2ef --- /dev/null +++ b/autogen.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# +echo "Generating build information using aclocal, automake and autoconf" +echo "This may take a while ..." + +# Touch the timestamps on all the files since CVS messes them up +touch configure.ac + +# Regenerate configuration files +aclocal +automake --add-missing --force-missing +autoconf +autoheader && touch config.h + +# Run configure for this platform +echo "Now you are ready to run ./configure" + diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..8b44cd3e --- /dev/null +++ b/configure.ac @@ -0,0 +1,47 @@ +dnl Process this file with autoconf to produce a configure script. +AC_INIT([snapraid], [0.1]) +AM_INIT_AUTOMAKE([foreign no-dependencies]) +AC_CONFIG_SRCDIR([snapraid.c]) +AC_CONFIG_HEADER([config.h]) +AC_CANONICAL_HOST +AC_COPYRIGHT([Copyright 2011 Andrea Mazzoleni]) + +dnl Checks for programs. +AC_PROG_CC + +dnl Checks for libraries. +AC_SYS_LARGEFILE + +dnl Checks for header files. +AC_HEADER_STDC +AC_HEADER_DIRENT +AC_HEADER_TIME +AC_FUNC_UTIME_NULL +AC_CHECK_HEADERS([unistd.h getopt.h stdint.h]) + +dnl Checks for typedefs, structures, and compiler characteristics. +AC_C_CONST +AC_C_INLINE +AC_C_RESTRICT +AC_C_VOLATILE +AC_TYPE_SIZE_T + +dnl Checks for library functions. +AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf]) +AC_CHECK_FUNCS([fsync fallocate]) + +dnl Configure for 64 bit files +CFLAGS="$CFLAGS -D_FILE_OFFSET_BITS=64" + +dnl Enable GLIBC symbols +AC_DEFINE(_GNU_SOURCE,1,[Enable GLIBC symbols]) + +dnl Checks for architecture +AC_C_BIGENDIAN + +dnl Checks for compiler +AC_CHECK_CC_OPT([-Wall], CFLAGS="$CFLAGS -Wall", []) + +AC_CONFIG_FILES([Makefile]) +AC_OUTPUT + diff --git a/getopt.c b/getopt.c new file mode 100644 index 00000000..b8c070aa --- /dev/null +++ b/getopt.c @@ -0,0 +1,113 @@ +/* + * This file is part of the Advance project. + * + * Copyright (C) 2001, 2002 Andrea Mazzoleni + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#if HAVE_CONFIG_H +#include +#endif + +#if !HAVE_GETOPT + +/* This source is extracted from the DJGPP LIBC library */ + +#define unconst(var, type) ((type)(var)) + +/* Copyright (C) 1995 DJ Delorie, see COPYING.DJ for details */ +#include +#include + +int opterr = 1, optind = 1, optopt = 0; +char *optarg = 0; + +#define BADCH (int)'?' +#define EMSG "" + +int getopt(int nargc, char *const nargv[], const char *ostr) +{ + static const char *place = EMSG; /* option letter processing */ + char *oli; /* option letter list index */ + char *p; + + if (!*place) + { + if (optind >= nargc || *(place = nargv[optind]) != '-') + { + place = EMSG; + return(EOF); + } + if (place[1] && *++place == '-') + { + ++optind; + place = EMSG; + return(EOF); + } + } + + if ((optopt = (int)*place++) == (int)':' + || !(oli = strchr(ostr, optopt))) + { + /* + * if the user didn't specify '-' as an option, + * assume it means EOF. + */ + if (optopt == (int)'-') + return EOF; + if (!*place) + ++optind; + if (opterr) + { + if (!(p = strrchr(*nargv, '/'))) + p = *nargv; + else + ++p; + fprintf(stderr, "%s: illegal option -- %c\n", p, optopt); + } + return BADCH; + } + if (*++oli != ':') + { /* don't need argument */ + optarg = NULL; + if (!*place) + ++optind; + } + else + { /* need an argument */ + if (*place) /* no white space */ + optarg = unconst(place, char *); + else if (nargc <= ++optind) + { /* no arg */ + place = EMSG; + if (!(p = strrchr(*nargv, '/'))) + p = *nargv; + else + ++p; + if (opterr) + fprintf(stderr, "%s: option requires an argument -- %c\n", p, optopt); + return BADCH; + } + else /* white space */ + optarg = nargv[optind]; + place = EMSG; + ++optind; + } + return optopt; /* dump back option letter */ +} + +#endif + diff --git a/md5.c b/md5.c new file mode 100644 index 00000000..fcad6ae1 --- /dev/null +++ b/md5.c @@ -0,0 +1,246 @@ +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + * + * Changed so as no longer to depend on Colin Plumb's `usual.h' header + * definitions; now uses stuff from dpkg's config.h. + * - Ian Jackson . + * Still in the public domain. + * + * Added convenience MD5(buffer,length,digest) function. + * - Gordon Mohr + * Still in the public domain. + */ + +#include "portable.h" + +#include "md5.h" + +#if WORDS_BIGENDIAN +void byteSwap(uint32_t *buf, unsigned words) +{ + md5byte *p = (md5byte *)buf; + + do { + *buf++ = (uint32_t)((unsigned)p[3] << 8 | p[2]) << 16 | + ((unsigned)p[1] << 8 | p[0]); + p += 4; + } while (--words); +} +#else +#define byteSwap(buf,words) +#endif + +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f,w,x,y,z,in,s) \ + (w += f(x,y,z) + in, w = (w<>(32-s)) + x) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. md5_update blocks + * the data and converts bytes into longwords for this routine. + */ +static void +md5_transform(uint32_t buf[4], uint32_t const in[16]) +{ + uint32_t a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +void md5_init(struct md5_t *ctx) +{ + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + + ctx->bytes[0] = 0; + ctx->bytes[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +void md5_update(struct md5_t* ctx, const unsigned char* buf, unsigned len) +{ + uint32_t t; + + /* Update byte count */ + + t = ctx->bytes[0]; + if ((ctx->bytes[0] = t + len) < t) + ctx->bytes[1]++; /* Carry from low to high */ + + t = 64 - (t & 0x3f); /* Space available in ctx->in (at least 1) */ + if (t > len) { + memcpy((uint8_t *)ctx->in + 64 - t, buf, len); + return; + } + /* First chunk is an odd size */ + memcpy((uint8_t *)ctx->in + 64 - t, buf, t); + byteSwap(ctx->in, 16); + md5_transform(ctx->buf, ctx->in); + buf += t; + len -= t; + + /* Process data in 64-byte chunks */ + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteSwap(ctx->in, 16); + md5_transform(ctx->buf, ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +void md5_final(struct md5_t* ctx, unsigned char* digest) +{ + int count = ctx->bytes[0] & 0x3f; /* Number of bytes in ctx->in */ + uint8_t *p = (uint8_t *)ctx->in + count; + + /* Set the first char of padding to 0x80. There is always room. */ + *p++ = 0x80; + + /* Bytes of padding needed to make 56 bytes (-8..55) */ + count = 56 - 1 - count; + + if (count < 0) { /* Padding forces an extra block */ + memset(p, 0, count + 8); + byteSwap(ctx->in, 16); + md5_transform(ctx->buf, ctx->in); + p = (uint8_t *)ctx->in; + count = 56; + } + memset(p, 0, count); + byteSwap(ctx->in, 14); + + /* Append length in bits and transform */ + ctx->in[14] = ctx->bytes[0] << 3; + ctx->in[15] = ctx->bytes[1] << 3 | ctx->bytes[0] >> 29; + md5_transform(ctx->buf, ctx->in); + + byteSwap(ctx->buf, 4); + memcpy(digest, ctx->buf, 16); +} + +/* + * Convenience function for when what you need to MD5 is + * in a single contiguous buffer. + */ +void MD5(uint8_t *buffer, unsigned int length, uint8_t *digest) +{ + struct md5_t md5ctx; + md5_init(&md5ctx); + md5_update(&md5ctx,buffer,length); + md5_final(&md5ctx,digest); +} + diff --git a/md5.h b/md5.h new file mode 100644 index 00000000..70d33b41 --- /dev/null +++ b/md5.h @@ -0,0 +1,46 @@ +/* + * This is the header file for the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + * + * Changed so as no longer to depend on Colin Plumb's `usual.h' + * header definitions; now uses stuff from dpkg's config.h + * - Ian Jackson . + * Still in the public domain. + */ + +#ifndef MD5_H +#define MD5_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define MD5_SIZE 16 + +struct md5_t { + uint32_t buf[4]; + uint32_t bytes[2]; + uint32_t in[16]; +}; + +void md5_init(struct md5_t* context); +void md5_update(struct md5_t* context, const unsigned char* buf, unsigned len); +void md5_final(struct md5_t* context, unsigned char* digest); + +#ifdef __cplusplus +} +#endif + +#endif /* !MD5_H */ diff --git a/portable.h b/portable.h new file mode 100644 index 00000000..a8738cf0 --- /dev/null +++ b/portable.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __PORTABLE_H +#define __PORTABLE_H + +#if HAVE_CONFIG_H +#include "config.h" /* Use " to include first in the same directory of this file */ +#endif + +/***************************************************************************/ +/* Config */ + +#ifdef __WIN32__ +/* Customize for Windows Mingw/Cygwin */ +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_STAT_H 1 +#define HAVE_DIRENT_H 1 +#define TIME_WITH_SYS_TIME 1 +#define HAVE_SYS_TIME_H 1 +#define HAVE_UNISTD_H 1 +#endif + +/* Include some standard headers */ +#include +#include /* On many systems (e.g., Darwin), `stdio.h' is a prerequisite. */ +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_STDINT_H +#include +#endif + +#if HAVE_UNISTD_H +#include +#endif + +#if TIME_WITH_SYS_TIME +#include +#include +#else +#if HAVE_SYS_TIME_H +#include +#else +#include +#endif +#endif + +#if HAVE_DIRENT_H +#include +#define NAMLEN(dirent) strlen((dirent)->d_name) +#else +#define dirent direct +#define NAMLEN(dirent) (dirent)->d_namlen +#if HAVE_SYS_NDIR_H +#include +#endif +#if HAVE_SYS_DIR_H +#include +#endif +#if HAVE_NDIR_H +#include +#endif +#endif + +#if HAVE_SYS_TYPES_H +#include +#endif + +#if HAVE_SYS_STAT_H +#include +#endif + +#if HAVE_GETOPT_H +#include +#endif + +#if !HAVE_GETOPT +int getopt(int argc, char * const *argv, const char *options); +extern char *optarg; +extern int optind, opterr, optopt; +#endif + +#if HAVE_GETOPT_LONG +#define SWITCH_GETOPT_LONG(a,b) a +#else +#define SWITCH_GETOPT_LONG(a,b) b +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#endif + diff --git a/snapraid.c b/snapraid.c new file mode 100644 index 00000000..a9f69ebd --- /dev/null +++ b/snapraid.c @@ -0,0 +1,1681 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "util.h" +#include "md5.h" +#include "tommylist.h" +#include "tommyhash.h" +#include "tommyhashdyn.h" +#include "tommyarray.h" + +/****************************************************************************/ +/* generic */ + +/** + * Max length of a line. + */ +#define LINE_MAX 1024 + +/** + * If Ctrl+C was pressed. + */ +static volatile int global_interrupt = 0; + +/** + * Safe malloc. + */ +void* malloc_nofail(size_t size) +{ + void* ptr = malloc(size); + + if (!ptr) { + fprintf(stderr, "Low memory\n"); + exit(EXIT_FAILURE); + } + + return ptr; +} + +/****************************************************************************/ +/* snapraid */ + +/** + * Size of the hash used as a checksum. + */ +#define HASH_MAX MD5_SIZE + +/** + * Invalid position. + */ +#define POS_INVALID -1 + +/** + * Basic position type. + * With a 32 bit and 128k block you can address 256 TB. + */ +typedef unsigned pos_t; + +struct snapraid_file; + +/** + * Block of a file. + */ +struct snapraid_block { + pos_t parity_pos; /**< Position of the block in the parity. */ + int is_hashed; /**< If the hash of the block is valid. */ + struct snapraid_file* file; /**< Back pointer to the file owning this block. */ + unsigned char hash[HASH_MAX]; /**< Hash of the block. */ +}; + +/** + * File. + */ +struct snapraid_file { + char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */ + uint64_t size; /**< Size of the file. */ + struct snapraid_block* blockvec; /**< All the blocks of the file. */ + unsigned blockmax; /**< Number of blocks. */ + time_t mtime; /**< Modification time. */ + int is_present; /**< If it's seen as present. */ + + /* nodes for data structures */ + tommy_node nodelist; + tommy_hashdyn_node nodeset; +}; + +/** + * Disk. + */ +struct snapraid_disk { + char name[PATH_MAX]; /**< Name of the disk. */ + char dir[PATH_MAX]; /**< Mount point of the disk. */ + pos_t first_free_block; /**< First free searching block. */ + tommy_list filelist; /**< List of all the files. */ + tommy_hashdyn fileset; /**< Hashtable by sub of all the files. */ + tommy_array blockarr; /**< Block array of the disk. */ +}; + +/****************************************************************************/ +/* snapraid */ + +/** + * Get the relative position of a block inside the file. + */ +pos_t block_file_pos(struct snapraid_block* block) +{ + struct snapraid_file* file = block->file; + + if (block < file->blockvec || block >= file->blockvec + file->blockmax) { + fprintf(stderr, "Internal inconsistency in block %u ownership\n", block->parity_pos); + exit(EXIT_FAILURE); + } + + return block - file->blockvec; +} + +/** + * Get the size in bytes of the block. + * If it's the last block of a file it could less than block_size. + */ +unsigned block_file_size(struct snapraid_block* block, unsigned block_size) +{ + pos_t pos = block_file_pos(block); + + /* if it's the last block */ + if (pos + 1 == block->file->blockmax) { + unsigned remainder; + if (block->file->size == 0) + return 0; + remainder = block->file->size % block_size; + if (remainder == 0) + remainder = block_size; + return remainder; + } + + return block_size; +} + +struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, time_t mtime) +{ + struct snapraid_file* file; + pos_t i; + + file = malloc_nofail(sizeof(struct snapraid_file)); + snprintf(file->sub, sizeof(file->sub), "%s", sub); + file->size = size; + file->blockmax = (size + block_size - 1) / block_size; + file->mtime = mtime; + file->is_present = 0; + file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); + + /* set the back pointer */ + for(i=0;iblockmax;++i) { + file->blockvec[i].parity_pos = POS_INVALID; + file->blockvec[i].is_hashed = 0; + file->blockvec[i].file = file; + } + + return file; +} + +void file_free(struct snapraid_file* file) +{ + free(file->blockvec); + free(file); +} + +int file_compare(const void* void_arg, const void* void_data) +{ + const char* arg = void_arg; + const struct snapraid_file* file = void_data; + return strcmp(arg, file->sub); +} + +tommy_uint32_t file_hash(const char* sub) +{ + return tommy_hash_u32(0, sub, strlen(sub)); +} + +struct snapraid_disk* disk_alloc(const char* name, const char* dir) +{ + struct snapraid_disk* disk; + + disk = malloc_nofail(sizeof(struct snapraid_disk)); + snprintf(disk->name, sizeof(disk->name), "%s", name); + snprintf(disk->dir, sizeof(disk->dir), "%s", dir); + disk->first_free_block = 0; + tommy_list_init(&disk->filelist); + tommy_hashdyn_init(&disk->fileset); + tommy_array_init(&disk->blockarr); + + return disk; +} + +void disk_free(struct snapraid_disk* disk) +{ + tommy_node* node = disk->filelist; + while (node) { + struct snapraid_file* file = node->data; + node = node->next; + file_free(file); + } + tommy_hashdyn_done(&disk->fileset); + tommy_array_done(&disk->blockarr); + free(disk); +} + +static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, pos_t pos) +{ + if (pos < tommy_array_size(&disk->blockarr)) + return tommy_array_get(&disk->blockarr, pos); + else + return 0; +} + +/****************************************************************************/ +/* state */ + +struct snapraid_state { + int verbose; /**< Verbose output. */ + unsigned block_size; /**< Block size in bytes. */ + char content[PATH_MAX]; /**< Path of the content file. */ + char parity[PATH_MAX]; /**< Path of the parity file. */ + tommy_array diskarr; /**< Disk array. */ +}; + +void state_init(struct snapraid_state* state) +{ + state->verbose = 0; + state->block_size = 128*1024; /* default 128 kB */ + state->content[0] = 0; + state->parity[0] = 0; + tommy_array_init(&state->diskarr); + + if (sizeof(off_t) < sizeof(uint64_t)) { + fprintf(stderr, "Internal inconsistency in off_t type size\n"); + exit(EXIT_FAILURE); + } +} + +void state_done(struct snapraid_state* state) +{ + unsigned i; + for(i=0;idiskarr);++i) + disk_free(tommy_array_get(&state->diskarr, i)); + tommy_array_done(&state->diskarr); +} + +/****************************************************************************/ +/* config */ + +void state_config(struct snapraid_state* state, const char* path, int verbose) +{ + FILE* f; + unsigned line; + + state->verbose = verbose; + + f = fopen(path, "rt"); + if (!f) { + if (errno == ENOENT) { + fprintf(stderr, "No configuration file found at '%s'\n", path); + } else if (errno == EACCES) { + fprintf(stderr, "You do not have rights to access the configuration file '%s'\n", path); + } else { + fprintf(stderr, "Error opening the configuration file '%s'. %s.\n", path, strerror(errno)); + } + exit(EXIT_FAILURE); + } + + line = 0; + while (1) { + char buffer[LINE_MAX]; + char* tag; + char* s; + int ret; + + ++line; + + ret = strgets(buffer, LINE_MAX, f); + if (ret < 0) { + fprintf(stderr, "Error reading the configuration file '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (ret == 0) + break; + + /* start */ + s = buffer; + s = strskip(s); + + /* ignore commens and empty lines */ + if (*s == '#' || *s == 0) + continue; + + tag = s; + s = strtoken(s); + + if (strcmp(tag, "block_size") == 0) { + ret = stru(s, &state->block_size); + if (ret != 0) { + fprintf(stderr, "Invalid 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (state->block_size < 1) { + fprintf(stderr, "Too small 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (state->block_size > 16*1024) { + fprintf(stderr, "Too big 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + /* check if it's a power of 2 */ + if ((state->block_size & (state->block_size - 1)) != 0) { + fprintf(stderr, "Not power of 2 'block_size' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + state->block_size *= 1024; + } else if (strcmp(tag, "parity") == 0) { + if (*state->parity) { + fprintf(stderr, "Multiple 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + snprintf(state->parity, sizeof(state->parity), "%s", s); + } else if (strcmp(tag, "content") == 0) { + if (*state->content) { + fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + snprintf(state->content, sizeof(state->content), "%s", s); + } else if (strcmp(tag, "disk") == 0) { + char* name = s; + s = strtoken(s); + tommy_array_insert(&state->diskarr, disk_alloc(name, s)); + } else { + fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path); + exit(EXIT_FAILURE); + } + } + + fclose(f); + + if (!state->parity) { + fprintf(stderr, "No 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (!state->content) { + fprintf(stderr, "No 'content' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } +} + +/****************************************************************************/ +/* read */ + +/** + * Read the state. + */ +void state_read(struct snapraid_state* state) +{ + FILE* f; + char path[PATH_MAX]; + struct snapraid_disk* disk; + struct snapraid_file* file; + unsigned block_index; + unsigned line; + unsigned count_file; + unsigned count_block; + + count_file = 0; + count_block = 0; + + snprintf(path, sizeof(path), "%s", state->content); + f = fopen(state->content, "r"); + if (!f) { + /* if not found, assume empty */ + if (errno == ENOENT) + return; + + fprintf(stderr, "Error opening the state file '%s'\n", path); + exit(EXIT_FAILURE); + } + + printf("Loading state...\n"); + + disk = 0; + file = 0; + line = 0; + block_index = 0; + while (1) { + char buffer[LINE_MAX]; + char* tag; + char* s; + int ret; + + ++line; + + ret = strgets(buffer, LINE_MAX, f); + if (ret < 0) { + fprintf(stderr, "Error reading the state file '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + if (ret == 0) + break; + + /* start */ + s = buffer; + s = strskip(s); + + /* ignore commens and empty lines */ + if (*s == '#' || *s == 0) + continue; + + tag = s; + s = strtoken(s); + + if (strcmp(tag, "file") == 0) { + char* name; + char* size; + char* mtime; + char* sub; + uint64_t v_size; + uint64_t v_mtime; + unsigned i; + + if (file) { + fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + name = s; + s = strtoken(s); + size = s; + s = strtoken(s); + mtime = s; + s = strtoken(s); + sub = s; + + ret = stru64(size, &v_size); + if (ret != 0) { + fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + ret = stru64(mtime, &v_mtime); + if (ret != 0) { + fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + if (!*sub) { + fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + /* find the disk */ + for(i=0;idiskarr);++i) { + disk = tommy_array_get(&state->diskarr, i); + if (strcmp(disk->name, name) == 0) + break; + } + if (i == tommy_array_size(&state->diskarr)) { + fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", name, path, line); + exit(EXIT_FAILURE); + } + + /* allocate the file */ + file = file_alloc(state->block_size, sub, v_size, v_mtime); + + /* insert the file in the file containers */ + tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub)); + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); + + /* start the block allocation of the file */ + block_index = 0; + + /* check for empty file */ + if (block_index == file->blockmax) { + file = 0; + disk = 0; + } + + /* stat */ + ++count_file; + } else if (strcmp(tag, "blk") == 0) { + char* pos; + char* hash; + pos_t v_pos; + struct snapraid_block* block; + char* e; + + if (!file) { + fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + pos = s; + s = strtoken(s); + hash = s; + + ret = stru(pos, &v_pos); + if (ret != 0) { + fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + if (block_index >= file->blockmax) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + block = &file->blockvec[block_index]; + + if (block->parity_pos != POS_INVALID) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + block->parity_pos = v_pos; + + /* set the hash only if present */ + if (*hash != 0) { + e = strdechex(block->hash, HASH_MAX, hash); + if (e) { + fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + block->is_hashed = 1; + } + + /* insert the block in the block array */ + tommy_array_grow(&disk->blockarr, block->parity_pos + 1); + tommy_array_set(&disk->blockarr, block->parity_pos, block); + + /* check for termination of the block list */ + ++block_index; + if (block_index == file->blockmax) { + file = 0; + disk = 0; + } + + /* stat */ + ++count_block; + } else { + fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line); + exit(EXIT_FAILURE); + } + } + + if (file) { + fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + fclose(f); + + if (state->verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + } +} + +/****************************************************************************/ +/* write */ + +/** + * Write the new state. + */ +void state_write(struct snapraid_state* state) +{ + FILE* f; + char path[PATH_MAX]; + unsigned i; + unsigned count_file; + unsigned count_block; + + count_file = 0; + count_block = 0; + + printf("Saving state...\n"); + + snprintf(path, sizeof(path), "%s.tmp", state->content); + f = fopen(path, "w"); + if (!f) { + fprintf(stderr, "Error opening for writing the state file '%s'\n", path); + exit(EXIT_FAILURE); + } + + /* for each disk */ + for(i=0;idiskarr);++i) { + tommy_node* j; + struct snapraid_disk* disk = tommy_array_get(&state->diskarr, i); + + /* for each file */ + for(j=disk->filelist;j!=0;j=j->next) { + pos_t k; + struct snapraid_file* file = j->data; + int ret; + + ret = fprintf(f,"file %s %lld %ld %s\n", disk->name, file->size, file->mtime, file->sub); + if (ret < 0) { + fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + /* for each block */ + for(k=0;kblockmax;++k) { + struct snapraid_block* block = &file->blockvec[k]; + + if (block->is_hashed) { + char s_hash[HASH_MAX*2+1]; + strenchex(s_hash, block->hash, HASH_MAX); + s_hash[HASH_MAX*2] = 0; + ret = fprintf(f, "blk %u %s\n", block->parity_pos, s_hash); + } else { + ret = fprintf(f, "blk %u\n", block->parity_pos); + } + if (ret < 0) { + fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ++count_block; + } + + ++count_file; + } + } + + /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */ + /* than even in a system crash event we have one valid copy of the file. */ + + if (fflush(f) != 0) { + fprintf(stderr, "Error writing the state file '%s', in fflush(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (fsync(fileno(f)) != 0) { + fprintf(stderr, "Error writing the state file '%s' in fsync(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (fclose(f) != 0) { + fprintf(stderr, "Error writing the state file '%s' in close(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (rename(path, state->content) != 0) { + fprintf(stderr, "Error renaming the state file '%s' to '%s' in rename(). %s.\n", path, state->content, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (state->verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + } +} + +/****************************************************************************/ +/* scan */ + +struct snapraid_scan { + unsigned count_equal; + unsigned count_change; + unsigned count_remove; + unsigned count_insert; +}; + +/** + * Remove the specified file from the data set. + */ +static void state_scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + pos_t i; + + /* free all the blocks of the file */ + for(i=0;iblockmax;++i) { + pos_t block_pos = file->blockvec[i].parity_pos; + + /* adjust the first free position */ + if (disk->first_free_block > block_pos) + disk->first_free_block = block_pos; + + tommy_array_set(&disk->blockarr, block_pos, 0); + } + + /* remove the file from the file containers */ + tommy_hashdyn_remove_existing(&disk->fileset, &file->nodeset); + tommy_list_remove_existing(&disk->filelist, &file->nodelist); + + /* deallocate */ + file_free(file); +} + +/** + * Insert the specified file in the data set. + */ +static void state_scan_file_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + pos_t i; + pos_t block_max; + pos_t block_pos; + + /* allocate the blocks of the file */ + block_pos = disk->first_free_block; + block_max = tommy_array_size(&disk->blockarr); + for(i=0;iblockmax;++i) { + /* find a free block */ + while (block_pos < block_max && tommy_array_get(&disk->blockarr, block_pos) != 0) + ++block_pos; + + /* if not found, allocate a new one */ + if (block_pos == block_max) { + ++block_max; + tommy_array_grow(&disk->blockarr, block_max); + } + + /* set it */ + file->blockvec[i].parity_pos = block_pos; + tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]); + } + if (file->blockmax) { + /* set the new free position, but only if allocated something */ + disk->first_free_block = block_pos + 1; + } + + /* insert the file in the file containers */ + tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub)); + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); +} + +static void state_scan_file(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* path, const char* sub) +{ + struct stat st; + struct snapraid_file* file; + + if (stat(path, &st) != 0) { + fprintf(stderr, "Error in stat file '%s'\n", path); + exit(EXIT_FAILURE); + } + + /* check if the file already exists */ + file = tommy_hashdyn_search(&disk->fileset, file_compare, sub, file_hash(sub)); + if (file) { + /* check if the file is the same */ + if (file->size == st.st_size && file->mtime == st.st_mtime) { + /* mark as present */ + ++scan->count_equal; + file->is_present = 1; + return; + } else { + /* remove and reinsert it */ + ++scan->count_change; + --scan->count_insert; + state_scan_file_remove(state, disk, file); + /* continue to insert it */ + } + } + + /* create the new file */ + ++scan->count_insert; + file = file_alloc(state->block_size, sub, st.st_size, st.st_mtime); + file->is_present = 1; + + /* insert it */ + state_scan_file_insert(state, disk, file); +} + +static void state_scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* dir, const char* sub) +{ + DIR* d; + struct dirent* dd; + + d = opendir(dir); + if (!d) { + fprintf(stderr, "Error accessing directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + } + + while ((dd = readdir(d)) != 0) { + if (dd->d_type == DT_REG) { + char path_next[PATH_MAX]; + char sub_next[PATH_MAX]; + snprintf(path_next, sizeof(path_next), "%s%s", dir, dd->d_name); + snprintf(sub_next, sizeof(sub_next), "%s%s", sub, dd->d_name); + state_scan_file(scan, state, disk, path_next, sub_next); + } else if (dd->d_type == DT_DIR) { + if (strcmp(dd->d_name, ".") != 0 && strcmp(dd->d_name, "..") != 0 && strcmp(dd->d_name, "lost+found") != 0 + ) { + char dir_next[PATH_MAX]; + char sub_next[PATH_MAX]; + snprintf(dir_next, sizeof(dir_next), "%s%s/", dir, dd->d_name); + snprintf(sub_next, sizeof(sub_next), "%s%s/", sub, dd->d_name); + state_scan_dir(scan, state, disk, dir_next, sub_next); + } + } else { + if (state->verbose) { + printf("warning: File '%s/%s' not processed\n", dir, dd->d_name); + } + } + } + + closedir(d); +} + +/** + * Update the internal state to represent the new state. + */ +void state_scan(struct snapraid_state* state) +{ + unsigned disk_count = tommy_array_size(&state->diskarr); + unsigned i; + struct snapraid_scan scan; + + scan.count_equal = 0; + scan.count_change = 0; + scan.count_remove = 0; + scan.count_insert = 0; + + for(i=0;idiskarr, i); + tommy_node* node; + + printf("Scanning disk %s...\n", disk->name); + + state_scan_dir(&scan, state, disk, disk->dir, ""); + + /* check for removed files */ + node = disk->filelist; + while (node) { + struct snapraid_file* file = node->data; + + /* next node */ + node = node->next; + + /* remove if not present */ + if (!file->is_present) { + state_scan_file_remove(state, disk, file); + ++scan.count_remove; + } + } + } + + if (state->verbose) { + printf("\tequal %d\n", scan.count_equal); + printf("\tchanged %d\n", scan.count_change); + printf("\tremoved %d\n", scan.count_remove); + printf("\tadded %d\n", scan.count_insert); + } +} + +/****************************************************************************/ +/* common */ + +/** + * Computes the size of the parity data in number of blocks. + */ +static pos_t state_sync_resize(struct snapraid_state* state) +{ + unsigned disk_count = tommy_array_size(&state->diskarr); + pos_t parity_block; + unsigned i; + + /* compute the size of the parity file */ + parity_block = 0; + for(i=0;idiskarr, i); + + /* start from the declared size */ + pos_t block = tommy_array_size(&disk->blockarr); + + /* decrease the block until an allocated block */ + while (block > 0 && tommy_array_get(&disk->blockarr, block - 1) == 0) + --block; + + if (i == 0 || parity_block < block) + parity_block = block; + } + + return parity_block; +} + +static void state_xor_block(unsigned char* xor, const unsigned char* block, unsigned size) +{ + while (size >= 16) { + uint32_t* xor32 = (uint32_t*)xor; + const uint32_t* block32 = (const uint32_t*)block; + xor32[0] ^= block32[0]; + xor32[1] ^= block32[1]; + xor32[2] ^= block32[2]; + xor32[3] ^= block32[3]; + + xor += 16; + block += 16; + size -= 16; + } + + while (size >= 4) { + uint32_t* xor32 = (uint32_t*)xor; + const uint32_t* block32 = (const uint32_t*)block; + xor32[0] ^= block32[0]; + + xor += 4; + block += 4; + size -= 4; + } + + while (size != 0) { + xor[0] ^= block[0]; + + xor += 1; + block += 1; + size -= 1; + } +} + +struct snapraid_sync_block { + char path[PATH_MAX]; /**< Path of the file. */ + struct snapraid_disk* disk; /**< Disk of the file. */ + struct snapraid_file* file; /**< File opened. */ + int f; /**< Handle of the file. */ + off_t offset; /**< Position in the file. */ +}; + +static int state_sync_read_block(int ret_on_error, struct snapraid_sync_block* sync, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size) +{ + ssize_t read_ret; + off_t offset; + unsigned read_size; + + if (sync->file != block->file) { + if (sync->f != -1) { + int ret; + ret = close(sync->f); + if (ret != 0) { + /* here we always abort, as close is supposed to never fail */ + fprintf(stderr, "Error closing file '%s'. %s.\n", sync->file->sub, strerror(errno)); + exit(EXIT_FAILURE); + } + } + + /* invalidate for error */ + sync->file = 0; + sync->f = -1; + + snprintf(sync->path, sizeof(sync->path), "%s%s", sync->disk->dir, block->file->sub); + sync->f = open(sync->path, O_RDONLY | O_BINARY); + if (sync->f == -1) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error opening file '%s'. %s.\n", sync->path, strerror(errno)); + exit(EXIT_FAILURE); + } + + sync->file = block->file; + sync->offset = 0; + } + + offset = block_file_pos(block) * (off_t)block_size; + + if (sync->offset != offset) { + off_t ret; + + /* invalidate for error */ + sync->offset = -1; + + ret = lseek(sync->f, offset, SEEK_SET); + if (ret != offset) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error seeking file '%s'. %s.\n", sync->path, strerror(errno)); + exit(EXIT_FAILURE); + } + sync->offset = offset; + } + + read_size = block_file_size(block, block_size); + + read_ret = read(sync->f, block_buffer, read_size); + if (read_ret != read_size) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error reading file '%s'. %s.\n", sync->path, strerror(errno)); + exit(EXIT_FAILURE); + } + + sync->offset += read_size; + + return read_size; +} + +static void state_sync_write_parity(const char* parity_path, int parity_f, off_t* parity_offset, pos_t pos, unsigned char* block_buffer, unsigned block_size) +{ + ssize_t write_ret; + off_t offset; + + offset = pos * (off_t)block_size; + + if (*parity_offset != offset) { + off_t ret; + ret = lseek(parity_f, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "Error seeking file '%s'. %s.\n", parity_path, strerror(errno)); + exit(EXIT_FAILURE); + } + *parity_offset = offset; + } + + write_ret = write(parity_f, block_buffer, block_size); + if (write_ret != block_size) { + if (errno == ENOSPC) { + fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", parity_path); + } else { + fprintf(stderr, "Error writing file '%s'. %s.\n", parity_path, strerror(errno)); + } + exit(EXIT_FAILURE); + } + + *parity_offset += block_size; +} + +static int state_sync_read_parity(int ret_on_error, const char* parity_path, int parity_f, off_t* parity_offset, pos_t pos, unsigned char* block_buffer, unsigned block_size) +{ + ssize_t read_ret; + off_t offset; + + offset = pos * (off_t)block_size; + + if (*parity_offset != offset) { + off_t ret; + + /* invalidate for error */ + *parity_offset = -1; + + ret = lseek(parity_f, offset, SEEK_SET); + if (ret != offset) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error seeking file '%s'. %s.\n", parity_path, strerror(errno)); + exit(EXIT_FAILURE); + } + *parity_offset = offset; + } + + read_ret = read(parity_f, block_buffer, block_size); + if (read_ret != block_size) { + if (ret_on_error) + return ret_on_error; + fprintf(stderr, "Error reading file '%s'. %s.\n", parity_path, strerror(errno)); + exit(EXIT_FAILURE); + } + + *parity_offset += block_size; + + return block_size; +} + +/****************************************************************************/ +/* sync */ + +static void state_sync_process(struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax) +{ + struct snapraid_sync_block* sync; + off_t parity_offset; + unsigned diskmax = tommy_array_size(&state->diskarr); + pos_t i; + unsigned char* block_buffer; + unsigned char* xor_buffer; + int ret; + uint64_t count_size; + time_t start; + time_t last; + + block_buffer = malloc_nofail(state->block_size); + xor_buffer = malloc_nofail(state->block_size); + + sync = malloc_nofail(diskmax * sizeof(struct snapraid_sync_block)); + for(i=0;idiskarr, i); + sync[i].file = 0; + sync[i].f = -1; + } + parity_offset = -1; + + start = time(0); + last = start; + count_size = 0; + + for(i=blockstart;iis_hashed) { + unhashed = 1; + break; + } + } + + if (!unhashed) + continue; + + /* start with 0 */ + memset(xor_buffer, 0, state->block_size); + + /* for each disk, process the block */ + for(j=0;jblock_size); + + /* now compute the hash */ + md5_init(&md5); + md5_update(&md5, block_buffer, read_size); + md5_final(&md5, hash); + + if (block->is_hashed) { + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_MAX) != 0) { + fprintf(stderr, "Data error for file %s at position %u\n", block->file->sub, block_file_pos(block)); + fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u fix'\n", block_file_pos(block)); + exit(EXIT_FAILURE); + } + } else { + /* copy the hash */ + memcpy(block->hash, hash, HASH_MAX); + + /* mark the block as hashed */ + block->is_hashed = 1; + } + + /* compute the parity */ + state_xor_block(xor_buffer, block_buffer, read_size); + + count_size += read_size; + } + + /* write the parity */ + state_sync_write_parity(state->parity, parity_f, &parity_offset, i, xor_buffer, state->block_size); + + /* progress */ + now = time(0); + if (last != now) { + printf("%u%%, %u MB", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + if (now != start) { + printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / (now - start))); + } + printf("\r"); + fflush(stdout); + last = now; + } + + /* stop if requested */ + if (global_interrupt) { + printf("\rStopping at block %u\n", i); + break; + } + } + + printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + + printf("Done\n"); + + for(i=0;isub, strerror(errno)); + exit(EXIT_FAILURE); + } + } + } + + free(sync); + free(block_buffer); + free(xor_buffer); +} + +/** + * Update the parity file. + */ +void state_sync(struct snapraid_state* state, pos_t blockstart) +{ + char path[PATH_MAX]; + pos_t blockmax; + off_t size; + int ret; + int f; + struct stat st; + + printf("Syncing...\n"); + + blockmax = state_sync_resize(state); + size = blockmax * (off_t)state->block_size; + + if (blockstart >= blockmax) { + fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax); + exit(EXIT_FAILURE); + } + + snprintf(path, sizeof(path), "%s", state->parity); + f = open(path, O_RDWR | O_CREAT, 0600); + if (f == -1) { + fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ret = fstat(f, &st); + if (ret != 0) { + fprintf(stderr, "Error accessing parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + if (st.st_size < size) { +#if HAVE_FALLOCATE + /* allocate real space. This is Linux specific. */ + ret = fallocate(f, 0, 0, size); +#else + /* allocate using a sparse file */ + ret = ftruncate(f, size); +#endif + if (ret != 0) { + if (errno == ENOSPC) { + fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", path); + } else { + fprintf(stderr, "Error growing parity file '%s'. %s.\n", path, strerror(errno)); + } + exit(EXIT_FAILURE); + } + } else { + ret = ftruncate(f, size); + if (ret != 0) { + fprintf(stderr, "Error truncating parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + } + + /* ensure that the file size change is written to disk */ + /* this is not really required, but just to avoid to result in a wrong sized file */ + /* in case of a system crash */ + ret = fsync(f); + if (ret != 0) { + fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + state_sync_process(state, f, blockstart, blockmax); + + /* ensure that data changes are written to disk */ + /* this is required to ensure that parity is more updated than content */ + /* in case of a system crash */ + ret = fsync(f); + if (ret != 0) { + fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ret = close(f); + if (ret != 0) { + fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } +} + +/****************************************************************************/ +/* check */ + + +static int state_check_process(int ret_on_error, struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax) +{ + struct snapraid_sync_block* sync; + off_t parity_offset; + unsigned diskmax = tommy_array_size(&state->diskarr); + pos_t i; + unsigned char* block_buffer; + unsigned char* xor_buffer; + int ret; + uint64_t count_size; + time_t start; + time_t last; + unsigned error; + unsigned unrecoverable_error; + + block_buffer = malloc_nofail(state->block_size); + xor_buffer = malloc_nofail(state->block_size); + + sync = malloc_nofail(diskmax * sizeof(struct snapraid_sync_block)); + for(i=0;idiskarr, i); + sync[i].file = 0; + sync[i].f = -1; + } + parity_offset = -1; + error = 0; + unrecoverable_error = 0; + + start = time(0); + last = start; + count_size = 0; + + for(i=blockstart;iis_hashed) { + hashed = 1; + break; + } + } + + /* if there is at least an hashed block we have to check */ + if (!hashed) + continue; + + /* start with 0 */ + memset(xor_buffer, 0, state->block_size); + + /* for each disk, process the block */ + unhashed = 0; + failed = 0; + for(j=0;jis_hashed) { + unhashed = 1; + continue; + } + + read_size = state_sync_read_block(-1, &sync[j], block, block_buffer, state->block_size); + + if (read_size == -1) { + ++failed; + + /* save the hash for the parity check */ + memcpy(failed_hash, block->hash, HASH_MAX); + failed_size = block_file_size(block, state->block_size); + + fprintf(stderr, "Read error for file %s at position %u\n", block->file->sub, block_file_pos(block)); + ++error; + } else { + /* now compute the hash */ + md5_init(&md5); + md5_update(&md5, block_buffer, read_size); + md5_final(&md5, hash); + + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_MAX) != 0) { + ++failed; + fprintf(stderr, "Data error for file %s at position %u\n", block->file->sub, block_file_pos(block)); + ++error; + } + + /* compute the parity */ + state_xor_block(xor_buffer, block_buffer, read_size); + + count_size += read_size; + } + } + + /* only if no unhashed block, we can check for parity */ + if (unhashed) + continue; + + if (failed == 0) { + int ret; + + /* read the parity */ + ret = state_sync_read_parity(-1, state->parity, parity_f, &parity_offset, i, block_buffer, state->block_size); + if (ret == -1) { + fprintf(stderr, "Parity read error for position %u\n", i); + ++error; + } else { + /* compare it */ + if (memcmp(xor_buffer, block_buffer, state->block_size) != 0) { + fprintf(stderr, "Parity data error for position %u\n", i); + ++error; + } + } + } else if (failed == 1) { + struct md5_t md5; + unsigned char hash[HASH_MAX]; + int ret; + + /* read the parity */ + ret = state_sync_read_parity(-1, state->parity, parity_f, &parity_offset, i, block_buffer, state->block_size); + if (ret == -1) { + fprintf(stderr, "UNRECOVERABLE parity read error for position %u\n", i); + ++unrecoverable_error; + } + + /* compute the failed block */ + state_xor_block(block_buffer, xor_buffer, state->block_size); + + /* now compute the hash */ + md5_init(&md5); + md5_update(&md5, block_buffer, failed_size); + md5_final(&md5, hash); + + /* compare the hash */ + if (memcmp(hash, failed_hash, HASH_MAX) != 0) { + fprintf(stderr, "UNRECOVERABLE parity data error for position %u\n", i); + ++unrecoverable_error; + } + } else { + fprintf(stderr, "UNRECOVERABLE parity missing for position %u\n", i); + ++unrecoverable_error; + } + + /* progress */ + now = time(0); + if (last != now) { + printf("%u%%, %u MB", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + if (now != start) { + printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / (now - start))); + } + printf("\r"); + fflush(stdout); + last = now; + } + + /* stop if requested */ + if (global_interrupt) { + printf("\rStopping at block %u\n", i); + break; + } + } + + printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024))); + + if (error || unrecoverable_error) { + if (error) + printf("%u recoverable errors\n", error); + else + printf("No recoverable errors\n"); + if (unrecoverable_error) + printf("%u UNRECOVERABLE errors\n", unrecoverable_error); + else + printf("No unrecoverable errors\n"); + } else { + printf("No error\n"); + } + + for(i=0;isub, strerror(errno)); + exit(EXIT_FAILURE); + } + } + } + + free(sync); + free(block_buffer); + free(xor_buffer); + + return unrecoverable_error != 0 ? ret_on_error : 0; +} + + +/** + * Check. + */ +void state_check(struct snapraid_state* state, pos_t blockstart) +{ + char path[PATH_MAX]; + pos_t blockmax; + off_t size; + int ret; + int f; + + printf("Checking...\n"); + + blockmax = state_sync_resize(state); + size = blockmax * (off_t)state->block_size; + + if (blockstart >= blockmax) { + fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax); + exit(EXIT_FAILURE); + } + + snprintf(path, sizeof(path), "%s", state->parity); + f = open(path, O_RDONLY); + if (f == -1) { + fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + ret = state_check_process(-1, state, f, blockstart, blockmax); + if (ret == -1) { + /* no extra message */ + exit(EXIT_FAILURE); + } + + ret = close(f); + if (ret != 0) { + fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } +} + +/****************************************************************************/ +/* main */ + +void version(void) { + printf(PACKAGE " v" VERSION " by Andrea Mazzoleni\n"); +} + +void usage(void) { + version(); + + printf("Usage: snapraid [options]\n"); + printf("\n"); + printf("Options:\n"); + printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file (default /etc/snapraid.conf)\n"); + printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); + printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n"); + printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n"); +} + +#if HAVE_GETOPT_LONG +struct option long_options[] = { + { "conf", 1, 0, 'c' }, + { "start", 1, 0, 's' }, + { "verbose", 0, 0, 'v' }, + { "help", 0, 0, 'h' }, + { "version", 0, 0, 'V' }, + { 0, 0, 0, 0 } +}; +#endif + +#define OPTIONS "c:s:vhV" + +void signal_handler(int signal) +{ + switch (signal) { + case SIGINT : + global_interrupt = 1; + break; + } +} + +#define OPERATION_SYNC 0 +#define OPERATION_CHECK 1 +#define OPERATION_FIX 2 + +int main(int argc, char* argv[]) +{ + int c; + int verbose = 0; + const char* conf; + struct snapraid_state state; + int operation; + pos_t blockstart; + + /* defaults */ + conf = "/etc/snapraid.conf"; + verbose = 0; + blockstart = 0; + + opterr = 0; + while ((c = +#if HAVE_GETOPT_LONG + getopt_long(argc, argv, OPTIONS, long_options, 0)) +#else + getopt(argc, argv, OPTIONS)) +#endif + != EOF) { + switch (c) { + case 'c' : + conf = optarg; + break; + case 's' : + if (stru(optarg, &blockstart) != 0) { + fprintf(stderr, "Invalid start position '%s'\n", optarg); + exit(EXIT_FAILURE); + } + break; + case 'v' : + verbose = 1; + break; + case 'h' : + usage(); + exit(EXIT_SUCCESS); + case 'V' : + version(); + exit(EXIT_SUCCESS); + default: + fprintf(stderr, "Unknown option '%c'\n", (char)c); + exit(EXIT_FAILURE); + } + } + + if (optind + 1 != argc) { + usage(); + exit(EXIT_FAILURE); + } + + if (strcmp(argv[optind], "sync") == 0) { + operation = OPERATION_SYNC; + } else if (strcmp(argv[optind], "check") == 0) { + operation = OPERATION_CHECK; + } else if (strcmp(argv[optind], "fix") == 0) { + operation = OPERATION_FIX; + } else { + fprintf(stderr, "Unknown command '%s'\n", argv[optind]); + exit(EXIT_FAILURE); + } + + struct sigaction sig; + sig.sa_handler = signal_handler; + sigemptyset(&sig.sa_mask); + sig.sa_flags = 0; + sigaction(SIGINT, &sig, 0); + + state_init(&state); + + state_config(&state, conf, verbose); + + if (operation == OPERATION_SYNC) { + state_read(&state); + + state_scan(&state); + + state_sync(&state, blockstart); + + state_write(&state); + } else if (operation == OPERATION_CHECK) { + state_read(&state); + + state_check(&state, blockstart); + } + + state_done(&state); + + return EXIT_SUCCESS; +} + diff --git a/test.conf b/test.conf new file mode 100644 index 00000000..8c02358f --- /dev/null +++ b/test.conf @@ -0,0 +1,7 @@ +block_size 1 +parity test/parity +content test/content +disk d1 test/disk1/ +disk d2 test/disk2/ +disk d3 test/disk3/ + diff --git a/testgen.c b/testgen.c new file mode 100644 index 00000000..e7b6b790 --- /dev/null +++ b/testgen.c @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +void gen(int disk, int size) +{ + char path[PATH_MAX]; + char name[PATH_MAX]; + FILE* f; + int i; + int l; + + l = 1 + rand() % 32; + + for(i=0;i. + */ + +#include "portable.h" + +/* redefine a safe malloc for tommy use */ +#define tommy_malloc malloc_nofail +#define tommy_free free + +void* malloc_nofail(size_t size); + +/* direct include tommy source */ +#include "tommylist.c" +#include "tommyhash.c" +#include "tommyhashdyn.c" +#include "tommyarray.c" + diff --git a/tommyarray.c b/tommyarray.c new file mode 100644 index 00000000..9fac6e9f --- /dev/null +++ b/tommyarray.c @@ -0,0 +1,72 @@ +/* + * Copyright 2011 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommyarray.h" + +#include /* for memset */ + +/******************************************************************************/ +/* array */ + +void tommy_array_init(tommy_array* array) +{ + /* fixed initial size */ + array->bucket_bit = TOMMY_ARRAY_BIT; + array->bucket_max = 1 << array->bucket_bit; + array->bucket[0] = tommy_cast(void**, tommy_malloc(array->bucket_max * sizeof(void*))); + memset(array->bucket[0], 0, array->bucket_max * sizeof(void*)); + array->bucket_mac = 1; + array->size = 0; +} + +void tommy_array_done(tommy_array* array) +{ + unsigned i; + for(i=0;ibucket_mac;++i) + tommy_free(array->bucket[i]); +} + +void tommy_array_grow(tommy_array* array, unsigned size) +{ + while (size > array->bucket_max) { + /* grow the hash size and allocate */ + array->bucket[array->bucket_mac] = tommy_cast(void**, tommy_malloc(array->bucket_max * sizeof(void*))); + memset(array->bucket[array->bucket_mac], 0, array->bucket_max * sizeof(void*)); + ++array->bucket_mac; + ++array->bucket_bit; + array->bucket_max = 1 << array->bucket_bit; + } + + if (array->size < size) + array->size = size; +} + +tommy_size_t tommy_array_memory_usage(tommy_array* array) +{ + return array->bucket_max * (tommy_size_t)sizeof(array->bucket[0][0]); +} + diff --git a/tommyarray.h b/tommyarray.h new file mode 100644 index 00000000..82b2e288 --- /dev/null +++ b/tommyarray.h @@ -0,0 +1,158 @@ +/* + * Copyright 2011 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Dynamic array. + * + * This array is able to grow dynamically upon request. + * + * The resize involve an allocation of a new array segment, without reallocating + * the already allocated memory, and then not increasing the heap fragmentation. + * + * This means that the address of the allocated segments never change. + */ + +#ifndef __TOMMYARRAY_H +#define __TOMMYARRAY_H + +#include "tommytypes.h" + +#include /* for assert */ + +/******************************************************************************/ +/* array */ + +/** + * Initial and minimal size of the array expressed as a power of 2. + * The initial size is 2^TOMMY_ARRAY_BIT. + */ +#define TOMMY_ARRAY_BIT 6 + +/** \internal + * Max number of elements as a power of 2. + */ +#define TOMMY_ARRAY_BIT_MAX 32 + +/** + * Array. + */ +typedef struct tommy_array_struct { + void** bucket[TOMMY_ARRAY_BIT_MAX]; /**< Dynamic array of buckets. */ + unsigned bucket_bit; /**< Bits used in the bit mask. */ + unsigned bucket_max; /**< Number of buckets. */ + unsigned bucket_mac; /**< Number of vectors allocated. */ + unsigned size; /**< Currently allocated and initialized size. */ +} tommy_array; + +/** + * Initializes the array. + */ +void tommy_array_init(tommy_array* array); + +/** + * Deinitializes the array. + */ +void tommy_array_done(tommy_array* array); + +/** + * Grow the size up to the specified value. + * All the new elements in the array are initialized with the 0 value. + */ +void tommy_array_grow(tommy_array* array, unsigned size); + +/** + * Gets a reference of the element at the specified position. + * You must be sure that space for this position is already + * allocated calling tommy_array_grow(). + */ +tommy_inline void** tommy_array_ref(tommy_array* array, unsigned pos) +{ + unsigned bsr; + + assert(pos < array->size); + + /* special case for the first bucket */ + if (pos < (1 << TOMMY_ARRAY_BIT)) { + return &array->bucket[0][pos]; + } + + /* get the highest bit set */ + bsr = tommy_ilog2_u32(pos); + + /* clear the highest bit */ + pos -= 1 << bsr; + + return &array->bucket[bsr - TOMMY_ARRAY_BIT + 1][pos]; +} + +/** + * Sets the element at the specified position. + * You must be sure that space for this position is already + * allocated calling tommy_array_grow(). + */ +tommy_inline void tommy_array_set(tommy_array* array, unsigned pos, void* node) +{ + *tommy_array_ref(array, pos) = node; +} + +/** + * Gets the element at the specified position. + * You must be sure that space for this position is already + * allocated calling tommy_array_grow(). + */ +tommy_inline void* tommy_array_get(tommy_array* array, unsigned pos) +{ + return *tommy_array_ref(array, pos); +} + +/** + * Grows and inserts a new element at the end of the array. + */ +tommy_inline void tommy_array_insert(tommy_array* array, void* node) +{ + unsigned pos = array->size; + + tommy_array_grow(array, pos + 1); + + tommy_array_set(array, pos, node); +} + +/** + * Gets the initialized size of the array. + */ +tommy_inline unsigned tommy_array_size(tommy_array* array) +{ + return array->size; +} + +/** + * Gets the size of allocated memory. + */ +tommy_size_t tommy_array_memory_usage(tommy_array* array); + +#endif + diff --git a/tommychain.h b/tommychain.h new file mode 100644 index 00000000..bc4a436d --- /dev/null +++ b/tommychain.h @@ -0,0 +1,224 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Chain of nodes. + * A chain of nodes is an abstraction used to implements complex list operations + * like sorting. + * + * Do not use this directly. Use lists instead. + */ + +#ifndef __TOMMYCHAIN_H +#define __TOMMYCHAIN_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* chain */ + +/** + * Chain of nodes. + * A chain of nodes is a sequence of nodes with the following properties: + * - It contains at least one node. A chains of zero nodes cannot exist. + * - The next field of the tail is of *undefined* value. + * - The prev field of the head is of *undefined* value. + * - All the other inner prev and next fields are correctly set. + */ +typedef struct tommy_chain_struct { + tommy_node* head; /**< Pointer at the head of the chain. */ + tommy_node* tail; /**< Pointer at the tail of the chain. */ +} tommy_chain; + +/** + * Splices a chain in the middle of another chain. + */ +tommy_inline void tommy_chain_splice(tommy_node* first_before, tommy_node* first_after, tommy_node* second_head, tommy_node* second_tail) +{ + /* set the prev list */ + first_after->prev = second_tail; + second_head->prev = first_before; + + /* set the next list */ + first_before->next = second_head; + second_tail->next = first_after; +} + +/** + * Concats two chains. + */ +tommy_inline void tommy_chain_concat(tommy_node* first_tail, tommy_node* second_head) +{ + /* set the prev list */ + second_head->prev = first_tail; + + /* set the next list */ + first_tail->next = second_head; +} + +/** + * Merges two chains. + */ +tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) +{ + tommy_node* first_i = first->head; + tommy_node* second_i = second->head; + + /* merge */ + while (1) { + if (cmp(first_i->data, second_i->data) > 0) { + tommy_node* next = second_i->next; + if (first_i == first->head) { + tommy_chain_concat(second_i, first_i); + first->head = second_i; + } else { + tommy_chain_splice(first_i->prev, first_i, second_i, second_i); + } + if (second_i == second->tail) + break; + second_i = next; + } else { + if (first_i == first->tail) { + tommy_chain_concat(first_i, second_i); + first->tail = second->tail; + break; + } + first_i = first_i->next; + } + } +} + +/** + * Merges two chains managing special degenerated cases. + * It's funtionally equivalent at tommy_chain_merge() but faster with already ordered chains. + */ +tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) +{ + /* identify the condition first <= second */ + if (cmp(first->tail->data, second->head->data) <= 0) { + tommy_chain_concat(first->tail, second->head); + first->tail = second->tail; + return; + } + + /* identify the condition second < first */ + /* here we must be strict on comparison to keep the sort stable */ + if (cmp(second->tail->data, first->head->data) < 0) { + tommy_chain_concat(second->tail, first->head); + first->head = second->head; + return; + } + + tommy_chain_merge(first, second, cmp); +} + +/** + * Max number of elements as a power of 2. + */ +#define TOMMY_CHAIN_BIT_MAX 32 + +/** + * Sorts a chain. + * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, + * similar at the one used in the SGI STL libraries and in the Linux Kernel, + * but faster on degenerated cases like already ordered lists. + * + * SGI STL stl_list.h + * http://www.sgi.com/tech/stl/stl_list.h + * + * Linux Kernel lib/list_sort.c + * http://lxr.linux.no/#linux+v2.6.36/lib/list_sort.c + */ +tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* cmp) +{ + /* + * Bit buckets of chains. + * Each bucket contains 2^i nodes or it's empty. + */ + tommy_chain bit[TOMMY_CHAIN_BIT_MAX]; + + /** + * Value stored inside the bit bucket. + * It's used to know which bucket is empty of full. + */ + unsigned counter; + tommy_node* node = chain->head; + tommy_node* tail = chain->tail; + unsigned mask; + unsigned i; + + counter = 0; + while (1) { + tommy_node* next; + tommy_chain carry; + tommy_chain* last; + + /* carry bit to add */ + last = &carry; + carry.head = node; + carry.tail = node; + next = node->next; + + /* add the bit, propagating the carry */ + i = 0; + mask = counter; + while ((mask & 1) != 0) { + tommy_chain_merge_degenerated(&bit[i], last, cmp); + mask >>= 1; + last = &bit[i]; + ++i; + } + + /* copy the carry in the first empty bit */ + bit[i] = *last; + + /* add the carry in the counter */ + ++counter; + + if (node == tail) + break; + node = next; + } + + /* merge the buckets */ + i = tommy_ctz_u32(counter); + mask = counter >> i; + while (mask != 1) { + mask >>= 1; + if (mask & 1) { + tommy_chain_merge_degenerated(&bit[i+1], &bit[i], cmp); + } else { + bit[i+1] = bit[i]; + } + ++i; + } + + *chain = bit[i]; +} + +#endif + diff --git a/tommyhash.c b/tommyhash.c new file mode 100644 index 00000000..77431417 --- /dev/null +++ b/tommyhash.c @@ -0,0 +1,167 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommyhash.h" + +/******************************************************************************/ +/* hash */ + +tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr) +{ + /* allow unaligned read on Intel x86 and x86_64 platforms */ +#if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || defined(__x86_64__) || defined(_M_X64) + /* defines from http://predef.sourceforge.net/ */ + return *(tommy_uint32_t*)ptr; +#else + const unsigned char* ptr8 = tommy_cast(const unsigned char*, ptr); + return ptr8[0] + ((tommy_uint32_t)ptr8[1] << 8) + ((tommy_uint32_t)ptr8[2] << 16) + ((tommy_uint32_t)ptr8[3] << 24); +#endif +} + +#define tommy_rot(x,k) \ + (((x)<<(k)) | ((x)>>(32-(k)))) + +#define tommy_mix(a,b,c) \ + do { \ + a -= c; a ^= tommy_rot(c, 4); c += b; \ + b -= a; b ^= tommy_rot(a, 6); a += c; \ + c -= b; c ^= tommy_rot(b, 8); b += a; \ + a -= c; a ^= tommy_rot(c,16); c += b; \ + b -= a; b ^= tommy_rot(a,19); a += c; \ + c -= b; c ^= tommy_rot(b, 4); b += a; \ + } while (0) + +#define tommy_final(a,b,c) \ + do { \ + c ^= b; c -= tommy_rot(b,14); \ + a ^= c; a -= tommy_rot(c,11); \ + b ^= a; b -= tommy_rot(a,25); \ + c ^= b; c -= tommy_rot(b,16); \ + a ^= c; a -= tommy_rot(c,4); \ + b ^= a; b -= tommy_rot(a,14); \ + c ^= b; c -= tommy_rot(b,24); \ + } while (0) + +tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len) +{ + const unsigned char* key = tommy_cast(const unsigned char*,void_key); + tommy_uint32_t a, b, c; + + a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val; + + while (key_len > 12) { + a += tommy_le_uint32_read(key + 0); + b += tommy_le_uint32_read(key + 4); + c += tommy_le_uint32_read(key + 8); + + tommy_mix(a,b,c); + + key_len -= 12; + key += 12; + } + + switch (key_len) { + case 0 : + return c; /* used only when called with a zero length */ + case 12 : + c += tommy_le_uint32_read(key + 8); + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; + case 10 : c += ((tommy_uint32_t)key[9]) << 8; + case 9 : c += key[8]; + case 8 : + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; + case 6 : b += ((tommy_uint32_t)key[5]) << 8; + case 5 : b += key[4]; + case 4 : + a += tommy_le_uint32_read(key + 0); + break; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; + case 2 : a += ((tommy_uint32_t)key[1]) << 8; + case 1 : a += key[0]; + } + + tommy_final(a,b,c); + + return c; +} + +tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len) +{ + const unsigned char* key = tommy_cast(const unsigned char*,void_key); + tommy_uint32_t a, b, c; + + a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff); + c += init_val >> 32; + + while (key_len > 12) { + a += tommy_le_uint32_read(key + 0); + b += tommy_le_uint32_read(key + 4); + c += tommy_le_uint32_read(key + 8); + + tommy_mix(a,b,c); + + key_len -= 12; + key += 12; + } + + switch (key_len) { + case 0 : + return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */ + case 12 : + c += tommy_le_uint32_read(key + 8); + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; + case 10 : c += ((tommy_uint32_t)key[9]) << 8; + case 9 : c += key[8]; + case 8 : + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; + case 6 : b += ((tommy_uint32_t)key[5]) << 8; + case 5 : b += key[4]; + case 4 : + a += tommy_le_uint32_read(key + 0); + break; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; + case 2 : a += ((tommy_uint32_t)key[1]) << 8; + case 1 : a += key[0]; + } + + tommy_final(a,b,c); + + return c + ((tommy_uint64_t)b << 32); +} + diff --git a/tommyhash.h b/tommyhash.h new file mode 100644 index 00000000..d584923e --- /dev/null +++ b/tommyhash.h @@ -0,0 +1,104 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. + */ + +#ifndef __TOMMYHASH_H +#define __TOMMYHASH_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* hash */ + +/** + * Hash type used in hashtables. + */ +typedef tommy_key_t tommy_hash_t; + +/** + * Hash function. + * Robert Jenkins "lookup3" hash, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). + * \note + * This function is platform independent as it read always in little endian format. + * \param init_val Initial value. You can use this to concatenate hash. Otherwise set it at 0. + * \param void_key Key to hash. + * \param key_len Length of the key. + * \return The 32 bit hash value. + */ +tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); + +/** + * Hash function for 64 bits. + * Robert Jenkins "lookup3" hash, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2(). + * \note + * This function is platform independent as it read always in little endian format. + */ +tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); + +/** + * Integer hash of 32 bits. + * Robert Jenkins "4-byte Integer Hashing", + * from http://burtleburtle.net/bob/hash/integer.html + */ +tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) +{ + key -= key << 6; + key ^= key >> 17; + key -= key << 9; + key ^= key << 4; + key -= key << 3; + key ^= key << 10; + key ^= key >> 15; + + return key; +} + +/** + * Integer hash of 64 bits. + * Thomas Wang "Integer Hash Function", + * from http://www.cris.com/~Ttwang/tech/inthash.htm + */ +tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) +{ + key = ~key + (key << 21); + key = key ^ (key >> 24); + key = key + (key << 3) + (key << 8); + key = key ^ (key >> 14); + key = key + (key << 2) + (key << 4); + key = key ^ (key >> 28); + key = key + (key << 31); + + return key; +} + +#endif + diff --git a/tommyhashdyn.c b/tommyhashdyn.c new file mode 100644 index 00000000..3e6bee06 --- /dev/null +++ b/tommyhashdyn.c @@ -0,0 +1,174 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommyhashdyn.h" +#include "tommylist.h" + +#include /* for memset */ + +/******************************************************************************/ +/* hashdyn */ + +void tommy_hashdyn_init(tommy_hashdyn* hashdyn) +{ + /* fixed initial size */ + hashdyn->bucket_bit = TOMMY_HASHDYN_BIT; + hashdyn->bucket_max = 1 << hashdyn->bucket_bit; + hashdyn->bucket_mask = hashdyn->bucket_max - 1; + hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(hashdyn->bucket_max * sizeof(tommy_hashdyn_node*))); + memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); + + hashdyn->count = 0; +} + +void tommy_hashdyn_done(tommy_hashdyn* hashdyn) +{ + tommy_free(hashdyn->bucket); +} + +/** + * Resize the bucket vector. + */ +static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, unsigned new_bucket_bit) +{ + unsigned bucket_bit; + unsigned bucket_max; + unsigned new_bucket_max; + unsigned new_bucket_mask; + tommy_hashdyn_node** new_bucket; + + bucket_bit = hashdyn->bucket_bit; + bucket_max = hashdyn->bucket_max; + + new_bucket_max = 1 << new_bucket_bit; + new_bucket_mask = new_bucket_max - 1; + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); + + /* reinsert all the elements */ + if (new_bucket_bit > hashdyn->bucket_bit) { + unsigned i; + + /* grow */ + for(i=0;ibucket[i]; + while (j) { + tommy_hashdyn_node* j_next = j->next; + unsigned index = j->key & new_bucket_mask; + if (new_bucket[index]) + tommy_list_insert_tail_not_empty(new_bucket[index], j); + else + tommy_list_insert_first(&new_bucket[index], j); + j = j_next; + } + } + } else { + unsigned i; + + /* shrink */ + for(i=0;ibucket[i]; + + /* concat the upper bucket */ + tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); + } + } + + tommy_free(hashdyn->bucket); + + /* setup */ + hashdyn->bucket_bit = new_bucket_bit; + hashdyn->bucket_max = new_bucket_max; + hashdyn->bucket_mask = new_bucket_mask; + hashdyn->bucket = new_bucket; +} + +void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) +{ + unsigned pos = hash & hashdyn->bucket_mask; + + tommy_list_insert_tail(&hashdyn->bucket[pos], node, data); + + node->key = hash; + + ++hashdyn->count; + + /* grow if more than 50% full */ + if (hashdyn->count >= hashdyn->bucket_max / 2) { + tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); + } +} + +void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) +{ + unsigned pos = node->key & hashdyn->bucket_mask; + + tommy_list_remove_existing(&hashdyn->bucket[pos], node); + + --hashdyn->count; + + /* shrink if less than 12.5% full */ + if (hashdyn->count <= hashdyn->bucket_max / 8 && hashdyn->bucket_bit > TOMMY_HASHDYN_BIT) { + tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); + } + + return node->data; +} + +void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_compare_func* cmp, const void* cmp_arg, tommy_hash_t hash) +{ + unsigned pos = hash % hashdyn->bucket_max; + tommy_hashdyn_node* i = hashdyn->bucket[pos]; + + while (i) { + /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ + if (i->key == hash && cmp(cmp_arg, i->data) == 0) { + tommy_list_remove_existing(&hashdyn->bucket[pos], i); + + --hashdyn->count; + + return i->data; + } + i = i->next; + } + + return 0; +} + +tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn) +{ + return hashdyn->bucket_max * (tommy_size_t)sizeof(hashdyn->bucket[0]) + + tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node); +} + diff --git a/tommyhashdyn.h b/tommyhashdyn.h new file mode 100644 index 00000000..b22175e9 --- /dev/null +++ b/tommyhashdyn.h @@ -0,0 +1,250 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Dynamic chained hashtable. + * + * This hashtable resizes dynamically. It starts with the minimal size of 16 buckets, it doubles + * the size then it reaches a load factor greater than 0.5 and it halves the size with a load + * factor lower than 0.125. + * + * All the elements are reallocated in a single resize operation done inside + * tommy_hashdyn_insert() or tommy_hashdyn_remove(). + * + * Note that the resize operation takes approximatively 100 [ms] with 1 million of elements, + * and 1 [second] with 10 millions. This could be a problem in real-time applications. + * + * The resize also fragment the heap, as it involves allocating a double-sized table, copy elements, + * and deallocating the older table. Leaving a big hole in the heap. + * + * The ::tommy_hashlin hashtable fixes both problems. + * + * To initialize the hashtable you have to call tommy_hashdyn_init(). + * + * \code + * tommy_hashslin hashdyn; + * + * tommy_hashdyn_init(&hashdyn); + * \endcode + * + * To insert elements in the hashtable you have to call tommy_hashdyn_insert() for + * each element. + * In the insertion call you have to specify the address of the node, the + * address of the object, and the hash value of the key to use. + * The address of the object is used to initialize the tommy_node::data field + * of the node, and the hash to initialize the tommy_node::key field. + * + * \code + * struct object { + * tommy_node node; + * // other fields + * int value; + * }; + * + * struct object* obj = malloc(sizeof(struct object)); // creates the object + * + * obj->value = ...; // initializes the object + * + * tommy_hashdyn_insert(&hashdyn, &obj->node, obj, tommy_inthash_u32(obj->value)); // inserts the object + * \endcode + * + * To find and element in the hashtable you have to call tommy_hashtable_search() + * providing a comparison function, its argument, and the hash of the key to search. + * + * \code + * int compare(const void* arg, const void* obj) + * { + * return (*(const unsigned*)arg != ((const struct object*)obj)->value; + * } + * + * struct object* obj = tommy_hashdyn_search(&hashdyn, compare, &value_to_find, tommy_inthash_u32(value_to_find)); + * if (!obj) { + * // not found + * } else { + * // found + * } + * \endcode + * + * To iterate over all the elements in the hashtable with the same key, you have to + * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL. + * You have also to check explicitely for the key, as the bucket may contains + * different keys. + * + * \code + * tommy_node* i = tommy_hashdyn_bucket(&hashdyn, tommy_inthash_u32(value_to_find)); + * while (i) { + * struct object* obj = i->data; // gets the object pointer + * + * if (obj->value == value_to_find) { + * printf("%d\n", obj->value); // process the object + * } + * + * i = i->next; // goes to the next element + * } + * \endcode + * + * To remove an element from the hashtable you have to call tommy_hashdyn_remove() + * providing a comparison function, its argument, and the hash of the key to search + * and remove. + * + * \code + * struct object* obj = tommy_trie_remove(&hashtable, compare, &value_to_remove, tommy_inthash_u32(value_to_remove)); + * if (obj) { + * free(obj); // frees the object allocated memory + * } + * \endcode + * + * To destroy the hashtable you have to remove all the elements, and deinitialize + * the hashtable calling tommy_hashdyn_done(). + * + * \code + * tommy_hashdyn_done(&hashdyn); + * \endcode + * + * Note that you cannot iterates over all the elements in the hashtable using the + * hashtable itself. You have to insert all the elements also in a ::tommy_list, + * and use the list to iterate. See the \ref multiindex example for more detail. + */ + +#ifndef __TOMMYHASHDYN_H +#define __TOMMYHASHDYN_H + +#include "tommyhash.h" + +/******************************************************************************/ +/* hashdyn */ + +/** \internal + * Initial and minimal size of the hashtable expressed as a power of 2. + * The initial size is 2^TOMMY_HASHDYN_BIT. + */ +#define TOMMY_HASHDYN_BIT 4 + +/** + * Dynamic hashtable node. + * This is the node that you have to include inside your objects. + */ +typedef tommy_node tommy_hashdyn_node; + +/** + * Dynamic chained hashtable. + */ +typedef struct tommy_hashdyn_struct { + tommy_hashdyn_node** bucket; /**< Hash buckets. One list for each hash modulus. */ + unsigned bucket_bit; /**< Bits used in the bit mask. */ + unsigned bucket_max; /**< Number of buckets. */ + unsigned bucket_mask; /**< Bit mask to access the buckets. */ + unsigned count; /**< Number of elements. */ +} tommy_hashdyn; + +/** + * Initializes the hashtable. + */ +void tommy_hashdyn_init(tommy_hashdyn* hashdyn); + +/** + * Deinitializes the hashtable. + */ +void tommy_hashdyn_done(tommy_hashdyn* hashdyn); + +/** + * Inserts an element in the hashtable. + */ +void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash); + +/** + * Searches and removes an element from the hashtable. + * You have to provide a compare function and the hash of the element you want to remove. + * If the element is not found, 0 is returned. + * If more equal elements are present, the first one is removed. + * This operation is faster than calling tommy_hashdyn_bucket() and tommy_hashdyn_remove_existing() separately. + * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. + * The function should return 0 for equal elements, anything other for different elements. + * \param cmp_arg Compare argument passed as first argument of the compare function. + * \param hash Hash of the element to find and remove. + * \return The removed element, or 0 if not found. + */ +void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_compare_func* cmp, const void* cmp_arg, tommy_hash_t hash); + +/** + * Gets the bucket of the specified hash. + * The bucket is guaranteed to contain ALL the elements with the specified hash, + * but it can contain also others. + * You can access elements in the bucket following the ::next pointer until 0. + * \param hash Hash of the element to find. + * \return The head of the bucket, or 0 if empty. + */ +tommy_inline tommy_hashdyn_node* tommy_hashdyn_bucket(tommy_hashdyn* hashdyn, tommy_hash_t hash) +{ + return hashdyn->bucket[hash & hashdyn->bucket_mask]; +} + +/** + * Searches an element in the hashtable. + * You have to provide a compare function and the hash of the element you want to find. + * If more equal elements are present, the first one is returned. + * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. + * The function should return 0 for equal elements, anything other for different elements. + * \param cmp_arg Compare argument passed as first argument of the compare function. + * \param hash Hash of the element to find. + * \return The first element found, or 0 if none. + */ +tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_compare_func* cmp, const void* cmp_arg, tommy_hash_t hash) +{ + tommy_hashdyn_node* i = tommy_hashdyn_bucket(hashdyn, hash); + while (i) { + /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ + if (i->key == hash && cmp(cmp_arg, i->data) == 0) + return i->data; + i = i->next; + } + return 0; +} + +/** + * Removes an element from the hashtable. + * You must already have the address of the element to remove. + * \return The tommy_node::data field of the node removed. + */ +void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); + +/** + * Gets the number of elements. + */ +tommy_inline unsigned tommy_hashdyn_count(tommy_hashdyn* hashdyn) +{ + return hashdyn->count; +} + +/** + * Gets the size of allocated memory. + * It includes the size of the ::tommy_trie_node of the stored elements. + */ +tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn); + +#endif + diff --git a/tommylist.c b/tommylist.c new file mode 100644 index 00000000..0b0125b8 --- /dev/null +++ b/tommylist.c @@ -0,0 +1,87 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommylist.h" +#include "tommychain.h" + +void tommy_list_concat(tommy_list* first, tommy_list* second) +{ + tommy_node* first_head; + tommy_node* first_tail; + tommy_node* second_head; + + if (tommy_list_empty(second)) { + return; + } + + if (tommy_list_empty(first)) { + *first = *second; + return; + } + + first_head = tommy_list_head(first); + second_head = tommy_list_head(second); + first_tail = tommy_list_tail(first); + + /* set the "circular" prev list */ + first_head->prev = second_head->prev; + second_head->prev = first_tail; + + /* set the "0 terminated" next list */ + first_tail->next = second_head; +} + +/** \internal + * Setup a list. + */ +tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) +{ + head->prev = tail; + tail->next = 0; + *list = head; +} + +void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp) +{ + tommy_chain chain; + tommy_node* head; + + if (tommy_list_empty(list)) + return; + + head = tommy_list_head(list); + + /* create a chain from the list */ + chain.head = head; + chain.tail = head->prev; + + tommy_chain_mergesort(&chain, cmp); + + /* restore the list */ + tommy_list_set(list, chain.head, chain.tail); +} + diff --git a/tommylist.h b/tommylist.h new file mode 100644 index 00000000..a90483ff --- /dev/null +++ b/tommylist.h @@ -0,0 +1,309 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Double linked list for collisions into hashtables. + * + * This list is a double linked list mainly targetted for collisions into an hashtables, + * but useable also as a generic list. + * + * The main feature of this list is to require only one pointer to represent the + * list, compared to a classic implementation requiring an head an a tail pointers. + * This reduces the memory usage in hashtables. + * + * Another feature is to support the insertion at the end of the list. This allow to store + * collisions in a stable order. Where for stable order we mean that equal elements keep + * their insertion order. + * + * To initialize the list, you have to call tommy_list_init(), or simply assign + * to it NULL, as an empty list is represented by the NULL value. + * + * \code + * tommy_list list; + * + * tommy_list_init(&list); // initializes the list + * \endcode + * + * To insert elements in the list you have to call tommy_list_insert_tail() + * or tommy_list_insert_head() for each element. + * In the insertion call you have to specify the address of the node and the + * address of the object. + * The address of the object is used to initialize the tommy_node::data field + * of the node. + * + * \code + * struct object { + * tommy_node node; + * // other fields + * int value; + * }; + * + * struct object* obj = malloc(sizeof(struct object)); // creates the object + * + * obj->value = ...; // initializes the object + * + * tommy_list_insert_tail(&list, &obj->node, obj); // inserts the object + * \endcode + * + * To iterates over all the elements in the list you have to call + * tommy_list_head() to get the head of the list and follow the + * tommy_node::next pointer until NULL. + * + * \code + * tommy_node* i = tommy_list_head(&list); + * while (i) { + * struct object* obj = i->data; // gets the object pointer + * + * printf("%d\n", obj->value); // process the object + * + * i = i->next; // go to the next element + * } + * \endcode + * + * To destroy the list you have only to remove all the elements, as the list is + * completely inplace and it doesn't allocate memory. + * + * \code + * tommy_node* i = tommy_list_head(&list); + * while (i) { + * tommy_node* i_next = i->next; // saves the next element before freeing + * + * free(i->data); // frees the object allocated memory + * + * i = i_next; // goes to the next element + * } + * \endcode + */ + +#ifndef __TOMMYLIST_H +#define __TOMMYLIST_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* list */ + +/** + * Double linked list for collisions into hashtables. + */ +typedef tommy_node* tommy_list; + +/** + * Initializes the list. + * The list is completely inplace, so it doesn't need to be deinitialized. + */ +tommy_inline void tommy_list_init(tommy_list* list) +{ + *list = 0; +} + +/** + * Gets the head of the list. + * \return The head node. For empty lists 0 is returned. + */ +tommy_inline tommy_node* tommy_list_head(tommy_list* list) +{ + return *list; +} + +/** + * Gets the tail of the list. + * \return The tail node. For empty lists 0 is returned. + */ +tommy_inline tommy_node* tommy_list_tail(tommy_list* list) +{ + tommy_node* head = tommy_list_head(list); + + if (!head) + return 0; + + return head->prev; +} + +/** \internal + * Creates a new list with a single element. + * \param list The list to initialize. + * \param node The node to insert. + */ +tommy_inline void tommy_list_insert_first(tommy_list* list, tommy_node* node) +{ + /* one element "circular" prev list */ + node->prev = node; + + /* one element "0 terminated" next list */ + node->next = 0; + + *list = node; +} + +/** \internal + * Inserts an element at the head of a not empty list. + * The element is inserted at the head of the list. The list cannot be empty. + * \param list The list. The list cannot be empty. + * \param node The node to insert. + */ +tommy_inline void tommy_list_insert_head_not_empty(tommy_list* list, tommy_node* node) +{ + tommy_node* head = tommy_list_head(list); + + /* insert in the "circular" prev list */ + node->prev = head->prev; + head->prev = node; + + /* insert in the "0 terminated" next list */ + node->next = head; + + *list = node; +} + +/** \internal + * Inserts an element at the tail of a not empty list. + * The element is inserted at the tail of the list. The list cannot be empty. + * \param head The node at the list head. It cannot be 0. + * \param node The node to insert. + */ +tommy_inline void tommy_list_insert_tail_not_empty(tommy_node* head, tommy_node* node) +{ + /* insert in the "circular" prev list */ + node->prev = head->prev; + head->prev = node; + + /* insert in the "0 terminated" next list */ + node->next = 0; + node->prev->next = node; +} + +/** + * Inserts an element at the head of a list. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + */ +tommy_inline void tommy_list_insert_head(tommy_list* list, tommy_node* node, void* data) +{ + tommy_node* head = tommy_list_head(list); + + if (head) { + tommy_list_insert_head_not_empty(list, node); + } else { + tommy_list_insert_first(list, node); + } + + node->data = data; +} + +/** + * Inserts an element at the tail of a list. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + */ +tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, void* data) +{ + tommy_node* head = tommy_list_head(list); + + if (head) { + tommy_list_insert_tail_not_empty(head, node); + } else { + tommy_list_insert_first(list, node); + } + + node->data = data; +} + +/** \internal + * Removes an element from the head of a not empty list. + * \param list The list. The list cannot be empty. + * \return The node removed. + */ +tommy_inline tommy_node* tommy_list_remove_head_not_empty(tommy_list* list) +{ + tommy_node* head = tommy_list_head(list); + + /* remove from the "circular" prev list */ + head->next->prev = head->prev; + + /* remove from the "0 terminated" next list */ + *list = head->next; /* the new head, in case 0 */ + + return head; +} + +/** + * Removes an element from the list. + * You must already have the address of the element to remove. + * \note The node content is left unchanged, including the tommy_node::next + * and tommy_node::prev fields that still contain pointers at the list. + * \param node The node to remove. The node must be in the list. + * \return The tommy_node::data field of the node removed. + */ +tommy_inline void* tommy_list_remove_existing(tommy_list* list, tommy_node* node) +{ + tommy_node* head = tommy_list_head(list); + + /* remove from the "circular" prev list */ + if (node->next) { + node->next->prev = node->prev; + } else { + head->prev = node->prev; /* the last */ + } + + /* remove from the "0 terminated" next list */ + if (head == node) { + *list = node->next; /* the new head, in case 0 */ + } else { + node->prev->next = node->next; + } + + return node->data; +} + +/** + * Concats two lists. + * The second list is concatenated at the first list. + * \param first The first list. + * \param second The second list. After this call the list content is undefined, + * and you should not use it anymore. + */ +void tommy_list_concat(tommy_list* first, tommy_list* second); + +/** + * Sorts a list. + * It's a stable merge sort with O(N*log(N)) worst complexity. + * It's faster on degenerated cases like partially ordered lists. + */ +void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp); + +/** + * Checks if empty. + * \return If the list is empty. + */ +tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list) +{ + return tommy_list_head(list) == 0; +} + +#endif + diff --git a/tommytypes.h b/tommytypes.h new file mode 100644 index 00000000..79d6ef52 --- /dev/null +++ b/tommytypes.h @@ -0,0 +1,313 @@ +/* + * Copyright 2010 Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ANDREA MAZZOLENI AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ANDREA MAZZOLENI OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Generic types. + */ + +#ifndef __TOMMYTYPES_H +#define __TOMMYTYPES_H + +/******************************************************************************/ +/* types */ + +#if defined(_MSC_VER) +#include +typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */ +typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */ +typedef size_t tommy_size_t; /**< Generic size_t type. */ +typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */ +#else +#include +typedef uint32_t tommy_uint32_t; /**< Generic uint32_t type. */ +typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */ +typedef uintptr_t tommy_size_t; /**< Generic size_t type. */ +typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */ +#endif +typedef int tommy_bool_t; /**< Generic boolean type. */ + +/** \internal + * Type cast required for the C++ compilation. + * When compiling in C++ we cannot convert a void* pointer to another pointer. + * In such case we need an explicit cast. + */ +#ifdef __cplusplus +#define tommy_cast(type, value) static_cast(value) +#else +#define tommy_cast(type, value) (value) +#endif + +/******************************************************************************/ +/* heap */ + +/* by default uses malloc/realloc/free */ + +/** + * Generic malloc(), realloc() and free() functions. + * Redefine them to what you need. By default they map to the C malloc(), realloc() and free(). + */ +#if !defined(tommy_malloc) && !defined(tommy_realloc) && !defined(tommy_free) +#include +#define tommy_malloc malloc +#define tommy_realloc realloc +#define tommy_free free +#endif + +/******************************************************************************/ +/* modificators */ + +/** \internal + * Definition of the inline keyword if available. + */ +#if !defined(tommy_inline) +#if __STDC_VERSION__ >= 199901L || defined(__cplusplus) +#define tommy_inline inline +#elif defined(_MSC_VER) || defined(__GNUC__) +#define tommy_inline static __inline +#else +#define tommy_inline static +#endif +#endif + +/** \internal + * Definition of the always_inline keyword if available. + */ +#if !defined(tommy_always_inline) +#if defined(__GNUC__) +#define tommy_always_inline static __attribute__((always_inline)) __inline +#else +#define tommy_always_inline tommy_inline +#endif +#endif + +/** \internal + * Definition of the restrict keyword if available. + */ +#if !defined(tommy_restrict) +#if __STDC_VERSION__ >= 199901L +#define tommy_restrict restrict +#elif defined(_MSC_VER) || defined(__GNUC__) +#define tommy_restrict __restrict +#else +#define tommy_restrict +#endif +#endif + +/** \internal + * Hints the compiler that a condition is likely true. + */ +#if !defined(tommy_likely) +#if defined(__GNUC__) +#define tommy_likely(x) __builtin_expect(!!(x),1) +#else +#define tommy_likely(x) (x) +#endif +#endif + +/** \internal + * Hints the compiler that a condition is likely false. + */ +#if !defined(tommy_unlikely) +#if defined(__GNUC__) +#define tommy_unlikely(x) __builtin_expect(!!(x),0) +#else +#define tommy_unlikely(x) (x) +#endif +#endif + +/******************************************************************************/ +/* key */ + +/** + * Key type used in indexed data structures to store the key or the hash value. + */ +typedef tommy_uint32_t tommy_key_t; + +/** + * Bits into the ::tommy_key_t type. + */ +#define TOMMY_KEY_BIT (sizeof(tommy_key_t) * 8) + +/******************************************************************************/ +/* node */ + +/** + * Data structure node. + * This node type is shared between all the data structures and used to store some + * info directly into the objects you want to store. + * + * A typical declaration is: + * \code + * struct object { + * tommy_node node; + * // other fields + * }; + * \endcode + */ +typedef struct tommy_node_struct { + /** + * Next node. + * The tail node has it at 0, like a 0 terminated list. + */ + struct tommy_node_struct* next; + + /** + * Previous node. + * The head node points to the tail node, like a circular list. + */ + struct tommy_node_struct* prev; + + /** + * Pointer at the object containing the node. + * This field is initialized when inserting nodes into a data structure. + */ + void* data; + + /** + * Key used to store the node. + * With hashtables this field is used to store the hash value. + * With lists this field is not used. + */ + tommy_key_t key; +} tommy_node; + +/******************************************************************************/ +/* compare */ + +/** + * Compare function for elements. + */ +typedef int tommy_compare_func(const void* void_a, const void* void_b); + +/******************************************************************************/ +/* bit hacks */ + +#if defined(_MSC_VER) && !defined(__cplusplus) +#include +#pragma intrinsic(_BitScanReverse) +#endif + +/** \internal + * Integer log2 for constants. + * You can use it only for exact power of 2 up to 256. + */ +#define TOMMY_ILOG2(value) ((value) == 256 ? 8 : (value) == 128 ? 7 :(value) == 64 ? 6 : (value) == 32 ? 5 : (value) == 16 ? 4 : (value) == 8 ? 3 : (value) == 4 ? 2 : (value) == 2 ? 1 : 0) + +/** + * Bit scan reverse or integer log2. + * Return the bit index of the most significant 1 bit. + * + * If no bit is set, the result is undefined. + * To force a return 0 in this case, you can use tommy_ilog2(value | 1). + * + * Other interesting ways for bitscan can be found at: + * + * Bit Twiddling Hacks + * http://graphics.stanford.edu/~seander/bithacks.html + * + * Chess Programming BitScan + * http://chessprogramming.wikispaces.com/BitScan + * + * \param value Value to scan. 0 is not allowed. + * \return The index of the most significan bit set. + */ +tommy_inline unsigned tommy_ilog2_u32(tommy_uint32_t value) +{ +#if defined(_MSC_VER) + unsigned long count; + _BitScanReverse(&count, value); + return count; +#elif defined(__GNUC__) + return 31 - __builtin_clz(value); +#else + /* Find the log base 2 of an N-bit integer in O(lg(N)) operations with multiply and lookup */ + /* from http://graphics.stanford.edu/~seander/bithacks.html */ + static const int TOMMY_DE_BRUIJN_INDEX_ILOG2[32] = { + 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 + }; + + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + + return TOMMY_DE_BRUIJN_INDEX_ILOG2[(tommy_uint32_t)(value * 0x07C4ACDDU) >> 27]; +#endif +} + +/** + * Bit scan forward or trailing zero count. + * Return the bit index of the least significant 1 bit. + * + * If no bit is set, the result is undefined. + * \param value Value to scan. 0 is not allowed. + * \return The index of the least significan bit set. + */ +tommy_inline unsigned tommy_ctz_u32(tommy_uint32_t value) +{ +#if defined(_MSC_VER) + unsigned long count; + _BitScanForward(&count, value); + return count; +#elif defined(__GNUC__) + return __builtin_ctz(value); +#else + /* Count the consecutive zero bits (trailing) on the right with multiply and lookup */ + /* from http://graphics.stanford.edu/~seander/bithacks.html */ + static const tommy_uint32_t TOMMY_DE_BRUIJN_INDEX_CTZ[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; + + return TOMMY_DE_BRUIJN_INDEX_CTZ[(tommy_uint32_t)(((value & -value) * 0x077CB531U)) >> 27]; +#endif +} + +/** + * Rounds up to the next power of 2. + * For the value 0, the result is undefined. + * \return The smallest power of 2 not less than the specified value. + */ +tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) +{ + /* Round up to the next highest power of 2 */ + /* from http://www-graphics.stanford.edu/~seander/bithacks.html */ + + --value; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + ++value; + + return value; +} +#endif + diff --git a/util.h b/util.h new file mode 100644 index 00000000..e59e14e0 --- /dev/null +++ b/util.h @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/****************************************************************************/ +/* string */ + +static inline char* strtoken(char* s) +{ + while (*s && *s != ' ') + ++s; + if (*s) + *s++ = 0; + while (*s == ' ') + ++s; + return s; +} + +static inline char* strskip(char* s) +{ + while (*s == ' ') + ++s; + return s; +} + +static inline int strgets(char* s, unsigned size, FILE* f) +{ + char* i = s; + char* send = s + size; + int c; + + while (1) { + c = fgetc_unlocked(f); + + if (c == EOF || c == '\n') + break; + + *i++ = c; + + if (i == send) { + return -1; + } + } + + if (c == EOF) { + if (ferror_unlocked(f)) { + return -1; + } + if (i == s) + return 0; + } + + /* remove ending spaces */ + while (i != s && isspace(i[-1])) + --i; + *i = 0; + + return 1; +} + +static inline int stru(const char* s, unsigned* value) +{ + unsigned v; + + if (!*s) + return -1; + + v = 0; + while (*s>='0' && *s<='9') { + v *= 10; + v += *s - '0'; + ++s; + } + + if (*s) + return -1; + + *value = v; + + return 0; +} + +static inline int stru64(const char* s, uint64_t* value) +{ + uint64_t v; + + if (!*s) + return -1; + + v = 0; + while (*s>='0' && *s<='9') { + v *= 10; + v += *s - '0'; + ++s; + } + + if (*s) + return -1; + + *value = v; + + return 0; +} + +static char strhexset[16] = "0123456789abcdef"; + +static inline void strenchex(char* str, const void* void_data, unsigned data_len) +{ + const unsigned char* data = void_data; + unsigned i; + + for(i=0;i> 4]; + *str++ = strhexset[b & 0xF]; + } +} + +static inline char* strdechex(void* void_data, unsigned data_len, char* str) +{ + unsigned char* data = void_data; + unsigned i; + + for(i=0;i= 'A' && c0 <= 'F') + b0 = c0 - 'A' + 10; + else if (c0 >= 'a' && c0 <= 'f') + b0 = c0 - 'a' + 10; + else if (c0 >= '0' && c0 <= '9') + b0 = c0 - '0'; + else + return str; + ++str; + + c1 = *str; + if (c1 >= 'A' && c1 <= 'F') + b1 = c1 - 'A' + 10; + else if (c1 >= 'a' && c1 <= 'f') + b1 = c1 - 'a' + 10; + else if (c1 >= '0' && c1 <= '9') + b1 = c1 - '0'; + else + return str; + ++str; + + data[i] = (b0 << 4) | b1; + } + + return 0; +} +