Better MD5 support with test vectors, speed test and libcrypto fast implementation.

HISTORY

@@ -1,6 +1,12 @@
 SnapRAID HISTORY
 ================
 
+0.3 2011/4
+==========
+* Added support for OpenSSL crypto library to use its optimized MD5 implementation.
+* Added test vector for MD5.
+* Added a speed test option for MD5.
+
 0.2 2011/3
 ==========
 * Second public test release of SnapRAID.

Makefile.am

@@ -3,7 +3,6 @@ check_PROGRAMS = mktest
 
 snapraid_SOURCES = \
 	snapraid.c \
-	md5.c \
 	tommy.c \
 	util.c \
 	elem.c \

TODO

@@ -13,8 +13,6 @@ http://lxr.linux.no/#linux+v2.6.38/lib/raid6
 
 * Implement a faster XOR implementation using SSE2.
 
-* Search for a faster MD5 implementation if exists.
-
 * Evaluate to change from MD5 to FVN HASH.
 See:
 http://isthe.com/chongo/tech/comp/fnv/

check.c

@@ -93,8 +93,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
 		/* for each disk, process the block */
 		for(j=0;j<diskmax;++j) {
 			int read_size;
-			struct md5_t md5;
-			unsigned char hash[HASH_MAX];
+			unsigned char hash[MD5_SIZE];
 			struct snapraid_block* block;
 
 			block = disk_block_get(handle[j].disk, i);
@@ -156,12 +155,10 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
 			}
 
 			/* now compute the hash */
-			md5_init(&md5);
-			md5_update(&md5, block_buffer, read_size);
-			md5_final(&md5, hash);
+			memmd5(hash, block_buffer, read_size);
 
 			/* compare the hash */
-			if (memcmp(hash, block->hash, HASH_MAX) != 0) {
+			if (memcmp(hash, block->hash, MD5_SIZE) != 0) {
 				/* save the failed block for the parity check */
 				++failed;
 				failed_block = block;
@@ -222,8 +219,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
 				++recovered_error;
 			}
 		} else if (failed == 1) {
-			struct md5_t md5;
-			unsigned char hash[HASH_MAX];
+			unsigned char hash[MD5_SIZE];
 			unsigned failed_size;
 			int ret;
 			int fixable;
@@ -243,12 +239,10 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
 				failed_size = block_file_size(failed_block, state->block_size);
 
 				/* now compute the hash */
-				md5_init(&md5);
-				md5_update(&md5, block_buffer, failed_size);
-				md5_final(&md5, hash);
+				memmd5(hash, block_buffer, failed_size);
 
 				/* compare the hash */
-				if (memcmp(hash, failed_block->hash, HASH_MAX) != 0) {
+				if (memcmp(hash, failed_block->hash, MD5_SIZE) != 0) {
 					fprintf(stderr, "%u: Parity data error\n", i);
 					fprintf(stderr, "%u: UNRECOVERABLE erros for this block\n", i);
 					++unrecoverable_error;

configure.ac

@@ -20,7 +20,7 @@ AC_HEADER_ASSERT
 AC_HEADER_STDC
 AC_HEADER_DIRENT
 AC_HEADER_TIME
-AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h wchar.h wctype.h])
+AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h])
 AC_CHECK_HEADERS([sys/time.h unistd.h getopt.h fnmatch.h io.h])
 
 dnl Checks for typedefs, structures, and compiler characteristics.
@@ -38,7 +38,6 @@ AC_TYPE_UINT8_T
 
 dnl Checks for library functions.
 AC_SYS_LARGEFILE
-AC_FUNC_MALLOC
 AC_CHECK_FUNCS([memset strchr strerror strrchr mkdir])
 AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf])
 AC_CHECK_FUNCS([ftruncate fsync posix_fallocate posix_fadvise pread pwrite fgetc_unlocked ferror_unlocked fnmatch])
@@ -49,6 +48,13 @@ AC_C_BIGENDIAN
 dnl Checks for compiler
 AC_CHECK_CC_OPT([-Wall], CFLAGS="$CFLAGS -Wall", [])
 
+AC_ARG_WITH([crypto],
+	[AS_HELP_STRING([--with-crypto],
+	[enable the use of the MD5 fast implementation of the OpenSSL crypto library])],
+	[],
+	[with_crypto=no])
+AS_IF([test "x$with_crypto" != xno], [AC_CHECK_LIB([crypto], [MD5], [], [AC_MSG_FAILURE([--with-crypto was given, but test for OpenSSL crypto library failed])])])
+
 dnl Checks for test environment
 AS_CASE([$host],
        [*-*-mingw*], [TESTENV="$WINE"],

elem.h

@@ -18,7 +18,7 @@
 #ifndef __ELEM_H
 #define __ELEM_H
 
-#include "md5.h"
+#include "util.h"
 #include "tommylist.h"
 #include "tommyhash.h"
 #include "tommyhashdyn.h"
@@ -27,11 +27,6 @@
 /****************************************************************************/
 /* snapraid */
 
-/**
- * Size of the hash used as a checksum.
- */
-#define HASH_MAX MD5_SIZE
-
 /**
  * Invalid position.
  */
@@ -70,7 +65,7 @@ struct snapraid_block {
 	block_off_t parity_pos; /**< Position of the block in the parity. */
 	unsigned flag; /**< If the hash of the block is valid. */
 	struct snapraid_file* file; /**< Back pointer to the file owning this block. */
-	unsigned char hash[HASH_MAX]; /**< Hash of the block. */
+	unsigned char hash[MD5_SIZE]; /**< Hash of the block. */
 };
 
 /**

snapraid.1

@@ -23,7 +23,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG]
 .PP
 .PD
 .PP
-snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp]
+snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp] [\(hyT, \(hy\(hyspeed\(hytest]
 .PD 0
 .PP
 .PD
@@ -269,6 +269,13 @@ It\(cqs present mainly for testing.
 .B \(hyv, \(hy\(hyverbose
 Prints more information in the processing.
 .TP
+.B \(hyT, \(hy\(hyspeed\(hytest
+Executes a speed test of the MD5 implementation used.
+As default, SnapRAID uses a C implementation, but if
+configured with the \(hy\(hywith\(hycrypto option it uses the
+MD5 implementation of the OpenSSL crypto library usually
+faster as written in highly optimized assembler.
+.TP
 .B \(hyh, \(hy\(hyhelp
 Prints a short help screen.
 .TP

snapraid.c

@@ -33,21 +33,101 @@
 #define CONF "/etc/" PACKAGE ".conf"
 #endif
 
-void version(void) {
+/**
+ * Test vectors for MD5 from RFC1321.
+ *
+ * MD5("") = d41d8cd98f00b204e9800998ecf8427e
+ * MD5("a") = 0cc175b9c0f1b6a831c399e269772661
+ * MD5("abc") = 900150983cd24fb0d6963f7d28e17f72
+ * MD5("message digest") = f96b697d7cb7938d525a2f31aaf161d0
+ * MD5("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
+ * MD5("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
+ * MD5("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
+ */
+struct test_vector {
+	const char* data;
+	unsigned char digest[MD5_SIZE];
+} TEST[] = {
+{ "", { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } },
+{ "a", { 0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8, 0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61 } },
+{ "abc", { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } },
+{ "message digest", { 0xf9, 0x6b, 0x69, 0x7d, 0x7c, 0xb7, 0x93, 0x8d, 0x52, 0x5a, 0x2f, 0x31, 0xaa, 0xf1, 0x61, 0xd0 } },
+{ "abcdefghijklmnopqrstuvwxyz", { 0xc3, 0xfc, 0xd3, 0xd7, 0x61, 0x92, 0xe4, 0x00, 0x7d, 0xfb, 0x49, 0x6c, 0xca, 0x67, 0xe1, 0x3b } },
+{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xd1, 0x74, 0xab, 0x98, 0xd2, 0x77, 0xd9, 0xf5, 0xa5, 0x61, 0x1c, 0x2c, 0x9f, 0x41, 0x9d, 0x9f } },
+{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0x57, 0xed, 0xf4, 0xa2, 0x2b, 0xe3, 0xc9, 0x55, 0xac, 0x49, 0xda, 0x2e, 0x21, 0x07, 0xb6, 0x7a, } },
+{ 0 }
+};
+
+void selftest(void)
+{
+	unsigned i;
+
+	printf("Self test...\n");
+
+	/* large file check */
+	if (sizeof(off_t) < sizeof(uint64_t)) {
+		fprintf(stderr, "Missing support for large files\n");
+		exit(EXIT_FAILURE);
+	}
+
+	/* test vectors for MD5. It could be dynamically linked, so we have to test it at runtime */
+	for(i=0;TEST[i].data;++i) {
+		unsigned char digest[MD5_SIZE];
+		memmd5(digest, TEST[i].data, strlen(TEST[i].data));
+		if (memcmp(digest, TEST[i].digest, MD5_SIZE) != 0) {
+			fprintf(stderr, "Failed MD5 test vector\n");
+			exit(EXIT_FAILURE);
+		}
+	}
+}
+
+void speed(void)
+{
+	struct timeval start;
+	struct timeval stop;
+	int64_t ds;
+	int64_t dt;
+	unsigned i;
+	unsigned char digest[MD5_SIZE];
+
+	unsigned count = 10000;
+	unsigned blocksize = 256 * 1024;
+	void* block = malloc_nofail(blocksize);
+
+	printf(PACKAGE " v" VERSION " by Andrea Mazzoleni, " PACKAGE_URL "\n");
+	printf("Hashing speed test...\n");
+
+	gettimeofday(&start, 0);
+	for(i=0;i<count;++i)
+		memmd5(digest, block, blocksize);
+	gettimeofday(&stop, 0);
+
+	ds = blocksize * (int64_t)count;
+	dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
+
+	printf("%llu [MB/s]\n", ds / dt);
+
+	free(block);
+}
+
+void version(void)
+{
 	printf(PACKAGE " v" VERSION " by Andrea Mazzoleni, " PACKAGE_URL "\n");
 }
 
-void usage(void) {
+void usage(void)
+{
 	version();
 
 	printf("Usage: " PACKAGE " [options]\n");
 	printf("\n");
 	printf("Options:\n");
 	printf("  " SWITCH_GETOPT_LONG("-c, --conf FILE     ", "-c") "  Configuration file (default " CONF ")\n");
-	printf("  " SWITCH_GETOPT_LONG("-s, --start BLKSTART", "-s") "  Start from the specified block number\n");
-	printf("  " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT", "-t") "  Count of block to process\n");
 	printf("  " SWITCH_GETOPT_LONG("-Z, --force-zero    ", "-Z") "  Force synching of files that get zero size\n");
 	printf("  " SWITCH_GETOPT_LONG("-E, --force-empty   ", "-E") "  Force synching of disks that get empty\n");
+	printf("  " SWITCH_GETOPT_LONG("-s, --start BLKSTART", "-s") "  Start from the specified block number\n");
+	printf("  " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT", "-t") "  Count of block to process\n");
+	printf("  " SWITCH_GETOPT_LONG("-T, --speed-test    ", "-T") "  Speed test of the MD5 implementation\n");
 	printf("  " SWITCH_GETOPT_LONG("-v, --verbose       ", "-v") "  Verbose\n");
 	printf("  " SWITCH_GETOPT_LONG("-h, --help          ", "-h") "  Help\n");
 	printf("  " SWITCH_GETOPT_LONG("-V, --version       ", "-V") "  Version\n");
@@ -56,10 +136,11 @@ void usage(void) {
 #if HAVE_GETOPT_LONG
 struct option long_options[] = {
 	{ "conf", 1, 0, 'c' },
-	{ "start", 1, 0, 's' },
-	{ "count", 1, 0, 't' },
 	{ "force-zero", 0, 0, 'Z' },
 	{ "force-empty", 0, 0, 'E' },
+	{ "start", 1, 0, 's' },
+	{ "count", 1, 0, 't' },
+	{ "speed-test", 0, 0, 'T' },
 	{ "verbose", 0, 0, 'v' },
 	{ "help", 0, 0, 'h' },
 	{ "version", 0, 0, 'V' },
@@ -67,7 +148,7 @@ struct option long_options[] = {
 };
 #endif
 
-#define OPTIONS "c:s:t:ZEvhV"
+#define OPTIONS "c:s:t:ZETvhV"
 
 volatile int global_interrupt = 0;
 
@@ -97,9 +178,6 @@ int main(int argc, char* argv[])
 	block_off_t blockcount;
 	int ret;
 
-	/* intercept Ctrl+C */
-	signal(SIGINT, &signal_handler);
-
 	/* defaults */
 	conf = CONF;
 	verbose = 0;
@@ -147,6 +225,9 @@ int main(int argc, char* argv[])
 		case 'V' :
 			version();
 			exit(EXIT_SUCCESS);
+		case 'T' :
+			speed();
+			exit(EXIT_SUCCESS);
 		default:
 			fprintf(stderr, "Unknown option '%c'\n", (char)c);
 			exit(EXIT_FAILURE);
@@ -169,6 +250,8 @@ int main(int argc, char* argv[])
 		exit(EXIT_FAILURE);
 	}
 
+	selftest();
+
 	state_init(&state);
 
 	state_config(&state, conf, verbose, force_zero, force_empty);
@@ -178,6 +261,9 @@ int main(int argc, char* argv[])
 
 		state_scan(&state);
 
+		/* intercept Ctrl+C */
+		signal(SIGINT, &signal_handler);
+
 		ret = state_sync(&state, blockstart, blockcount);
 
 		/* save the new state if required */
@@ -190,6 +276,9 @@ int main(int argc, char* argv[])
 	} else {
 		state_read(&state);
 
+		/* intercept Ctrl+C */
+		signal(SIGINT, &signal_handler);
+
 		state_check(&state, operation == OPERATION_FIX, blockstart, blockcount);
 	}
 

snapraid.d

@@ -8,7 +8,7 @@ Synopsis
 	:	[-v, --verbose]
 	:	COMMAND
 
-	:snapraid [-V, --version] [-h, --help]
+	:snapraid [-V, --version] [-h, --help] [-T, --speed-test]
 
 Description
 	SnapRAID is a backup program for a disk array using redundancy.
@@ -184,6 +184,13 @@ Options
 	-v, --verbose
 		Prints more information in the processing.
 
+	-T, --speed-test
+		Executes a speed test of the MD5 implementation used.
+		As default, SnapRAID uses a C implementation, but if
+		configured with the --with-crypto option it uses the
+		MD5 implementation of the OpenSSL crypto library usually
+		faster as written in highly optimized assembler.
+
 	-h, --help
 		Prints a short help screen.
 

snapraid.txt

@@ -12,7 +12,7 @@ snapraid [-c, --conf CONFIG]
 	[-v, --verbose]
 	COMMAND
 
-snapraid [-V, --version] [-h, --help]
+snapraid [-V, --version] [-h, --help] [-T, --speed-test]
 
 
 2 DESCRIPTION
@@ -203,6 +203,13 @@ SnapRAID provides three simple commands that allow to:
     -v, --verbose
         Prints more information in the processing.
 
+    -T, --speed-test
+        Executes a speed test of the MD5 implementation used.
+        As default, SnapRAID uses a C implementation, but if
+        configured with the --with-crypto option it uses the
+        MD5 implementation of the OpenSSL crypto library usually
+        faster as written in highly optimized assembler.
+
     -h, --help
         Prints a short help screen.
 

state.c

@@ -37,11 +37,6 @@ void state_init(struct snapraid_state* state)
 	state->parity[0] = 0;
 	tommy_array_init(&state->diskarr);
 	tommy_list_init(&state->excludelist);
-
-	if (sizeof(off_t) < sizeof(uint64_t)) {
-		fprintf(stderr, "Missing support for large files\n");
-		exit(EXIT_FAILURE);
-	}
 }
 
 void state_done(struct snapraid_state* state)
@@ -260,7 +255,7 @@ void state_read(struct snapraid_state* state)
 
 			/* set the hash only if present */
 			if (*hash != 0) {
-				e = strdechex(block->hash, HASH_MAX, hash);
+				e = strdechex(block->hash, MD5_SIZE, hash);
 				if (e) {
 					fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
 					exit(EXIT_FAILURE);
@@ -465,9 +460,9 @@ void state_write(struct snapraid_state* state)
 				}
 
 				if (bit_has(block->flag, BLOCK_HAS_HASH)) {
-					char s_hash[HASH_MAX*2+1];
-					strenchex(s_hash, block->hash, HASH_MAX);
-					s_hash[HASH_MAX*2] = 0;
+					char s_hash[MD5_SIZE*2+1];
+					strenchex(s_hash, block->hash, MD5_SIZE);
+					s_hash[MD5_SIZE*2] = 0;
 					ret = fprintf(f, "%s %u %s\n", tag, block->parity_pos, s_hash);
 				} else {
 					ret = fprintf(f, "%s %u\n", tag, block->parity_pos);

sync.c

@@ -81,8 +81,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
 		/* for each disk, process the block */
 		for(j=0;j<diskmax;++j) {
 			int read_size;
-			struct md5_t md5;
-			unsigned char hash[HASH_MAX];
+			unsigned char hash[MD5_SIZE];
 			struct snapraid_block* block;
 
 			block = disk_block_get(handle[j].disk, i);
@@ -138,13 +137,11 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
 			}
 
 			/* now compute the hash */
-			md5_init(&md5);
-			md5_update(&md5, block_buffer, read_size);
-			md5_final(&md5, hash);
+			memmd5(hash, block_buffer, read_size);
 
 			if (bit_has(block->flag, BLOCK_HAS_HASH)) {
 				/* compare the hash */
-				if (memcmp(hash, block->hash, HASH_MAX) != 0) {
+				if (memcmp(hash, block->hash, MD5_SIZE) != 0) {
 					fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
 					fprintf(stderr, "DANGER! Unexpected data error in a data disk, it isn't possible to sync.\n");
 					fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u check'\n", i);
@@ -154,7 +151,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
 			} else {
 				/* copy the hash, but doesn't mark the block as hashed */
 				/* this allow on error to do not save the failed computation */
-				memcpy(block->hash, hash, HASH_MAX);
+				memcpy(block->hash, hash, MD5_SIZE);
 			}
 
 			/* compute the parity */

util.c

@@ -257,3 +257,21 @@ void memxor(unsigned char* xor, const unsigned char* block, unsigned size)
 	}
 }
 
+#if HAVE_LIBCRYPTO
+#include <openssl/md5.h>
+#else
+#include "md5.c"
+#endif
+
+void memmd5(void* digest, const void* src, unsigned size)
+{
+#if HAVE_LIBCRYPTO
+	MD5((void*)src, size, digest);
+#else
+	struct md5_t md5;
+	md5_init(&md5);
+	md5_update(&md5, src, size);
+	md5_final(&md5, digest);
+#endif
+}
+

util.h

@@ -126,5 +126,16 @@ void* malloc_nofail(size_t size);
  */
 void memxor(unsigned char* xor, const unsigned char* block, unsigned size);
 
+/**
+ * Size of MD5 digest.
+ */
+
+#define MD5_SIZE 16
+
+/**
+ * Computes the MD5 of a memory block.
+ */
+void memmd5(void* digest, const void* src, unsigned size);
+
 #endif