diff --git a/check.c b/check.c
new file mode 100644
index 00000000..c5fd9016
--- /dev/null
+++ b/check.c
@@ -0,0 +1,316 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "util.h"
+#include "elem.h"
+#include "state.h"
+#include "parity.h"
+#include "handle.h"
+
+/****************************************************************************/
+/* check */
+
+static int state_check_process(int ret_on_error, struct snapraid_state* state, int fix, int parity_f, pos_t blockstart, pos_t blockmax)
+{
+ struct snapraid_handle* handle;
+ unsigned diskmax = tommy_array_size(&state->diskarr);
+ pos_t i;
+ unsigned j;
+ unsigned char* block_buffer;
+ unsigned char* xor_buffer;
+ int ret;
+ uint64_t count_size;
+ uint64_t count_block;
+ time_t start;
+ time_t last;
+ unsigned error;
+ unsigned unrecoverable_error;
+ unsigned recovered_error;
+
+ block_buffer = malloc_nofail(state->block_size);
+ xor_buffer = malloc_nofail(state->block_size);
+
+ handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle));
+ for(i=0;idiskarr, i);
+ handle[i].file = 0;
+ handle[i].f = -1;
+ }
+ error = 0;
+ unrecoverable_error = 0;
+ recovered_error = 0;
+
+ count_size = 0;
+ count_block = 0;
+ start = time(0);
+ last = start;
+
+ for(i=blockstart;iis_hashed) {
+ hashed = 1;
+ break;
+ }
+ }
+
+ /* if there is at least an hashed block we have to check */
+ if (!hashed)
+ continue;
+
+ /* start with 0 */
+ memset(xor_buffer, 0, state->block_size);
+
+ /* for each disk, process the block */
+ failed = 0;
+ failed_block = 0;
+ failed_handle = 0;
+ for(j=0;jis_hashed)
+ continue;
+
+ if (fix) {
+ /* if fixing, create the file and open for writing */
+ handle_create(&handle[j], block->file);
+ } else {
+ /* if checking, open the file for reading */
+ ret = handle_open(-1, &handle[j], block->file);
+ if (ret == -1) {
+ /* save the failed block for the parity check */
+ ++failed;
+ failed_block = block;
+ failed_handle = &handle[j];
+
+ fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
+ ++error;
+ continue;
+ }
+ }
+
+ read_size = handle_read(-1, &handle[j], block, block_buffer, state->block_size);
+ if (read_size == -1) {
+ /* save the failed block for the parity check */
+ ++failed;
+ failed_block = block;
+ failed_handle = &handle[j];
+
+ fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
+ ++error;
+ continue;
+ }
+
+ /* now compute the hash */
+ md5_init(&md5);
+ md5_update(&md5, block_buffer, read_size);
+ md5_final(&md5, hash);
+
+ /* compare the hash */
+ if (memcmp(hash, block->hash, HASH_MAX) != 0) {
+ /* save the failed block for the parity check */
+ ++failed;
+ failed_block = block;
+ failed_handle = &handle[j];
+
+ fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
+ ++error;
+ continue;
+ }
+
+ /* compute the parity */
+ memxor(xor_buffer, block_buffer, read_size);
+
+ count_size += read_size;
+ }
+
+ if (failed == 0) {
+ int ret;
+ int fixable;
+
+ fixable = 0;
+
+ /* read the parity */
+ ret = parity_read(-1, state->parity, parity_f, i, block_buffer, state->block_size);
+ if (ret == -1) {
+ fprintf(stderr, "%u: Parity read error\n", i);
+ ++error;
+
+ fixable = 1;
+ } else {
+ /* compare it */
+ if (memcmp(xor_buffer, block_buffer, state->block_size) != 0) {
+ fprintf(stderr, "%u: Parity data error\n", i);
+ ++error;
+
+ fixable = 1;
+ }
+ }
+
+ if (fix && fixable) {
+ /* fix the parity */
+ /* if all the file hashes are matching, we can recompute the parity */
+ parity_write(state->parity, parity_f, i, xor_buffer, state->block_size);
+
+ fprintf(stderr, "%u: Fixed\n", i);
+ ++recovered_error;
+ }
+ } else if (failed == 1) {
+ struct md5_t md5;
+ unsigned char hash[HASH_MAX];
+ unsigned failed_size;
+ int ret;
+ int fixable;
+
+ fixable = 0;
+
+ /* read the parity */
+ ret = parity_read(-1, state->parity, parity_f, i, block_buffer, state->block_size);
+ if (ret == -1) {
+ fprintf(stderr, "%u: Parity read error\n", i);
+ fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i);
+ ++unrecoverable_error;
+ } else {
+ /* compute the failed block */
+ memxor(block_buffer, xor_buffer, state->block_size);
+
+ failed_size = block_file_size(failed_block, state->block_size);
+
+ /* now compute the hash */
+ md5_init(&md5);
+ md5_update(&md5, block_buffer, failed_size);
+ md5_final(&md5, hash);
+
+ /* compare the hash */
+ if (memcmp(hash, failed_block->hash, HASH_MAX) != 0) {
+ fprintf(stderr, "%u: Parity data error\n", i);
+ fprintf(stderr, "%u: UNRECOVERABLE erros for this block\n", i);
+ ++unrecoverable_error;
+ } else {
+ fixable = 1;
+ }
+ }
+
+ if (fix && fixable) {
+ /* fix the file */
+ /* if only one file is wrong, we already have recomputed it */
+ handle_write(failed_handle, failed_block, block_buffer, state->block_size);
+
+ fprintf(stderr, "%u: Fixed\n", i);
+ ++recovered_error;
+ }
+ } else {
+ fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i);
+ ++unrecoverable_error;
+ }
+
+ /* count the number of processed block */
+ ++count_block;
+
+ /* progress */
+ if (state_progress(&start, &last, i, blockmax, count_block, count_size))
+ break;
+ }
+
+ printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
+
+ if (error || unrecoverable_error) {
+ if (error)
+ printf("%u recoverable errors\n", error);
+ else
+ printf("No recoverable errors\n");
+ if (fix)
+ printf("%u fixed errors\n", recovered_error);
+ if (unrecoverable_error)
+ printf("%u UNRECOVERABLE errors\n", unrecoverable_error);
+ else
+ printf("No unrecoverable errors\n");
+ } else {
+ printf("No error\n");
+ }
+
+ for(j=0;jblock_size;
+
+ if (blockstart >= blockmax) {
+ fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax);
+ exit(EXIT_FAILURE);
+ }
+
+ snprintf(path, sizeof(path), "%s", state->parity);
+ if (fix) {
+ /* if fixing, create the file and open for writing */
+ /* it never fails */
+ f = parity_create(path, size);
+ } else {
+ /* if checking, open the file for reading */
+ /* it may fail if the file doesn't exist */
+ f = parity_open(-1, path);
+ }
+
+ ret = state_check_process(-1, state, fix, f, blockstart, blockmax);
+ if (ret == -1) {
+ /* no extra message */
+ exit(EXIT_FAILURE);
+ }
+
+ if (f != -1) {
+ parity_close(path, f);
+ }
+}
+
diff --git a/config.h b/config.h
new file mode 100644
index 00000000..4028bb24
--- /dev/null
+++ b/config.h
@@ -0,0 +1,228 @@
+/* config.h. Generated from config.h.in by configure. */
+/* config.h.in. Generated from configure.ac by autoheader. */
+
+/* Define if building universal (internal helper macro) */
+/* #undef AC_APPLE_UNIVERSAL_BUILD */
+
+/* Define to 1 if you have the header file, and it defines `DIR'.
+ */
+#define HAVE_DIRENT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_FCNTL_H 1
+
+/* Define to 1 if you have the `fsync' function. */
+#define HAVE_FSYNC 1
+
+/* Define to 1 if you have the `ftruncate' function. */
+#define HAVE_FTRUNCATE 1
+
+/* Define to 1 if you have the `getopt' function. */
+#define HAVE_GETOPT 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GETOPT_H 1
+
+/* Define to 1 if you have the `getopt_long' function. */
+#define HAVE_GETOPT_LONG 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
+ to 0 otherwise. */
+#define HAVE_MALLOC 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `memset' function. */
+#define HAVE_MEMSET 1
+
+/* Define to 1 if you have the header file, and it defines `DIR'. */
+/* #undef HAVE_NDIR_H */
+
+/* Define to 1 if you have the `posix_fadvise' function. */
+#define HAVE_POSIX_FADVISE 1
+
+/* Define to 1 if you have the `posix_fallocate' function. */
+#define HAVE_POSIX_FALLOCATE 1
+
+/* Define to 1 if you have the `pread' function. */
+#define HAVE_PREAD 1
+
+/* Define to 1 if you have the `pwrite' function. */
+#define HAVE_PWRITE 1
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDDEF_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the `strchr' function. */
+#define HAVE_STRCHR 1
+
+/* Define to 1 if you have the `strerror' function. */
+#define HAVE_STRERROR 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the `strrchr' function. */
+#define HAVE_STRRCHR 1
+
+/* Define to 1 if you have the header file, and it defines `DIR'.
+ */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define to 1 if you have the header file, and it defines `DIR'.
+ */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UTIME_H 1
+
+/* Define to 1 if `utime(file, NULL)' sets file's timestamp to the present. */
+#define HAVE_UTIME_NULL 1
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Name of package */
+#define PACKAGE "snapraid"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "snapraid"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "snapraid 0.1"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "snapraid"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "0.1"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Define to 1 if you can safely include both and . */
+#define TIME_WITH_SYS_TIME 1
+
+/* Version number of package */
+#define VERSION "0.1"
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+# define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+/* # undef WORDS_BIGENDIAN */
+# endif
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+#define _FILE_OFFSET_BITS 64
+
+/* Enable GLIBC symbols */
+#define _GNU_SOURCE 1
+
+/* Define for large files, on AIX-style hosts. */
+/* #undef _LARGE_FILES */
+
+/* Define for Solaris 2.5.1 so the uint32_t typedef from ,
+ , or is not used. If the typedef were allowed, the
+ #define below would cause a syntax error. */
+/* #undef _UINT32_T */
+
+/* Define for Solaris 2.5.1 so the uint64_t typedef from ,
+ , or is not used. If the typedef were allowed, the
+ #define below would cause a syntax error. */
+/* #undef _UINT64_T */
+
+/* Define for Solaris 2.5.1 so the uint8_t typedef from ,
+ , or is not used. If the typedef were allowed, the
+ #define below would cause a syntax error. */
+/* #undef _UINT8_T */
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+ calls it, or to nothing if 'inline' is not supported under any name. */
+#ifndef __cplusplus
+/* #undef inline */
+#endif
+
+/* Define to rpl_malloc if the replacement function should be used. */
+/* #undef malloc */
+
+/* Define to `long int' if does not define. */
+/* #undef off_t */
+
+/* Define to the equivalent of the C99 'restrict' keyword, or to
+ nothing if this is not supported. Do not define if restrict is
+ supported directly. */
+#define restrict __restrict
+/* Work around a bug in Sun C++: it does not support _Restrict or
+ __restrict__, even though the corresponding Sun C compiler ends up with
+ "#define restrict _Restrict" or "#define restrict __restrict__" in the
+ previous line. Perhaps some future version of Sun C++ will work with
+ restrict; if so, hopefully it defines __RESTRICT like Sun C does. */
+#if defined __SUNPRO_CC && !defined __RESTRICT
+# define _Restrict
+# define __restrict__
+#endif
+
+/* Define to `unsigned int' if does not define. */
+/* #undef size_t */
+
+/* Define to `int' if does not define. */
+/* #undef ssize_t */
+
+/* Define to the type of an unsigned integer type of width exactly 32 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint32_t */
+
+/* Define to the type of an unsigned integer type of width exactly 64 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint64_t */
+
+/* Define to the type of an unsigned integer type of width exactly 8 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint8_t */
+
+/* Define to empty if the keyword `volatile' does not work. Warning: valid
+ code using `volatile' can become incorrect without. Disable with care. */
+/* #undef volatile */
diff --git a/elem.c b/elem.c
new file mode 100644
index 00000000..dcaee407
--- /dev/null
+++ b/elem.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "elem.h"
+#include "util.h"
+
+/****************************************************************************/
+/* snapraid */
+
+pos_t block_file_pos(struct snapraid_block* block)
+{
+ struct snapraid_file* file = block->file;
+
+ if (block < file->blockvec || block >= file->blockvec + file->blockmax) {
+ fprintf(stderr, "Internal inconsistency in block %u ownership\n", block->parity_pos);
+ exit(EXIT_FAILURE);
+ }
+
+ return block - file->blockvec;
+}
+
+unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
+{
+ pos_t pos = block_file_pos(block);
+
+ /* if it's the last block */
+ if (pos + 1 == block->file->blockmax) {
+ unsigned remainder;
+ if (block->file->size == 0)
+ return 0;
+ remainder = block->file->size % block_size;
+ if (remainder == 0)
+ remainder = block_size;
+ return remainder;
+ }
+
+ return block_size;
+}
+
+struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, time_t mtime)
+{
+ struct snapraid_file* file;
+ pos_t i;
+
+ file = malloc_nofail(sizeof(struct snapraid_file));
+ snprintf(file->sub, sizeof(file->sub), "%s", sub);
+ file->size = size;
+ file->blockmax = (size + block_size - 1) / block_size;
+ file->mtime = mtime;
+ file->is_present = 0;
+ file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
+
+ /* set the back pointer */
+ for(i=0;iblockmax;++i) {
+ file->blockvec[i].parity_pos = POS_INVALID;
+ file->blockvec[i].is_hashed = 0;
+ file->blockvec[i].file = file;
+ }
+
+ return file;
+}
+
+void file_free(struct snapraid_file* file)
+{
+ free(file->blockvec);
+ free(file);
+}
+
+int file_compare(const void* void_arg, const void* void_data)
+{
+ const char* arg = void_arg;
+ const struct snapraid_file* file = void_data;
+ return strcmp(arg, file->sub);
+}
+
+struct snapraid_disk* disk_alloc(const char* name, const char* dir)
+{
+ struct snapraid_disk* disk;
+
+ disk = malloc_nofail(sizeof(struct snapraid_disk));
+ snprintf(disk->name, sizeof(disk->name), "%s", name);
+ snprintf(disk->dir, sizeof(disk->dir), "%s", dir);
+ disk->first_free_block = 0;
+ tommy_list_init(&disk->filelist);
+ tommy_hashdyn_init(&disk->fileset);
+ tommy_array_init(&disk->blockarr);
+
+ return disk;
+}
+
+void disk_free(struct snapraid_disk* disk)
+{
+ tommy_node* node = disk->filelist;
+ while (node) {
+ struct snapraid_file* file = node->data;
+ node = node->next;
+ file_free(file);
+ }
+ tommy_hashdyn_done(&disk->fileset);
+ tommy_array_done(&disk->blockarr);
+ free(disk);
+}
diff --git a/elem.h b/elem.h
new file mode 100644
index 00000000..9f77e37c
--- /dev/null
+++ b/elem.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef __ELEM_H
+#define __ELEM_H
+
+#include "md5.h"
+#include "tommylist.h"
+#include "tommyhash.h"
+#include "tommyhashdyn.h"
+#include "tommyarray.h"
+
+/****************************************************************************/
+/* snapraid */
+
+/**
+ * Size of the hash used as a checksum.
+ */
+#define HASH_MAX MD5_SIZE
+
+/**
+ * Invalid position.
+ */
+#define POS_INVALID -1
+
+/**
+ * Basic position type.
+ * With a 32 bit and 128k block you can address 256 TB.
+ */
+typedef unsigned pos_t;
+
+struct snapraid_file;
+
+/**
+ * Block of a file.
+ */
+struct snapraid_block {
+ pos_t parity_pos; /**< Position of the block in the parity. */
+ int is_hashed; /**< If the hash of the block is valid. */
+ struct snapraid_file* file; /**< Back pointer to the file owning this block. */
+ unsigned char hash[HASH_MAX]; /**< Hash of the block. */
+};
+
+/**
+ * File.
+ */
+struct snapraid_file {
+ char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
+ uint64_t size; /**< Size of the file. */
+ struct snapraid_block* blockvec; /**< All the blocks of the file. */
+ unsigned blockmax; /**< Number of blocks. */
+ time_t mtime; /**< Modification time. */
+ int is_present; /**< If it's seen as present. */
+
+ /* nodes for data structures */
+ tommy_node nodelist;
+ tommy_hashdyn_node nodeset;
+};
+
+/**
+ * Disk.
+ */
+struct snapraid_disk {
+ char name[PATH_MAX]; /**< Name of the disk. */
+ char dir[PATH_MAX]; /**< Mount point of the disk. */
+ pos_t first_free_block; /**< First free searching block. */
+ tommy_list filelist; /**< List of all the files. */
+ tommy_hashdyn fileset; /**< Hashtable by sub of all the files. */
+ tommy_array blockarr; /**< Block array of the disk. */
+};
+
+/**
+ * Get the relative position of a block inside the file.
+ */
+pos_t block_file_pos(struct snapraid_block* block);
+
+/**
+ * Get the size in bytes of the block.
+ * If it's the last block of a file it could less than block_size.
+ */
+unsigned block_file_size(struct snapraid_block* block, unsigned block_size);
+
+struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, time_t mtime);
+
+void file_free(struct snapraid_file* file);
+
+int file_compare(const void* void_arg, const void* void_data);
+
+static inline tommy_uint32_t file_hash(const char* sub)
+{
+ return tommy_hash_u32(0, sub, strlen(sub));
+}
+
+struct snapraid_disk* disk_alloc(const char* name, const char* dir);
+
+void disk_free(struct snapraid_disk* disk);
+
+static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, pos_t pos)
+{
+ if (pos < tommy_array_size(&disk->blockarr))
+ return tommy_array_get(&disk->blockarr, pos);
+ else
+ return 0;
+}
+
+#endif
+
diff --git a/handle.c b/handle.c
new file mode 100644
index 00000000..3fa579b4
--- /dev/null
+++ b/handle.c
@@ -0,0 +1,203 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "elem.h"
+#include "handle.h"
+
+/****************************************************************************/
+/* handle */
+
+int handle_create(struct snapraid_handle* handle, struct snapraid_file* file)
+{
+ struct stat st;
+ int ret;
+
+ /* if already opened, nothing to do */
+ if (handle->file == file && handle->f != -1) {
+ return 0;
+ }
+
+ /* close if already open */
+ if (handle->f != -1) {
+ ret = close(handle->f);
+ if (ret != 0) {
+ fprintf(stderr, "Error closing file '%s'. %s.\n", handle->file->sub, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ snprintf(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub);
+ handle->f = open(handle->path, O_RDWR | O_CREAT | O_BINARY, 0600);
+
+ if (handle->f == -1) {
+ fprintf(stderr, "Error opening file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ /* just opened */
+ handle->file = file;
+
+ /* set the size */
+ ret = fstat(handle->f, &st);
+ if (ret != 0) {
+ fprintf(stderr, "Error accessing file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (st.st_size < file->size) {
+#if HAVE_POSIX_FALLOCATE
+ /* allocate real space */
+ ret = posix_fallocate(handle->f, 0, file->size);
+#else
+ /* allocate using a sparse file */
+ ret = ftruncate(handle->f, file->size);
+#endif
+ if (ret != 0) {
+ if (errno == ENOSPC) {
+ fprintf(stderr, "Failed to grow file '%s' due lack of space.\n", handle->path);
+ } else {
+ fprintf(stderr, "Error growing file '%s'. %s.\n", handle->path, strerror(errno));
+ }
+ exit(EXIT_FAILURE);
+ }
+ } else if (st.st_size > file->size) {
+ ret = ftruncate(handle->f, file->size);
+ if (ret != 0) {
+ fprintf(stderr, "Error truncating file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+#if HAVE_POSIX_FADVISE
+ /* advise sequential access */
+ ret = posix_fadvise(handle->f, 0, 0, POSIX_FADV_SEQUENTIAL);
+ if (ret != 0) {
+ fprintf(stderr, "Error advising file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+#endif
+
+ return 0;
+}
+
+int handle_open(int ret_on_error, struct snapraid_handle* handle, struct snapraid_file* file)
+{
+ int ret;
+
+ /* if already opened, nothing to do */
+ if (handle->file == file && handle->f != -1) {
+ return 0;
+ }
+
+ /* close if already open */
+ if (handle->f != -1) {
+ ret = close(handle->f);
+ if (ret != 0) {
+ /* here we always abort on error, as close is supposed to never fail */
+ fprintf(stderr, "Error closing file '%s'. %s.\n", handle->file->sub, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ snprintf(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub);
+ handle->f = open(handle->path, O_RDONLY | O_BINARY);
+
+ if (handle->f == -1) {
+ /* invalidate for error */
+ handle->file = 0;
+ handle->f = -1;
+
+ if (ret_on_error)
+ return ret_on_error;
+
+ fprintf(stderr, "Error opening file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ /* just opened */
+ handle->file = file;
+
+#if HAVE_POSIX_FADVISE
+ /* advise sequential access */
+ ret = posix_fadvise(handle->f, 0, 0, POSIX_FADV_SEQUENTIAL);
+ if (ret != 0) {
+ /* here we always abort on error, as advise is supposed to never fail */
+ fprintf(stderr, "Error advising file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+#endif
+
+ return 0;
+}
+
+void handle_close(struct snapraid_handle* handle)
+{
+ int ret;
+
+ if (handle->f != -1) {
+ ret = close(handle->f);
+ if (ret != 0) {
+ fprintf(stderr, "Error closing file '%s'. %s.\n", handle->file->sub, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ handle->file = 0;
+ handle->f = -1;
+}
+
+int handle_read(int ret_on_error, struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size)
+{
+ ssize_t read_ret;
+ off_t offset;
+ unsigned read_size;
+
+ offset = block_file_pos(block) * (off_t)block_size;
+
+ read_size = block_file_size(block, block_size);
+
+ read_ret = pread(handle->f, block_buffer, read_size, offset);
+ if (read_ret != read_size) {
+ if (ret_on_error)
+ return ret_on_error;
+
+ fprintf(stderr, "Error reading file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ return read_size;
+}
+
+void handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size)
+{
+ ssize_t write_ret;
+ off_t offset;
+ unsigned write_size;
+
+ offset = block_file_pos(block) * (off_t)block_size;
+
+ write_size = block_file_size(block, block_size);
+
+ write_ret = pwrite(handle->f, block_buffer, write_size, offset);
+ if (write_ret != write_size) {
+ fprintf(stderr, "Error writing file '%s'. %s.\n", handle->path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+}
+
diff --git a/handle.h b/handle.h
new file mode 100644
index 00000000..22c01a9d
--- /dev/null
+++ b/handle.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef __HANDLE_H
+#define __HANDLE_H
+
+/****************************************************************************/
+/* handle */
+
+struct snapraid_handle {
+ char path[PATH_MAX]; /**< Path of the file. */
+ struct snapraid_disk* disk; /**< Disk of the file. */
+ struct snapraid_file* file; /**< File opened. */
+ int f; /**< Handle of the file. */
+};
+
+/**
+ * Create a file.
+ * The file is created if missing, and opening with write access.
+ */
+int handle_create(struct snapraid_handle* handle, struct snapraid_file* file);
+
+/**
+ * Open a file.
+ * The file is opened for reading.
+ */
+int handle_open(int ret_on_error, struct snapraid_handle* handle, struct snapraid_file* file);
+
+void handle_close(struct snapraid_handle* handle);
+
+int handle_read(int ret_on_error, struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size);
+
+void handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size);
+
+#endif
+
diff --git a/parity.c b/parity.c
new file mode 100644
index 00000000..2d1de6f5
--- /dev/null
+++ b/parity.c
@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "elem.h"
+#include "state.h"
+#include "parity.h"
+
+/****************************************************************************/
+/* parity */
+
+pos_t parity_resize(struct snapraid_state* state)
+{
+ unsigned disk_count = tommy_array_size(&state->diskarr);
+ pos_t parity_block;
+ unsigned i;
+
+ /* compute the size of the parity file */
+ parity_block = 0;
+ for(i=0;idiskarr, i);
+
+ /* start from the declared size */
+ pos_t block = tommy_array_size(&disk->blockarr);
+
+ /* decrease the block until an allocated block */
+ while (block > 0 && tommy_array_get(&disk->blockarr, block - 1) == 0)
+ --block;
+
+ if (i == 0 || parity_block < block)
+ parity_block = block;
+ }
+
+ return parity_block;
+}
+
+int parity_create(const char* path, off_t size)
+{
+ struct stat st;
+ int ret;
+ int f;
+
+ f = open(path, O_RDWR | O_CREAT | O_BINARY, 0600);
+ if (f == -1) {
+ fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ret = fstat(f, &st);
+ if (ret != 0) {
+ fprintf(stderr, "Error accessing parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (st.st_size < size) {
+#if HAVE_POSIX_FALLOCATE
+ /* allocate real space */
+ ret = posix_fallocate(f, 0, size);
+#else
+ /* allocate using a sparse file */
+ ret = ftruncate(f, size);
+#endif
+ if (ret != 0) {
+ if (errno == ENOSPC) {
+ fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", path);
+ } else {
+ fprintf(stderr, "Error growing parity file '%s'. %s.\n", path, strerror(errno));
+ }
+ exit(EXIT_FAILURE);
+ }
+ } else if (st.st_size > size) {
+ ret = ftruncate(f, size);
+ if (ret != 0) {
+ fprintf(stderr, "Error truncating parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+#if HAVE_POSIX_FADVISE
+ /* advise sequential access */
+ ret = posix_fadvise(f, 0, 0, POSIX_FADV_SEQUENTIAL);
+ if (ret != 0) {
+ fprintf(stderr, "Error advising parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+#endif
+
+ return f;
+}
+
+int parity_open(int ret_on_error, const char* path)
+{
+ int ret;
+ int f;
+
+ f = open(path, O_RDONLY | O_BINARY);
+ if (f == -1) {
+ if (ret_on_error)
+ return -1;
+ fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+#if HAVE_POSIX_FADVISE
+ /* advise sequential access */
+ ret = posix_fadvise(f, 0, 0, POSIX_FADV_SEQUENTIAL);
+ if (ret != 0) {
+ /* here we always abort on error, as advise is supposed to never fail */
+ fprintf(stderr, "Error advising parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+#endif
+
+ return f;
+}
+
+void parity_close(const char* path, int f)
+{
+ int ret;
+
+ /* ensure that data changes are written to disk */
+ /* this is required to ensure that parity is more updated than content */
+ /* in case of a system crash */
+ ret = fsync(f);
+ if (ret != 0) {
+ fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ret = close(f);
+ if (ret != 0) {
+ fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+}
+
+void parity_write(const char* path, int f, pos_t pos, unsigned char* block_buffer, unsigned block_size)
+{
+ ssize_t write_ret;
+ off_t offset;
+
+ offset = pos * (off_t)block_size;
+
+ write_ret = pwrite(f, block_buffer, block_size, offset);
+ if (write_ret != block_size) {
+ if (errno == ENOSPC) {
+ fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", path);
+ } else {
+ fprintf(stderr, "Error writing file '%s'. %s.\n", path, strerror(errno));
+ }
+ exit(EXIT_FAILURE);
+ }
+}
+
+int parity_read(int ret_on_error, const char* path, int f, pos_t pos, unsigned char* block_buffer, unsigned block_size)
+{
+ ssize_t read_ret;
+ off_t offset;
+
+ /* if the file is not opened, return an error without trying */
+ if (f == -1 && ret_on_error)
+ return -1;
+
+ offset = pos * (off_t)block_size;
+
+ read_ret = pread(f, block_buffer, block_size, offset);
+ if (read_ret != block_size) {
+ if (ret_on_error)
+ return ret_on_error;
+ fprintf(stderr, "Error reading file '%s'. %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ return block_size;
+}
+
diff --git a/parity.h b/parity.h
new file mode 100644
index 00000000..552f32b2
--- /dev/null
+++ b/parity.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef __PARITY_H
+#define __PARITY_H
+
+/****************************************************************************/
+/* parity */
+
+/**
+ * Computes the size of the parity data in number of blocks.
+ */
+pos_t parity_resize(struct snapraid_state* state);
+
+int parity_create(const char* path, off_t size);
+
+int parity_open(int ret_on_error, const char* path);
+
+void parity_close(const char* path, int f);
+
+void parity_write(const char* parity_path, int parity_f, pos_t pos, unsigned char* block_buffer, unsigned block_size);
+
+int parity_read(int ret_on_error, const char* parity_path, int parity_f, pos_t pos, unsigned char* block_buffer, unsigned block_size);
+
+#endif
+
diff --git a/scan.c b/scan.c
new file mode 100644
index 00000000..bd64b20f
--- /dev/null
+++ b/scan.c
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "util.h"
+#include "elem.h"
+#include "state.h"
+
+struct snapraid_scan {
+ unsigned count_equal;
+ unsigned count_change;
+ unsigned count_remove;
+ unsigned count_insert;
+};
+
+/**
+ * Remove the specified file from the data set.
+ */
+static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
+{
+ pos_t i;
+
+ /* free all the blocks of the file */
+ for(i=0;iblockmax;++i) {
+ pos_t block_pos = file->blockvec[i].parity_pos;
+
+ /* adjust the first free position */
+ if (disk->first_free_block > block_pos)
+ disk->first_free_block = block_pos;
+
+ tommy_array_set(&disk->blockarr, block_pos, 0);
+ }
+
+ /* remove the file from the file containers */
+ tommy_hashdyn_remove_existing(&disk->fileset, &file->nodeset);
+ tommy_list_remove_existing(&disk->filelist, &file->nodelist);
+
+ /* deallocate */
+ file_free(file);
+}
+
+/**
+ * Insert the specified file in the data set.
+ */
+static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
+{
+ pos_t i;
+ pos_t block_max;
+ pos_t block_pos;
+
+ /* allocate the blocks of the file */
+ block_pos = disk->first_free_block;
+ block_max = tommy_array_size(&disk->blockarr);
+ for(i=0;iblockmax;++i) {
+ /* find a free block */
+ while (block_pos < block_max && tommy_array_get(&disk->blockarr, block_pos) != 0)
+ ++block_pos;
+
+ /* if not found, allocate a new one */
+ if (block_pos == block_max) {
+ ++block_max;
+ tommy_array_grow(&disk->blockarr, block_max);
+ }
+
+ /* set it */
+ file->blockvec[i].parity_pos = block_pos;
+ tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]);
+ }
+ if (file->blockmax) {
+ /* set the new free position, but only if allocated something */
+ disk->first_free_block = block_pos + 1;
+ }
+
+ /* insert the file in the file containers */
+ tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub));
+ tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
+}
+
+static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* path, const char* sub)
+{
+ struct stat st;
+ struct snapraid_file* file;
+
+ if (stat(path, &st) != 0) {
+ fprintf(stderr, "Error in stat file '%s'\n", path);
+ exit(EXIT_FAILURE);
+ }
+
+ /* check if the file already exists */
+ file = tommy_hashdyn_search(&disk->fileset, file_compare, sub, file_hash(sub));
+ if (file) {
+ /* check if the file is the same */
+ if (file->size == st.st_size && file->mtime == st.st_mtime) {
+ /* mark as present */
+ ++scan->count_equal;
+ file->is_present = 1;
+ return;
+ } else {
+ /* remove and reinsert it */
+ ++scan->count_change;
+ --scan->count_insert;
+ scan_file_remove(state, disk, file);
+ /* continue to insert it */
+ }
+ }
+
+ /* create the new file */
+ ++scan->count_insert;
+ file = file_alloc(state->block_size, sub, st.st_size, st.st_mtime);
+ file->is_present = 1;
+
+ /* insert it */
+ scan_file_insert(state, disk, file);
+}
+
+static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* dir, const char* sub)
+{
+ DIR* d;
+ struct dirent* dd;
+
+ d = opendir(dir);
+ if (!d) {
+ fprintf(stderr, "Error accessing directory '%s'. %s.\n", dir, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ while ((dd = readdir(d)) != 0) {
+ if (dd->d_type == DT_REG) {
+ char path_next[PATH_MAX];
+ char sub_next[PATH_MAX];
+ snprintf(path_next, sizeof(path_next), "%s%s", dir, dd->d_name);
+ snprintf(sub_next, sizeof(sub_next), "%s%s", sub, dd->d_name);
+ scan_file(scan, state, disk, path_next, sub_next);
+ } else if (dd->d_type == DT_DIR) {
+ if (strcmp(dd->d_name, ".") != 0 && strcmp(dd->d_name, "..") != 0 && strcmp(dd->d_name, "lost+found") != 0
+ ) {
+ char dir_next[PATH_MAX];
+ char sub_next[PATH_MAX];
+ snprintf(dir_next, sizeof(dir_next), "%s%s/", dir, dd->d_name);
+ snprintf(sub_next, sizeof(sub_next), "%s%s/", sub, dd->d_name);
+ scan_dir(scan, state, disk, dir_next, sub_next);
+ }
+ } else {
+ if (state->verbose) {
+ printf("warning: File '%s/%s' not processed\n", dir, dd->d_name);
+ }
+ }
+ }
+
+ closedir(d);
+}
+
+void state_scan(struct snapraid_state* state)
+{
+ unsigned diskmax = tommy_array_size(&state->diskarr);
+ unsigned i;
+ struct snapraid_scan* scan;
+
+ scan = malloc_nofail(diskmax * sizeof(struct snapraid_scan));
+ for(i=0;idiskarr, i);
+ tommy_node* node;
+
+ printf("Scanning disk %s...\n", disk->name);
+
+ scan_dir(&scan[i], state, disk, disk->dir, "");
+
+ /* check for removed files */
+ node = disk->filelist;
+ while (node) {
+ struct snapraid_file* file = node->data;
+
+ /* next node */
+ node = node->next;
+
+ /* remove if not present */
+ if (!file->is_present) {
+ scan_file_remove(state, disk, file);
+ ++scan[i].count_remove;
+ }
+ }
+
+ /* if all the previous file were removed */
+ if (scan[i].count_equal == 0 && scan[i].count_remove != 0) {
+ if (!state->force) {
+ fprintf(stderr, "All the file in disk '%s' at dir '%s' are missing!\n", disk->name, disk->dir);
+ fprintf(stderr, "If it's really what you want to do, use 'snapraid -f sync\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+ }
+
+ if (state->verbose) {
+ struct snapraid_scan total;
+
+ total.count_equal = 0;
+ total.count_change = 0;
+ total.count_remove = 0;
+ total.count_insert = 0;
+
+ for(i=0;ifile;
-
- if (block < file->blockvec || block >= file->blockvec + file->blockmax) {
- fprintf(stderr, "Internal inconsistency in block %u ownership\n", block->parity_pos);
- exit(EXIT_FAILURE);
- }
-
- return block - file->blockvec;
-}
-
-/**
- * Get the size in bytes of the block.
- * If it's the last block of a file it could less than block_size.
- */
-unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
-{
- pos_t pos = block_file_pos(block);
-
- /* if it's the last block */
- if (pos + 1 == block->file->blockmax) {
- unsigned remainder;
- if (block->file->size == 0)
- return 0;
- remainder = block->file->size % block_size;
- if (remainder == 0)
- remainder = block_size;
- return remainder;
- }
-
- return block_size;
-}
-
-struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, time_t mtime)
-{
- struct snapraid_file* file;
- pos_t i;
-
- file = malloc_nofail(sizeof(struct snapraid_file));
- snprintf(file->sub, sizeof(file->sub), "%s", sub);
- file->size = size;
- file->blockmax = (size + block_size - 1) / block_size;
- file->mtime = mtime;
- file->is_present = 0;
- file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
-
- /* set the back pointer */
- for(i=0;iblockmax;++i) {
- file->blockvec[i].parity_pos = POS_INVALID;
- file->blockvec[i].is_hashed = 0;
- file->blockvec[i].file = file;
- }
-
- return file;
-}
-
-void file_free(struct snapraid_file* file)
-{
- free(file->blockvec);
- free(file);
-}
-
-int file_compare(const void* void_arg, const void* void_data)
-{
- const char* arg = void_arg;
- const struct snapraid_file* file = void_data;
- return strcmp(arg, file->sub);
-}
-
-tommy_uint32_t file_hash(const char* sub)
-{
- return tommy_hash_u32(0, sub, strlen(sub));
-}
-
-struct snapraid_disk* disk_alloc(const char* name, const char* dir)
-{
- struct snapraid_disk* disk;
-
- disk = malloc_nofail(sizeof(struct snapraid_disk));
- snprintf(disk->name, sizeof(disk->name), "%s", name);
- snprintf(disk->dir, sizeof(disk->dir), "%s", dir);
- disk->first_free_block = 0;
- tommy_list_init(&disk->filelist);
- tommy_hashdyn_init(&disk->fileset);
- tommy_array_init(&disk->blockarr);
-
- return disk;
-}
-
-void disk_free(struct snapraid_disk* disk)
-{
- tommy_node* node = disk->filelist;
- while (node) {
- struct snapraid_file* file = node->data;
- node = node->next;
- file_free(file);
- }
- tommy_hashdyn_done(&disk->fileset);
- tommy_array_done(&disk->blockarr);
- free(disk);
-}
-
-static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, pos_t pos)
-{
- if (pos < tommy_array_size(&disk->blockarr))
- return tommy_array_get(&disk->blockarr, pos);
- else
- return 0;
-}
-
-/****************************************************************************/
-/* state */
-
-struct snapraid_state {
- int verbose; /**< Verbose output. */
- unsigned block_size; /**< Block size in bytes. */
- char content[PATH_MAX]; /**< Path of the content file. */
- char parity[PATH_MAX]; /**< Path of the parity file. */
- tommy_array diskarr; /**< Disk array. */
-};
-
-void state_init(struct snapraid_state* state)
-{
- state->verbose = 0;
- state->block_size = 128*1024; /* default 128 kB */
- state->content[0] = 0;
- state->parity[0] = 0;
- tommy_array_init(&state->diskarr);
-
- if (sizeof(off_t) < sizeof(uint64_t)) {
- fprintf(stderr, "Internal inconsistency in off_t type size\n");
- exit(EXIT_FAILURE);
- }
-}
-
-void state_done(struct snapraid_state* state)
-{
- unsigned i;
- for(i=0;idiskarr);++i)
- disk_free(tommy_array_get(&state->diskarr, i));
- tommy_array_done(&state->diskarr);
-}
-
-/****************************************************************************/
-/* config */
-
-void state_config(struct snapraid_state* state, const char* path, int verbose)
-{
- FILE* f;
- unsigned line;
-
- state->verbose = verbose;
-
- f = fopen(path, "rt");
- if (!f) {
- if (errno == ENOENT) {
- fprintf(stderr, "No configuration file found at '%s'\n", path);
- } else if (errno == EACCES) {
- fprintf(stderr, "You do not have rights to access the configuration file '%s'\n", path);
- } else {
- fprintf(stderr, "Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
- }
- exit(EXIT_FAILURE);
- }
-
- line = 0;
- while (1) {
- char buffer[LINE_MAX];
- char* tag;
- char* s;
- int ret;
-
- ++line;
-
- ret = strgets(buffer, LINE_MAX, f);
- if (ret < 0) {
- fprintf(stderr, "Error reading the configuration file '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- if (ret == 0)
- break;
-
- /* start */
- s = buffer;
- s = strskip(s);
-
- /* ignore commens and empty lines */
- if (*s == '#' || *s == 0)
- continue;
-
- tag = s;
- s = strtoken(s);
-
- if (strcmp(tag, "block_size") == 0) {
- ret = stru(s, &state->block_size);
- if (ret != 0) {
- fprintf(stderr, "Invalid 'block_size' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- if (state->block_size < 1) {
- fprintf(stderr, "Too small 'block_size' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- if (state->block_size > 16*1024) {
- fprintf(stderr, "Too big 'block_size' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- /* check if it's a power of 2 */
- if ((state->block_size & (state->block_size - 1)) != 0) {
- fprintf(stderr, "Not power of 2 'block_size' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- state->block_size *= 1024;
- } else if (strcmp(tag, "parity") == 0) {
- if (*state->parity) {
- fprintf(stderr, "Multiple 'parity' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- snprintf(state->parity, sizeof(state->parity), "%s", s);
- } else if (strcmp(tag, "content") == 0) {
- if (*state->content) {
- fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- snprintf(state->content, sizeof(state->content), "%s", s);
- } else if (strcmp(tag, "disk") == 0) {
- char* name = s;
- s = strtoken(s);
- tommy_array_insert(&state->diskarr, disk_alloc(name, s));
- } else {
- fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path);
- exit(EXIT_FAILURE);
- }
- }
-
- fclose(f);
-
- if (!state->parity) {
- fprintf(stderr, "No 'parity' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- if (!state->content) {
- fprintf(stderr, "No 'content' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-}
-
-/****************************************************************************/
-/* read */
-
-/**
- * Read the state.
- */
-void state_read(struct snapraid_state* state)
-{
- FILE* f;
- char path[PATH_MAX];
- struct snapraid_disk* disk;
- struct snapraid_file* file;
- unsigned block_index;
- unsigned line;
- unsigned count_file;
- unsigned count_block;
-
- count_file = 0;
- count_block = 0;
-
- snprintf(path, sizeof(path), "%s", state->content);
- f = fopen(state->content, "r");
- if (!f) {
- /* if not found, assume empty */
- if (errno == ENOENT)
- return;
-
- fprintf(stderr, "Error opening the state file '%s'\n", path);
- exit(EXIT_FAILURE);
- }
-
- printf("Loading state...\n");
-
- disk = 0;
- file = 0;
- line = 0;
- block_index = 0;
- while (1) {
- char buffer[LINE_MAX];
- char* tag;
- char* s;
- int ret;
-
- ++line;
-
- ret = strgets(buffer, LINE_MAX, f);
- if (ret < 0) {
- fprintf(stderr, "Error reading the state file '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- if (ret == 0)
- break;
-
- /* start */
- s = buffer;
- s = strskip(s);
-
- /* ignore commens and empty lines */
- if (*s == '#' || *s == 0)
- continue;
-
- tag = s;
- s = strtoken(s);
-
- if (strcmp(tag, "file") == 0) {
- char* name;
- char* size;
- char* mtime;
- char* sub;
- uint64_t v_size;
- uint64_t v_mtime;
- unsigned i;
-
- if (file) {
- fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- name = s;
- s = strtoken(s);
- size = s;
- s = strtoken(s);
- mtime = s;
- s = strtoken(s);
- sub = s;
-
- ret = stru64(size, &v_size);
- if (ret != 0) {
- fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- ret = stru64(mtime, &v_mtime);
- if (ret != 0) {
- fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- if (!*sub) {
- fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- /* find the disk */
- for(i=0;idiskarr);++i) {
- disk = tommy_array_get(&state->diskarr, i);
- if (strcmp(disk->name, name) == 0)
- break;
- }
- if (i == tommy_array_size(&state->diskarr)) {
- fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", name, path, line);
- exit(EXIT_FAILURE);
- }
-
- /* allocate the file */
- file = file_alloc(state->block_size, sub, v_size, v_mtime);
-
- /* insert the file in the file containers */
- tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub));
- tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
-
- /* start the block allocation of the file */
- block_index = 0;
-
- /* check for empty file */
- if (block_index == file->blockmax) {
- file = 0;
- disk = 0;
- }
-
- /* stat */
- ++count_file;
- } else if (strcmp(tag, "blk") == 0) {
- char* pos;
- char* hash;
- pos_t v_pos;
- struct snapraid_block* block;
- char* e;
-
- if (!file) {
- fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- pos = s;
- s = strtoken(s);
- hash = s;
-
- ret = stru(pos, &v_pos);
- if (ret != 0) {
- fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- if (block_index >= file->blockmax) {
- fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- block = &file->blockvec[block_index];
-
- if (block->parity_pos != POS_INVALID) {
- fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- block->parity_pos = v_pos;
-
- /* set the hash only if present */
- if (*hash != 0) {
- e = strdechex(block->hash, HASH_MAX, hash);
- if (e) {
- fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
- block->is_hashed = 1;
- }
-
- /* insert the block in the block array */
- tommy_array_grow(&disk->blockarr, block->parity_pos + 1);
- tommy_array_set(&disk->blockarr, block->parity_pos, block);
-
- /* check for termination of the block list */
- ++block_index;
- if (block_index == file->blockmax) {
- file = 0;
- disk = 0;
- }
-
- /* stat */
- ++count_block;
- } else {
- fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line);
- exit(EXIT_FAILURE);
- }
- }
-
- if (file) {
- fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line);
- exit(EXIT_FAILURE);
- }
-
- fclose(f);
-
- if (state->verbose) {
- printf("\tfile %u\n", count_file);
- printf("\tblock %u\n", count_block);
- }
-}
-
-/****************************************************************************/
-/* write */
-
-/**
- * Write the new state.
- */
-void state_write(struct snapraid_state* state)
-{
- FILE* f;
- char path[PATH_MAX];
- unsigned i;
- unsigned count_file;
- unsigned count_block;
-
- count_file = 0;
- count_block = 0;
-
- printf("Saving state...\n");
-
- snprintf(path, sizeof(path), "%s.tmp", state->content);
- f = fopen(path, "w");
- if (!f) {
- fprintf(stderr, "Error opening for writing the state file '%s'\n", path);
- exit(EXIT_FAILURE);
- }
-
- /* for each disk */
- for(i=0;idiskarr);++i) {
- tommy_node* j;
- struct snapraid_disk* disk = tommy_array_get(&state->diskarr, i);
-
- /* for each file */
- for(j=disk->filelist;j!=0;j=j->next) {
- pos_t k;
- struct snapraid_file* file = j->data;
- int ret;
-
- ret = fprintf(f,"file %s %lld %ld %s\n", disk->name, file->size, file->mtime, file->sub);
- if (ret < 0) {
- fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- /* for each block */
- for(k=0;kblockmax;++k) {
- struct snapraid_block* block = &file->blockvec[k];
-
- if (block->is_hashed) {
- char s_hash[HASH_MAX*2+1];
- strenchex(s_hash, block->hash, HASH_MAX);
- s_hash[HASH_MAX*2] = 0;
- ret = fprintf(f, "blk %u %s\n", block->parity_pos, s_hash);
- } else {
- ret = fprintf(f, "blk %u\n", block->parity_pos);
- }
- if (ret < 0) {
- fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- ++count_block;
- }
-
- ++count_file;
- }
- }
-
- /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
- /* than even in a system crash event we have one valid copy of the file. */
-
- if (fflush(f) != 0) {
- fprintf(stderr, "Error writing the state file '%s', in fflush(). %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- if (fsync(fileno(f)) != 0) {
- fprintf(stderr, "Error writing the state file '%s' in fsync(). %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- if (fclose(f) != 0) {
- fprintf(stderr, "Error writing the state file '%s' in close(). %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- if (rename(path, state->content) != 0) {
- fprintf(stderr, "Error renaming the state file '%s' to '%s' in rename(). %s.\n", path, state->content, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- if (state->verbose) {
- printf("\tfile %u\n", count_file);
- printf("\tblock %u\n", count_block);
- }
-}
-
-/****************************************************************************/
-/* scan */
-
-struct snapraid_scan {
- unsigned count_equal;
- unsigned count_change;
- unsigned count_remove;
- unsigned count_insert;
-};
-
-/**
- * Remove the specified file from the data set.
- */
-static void state_scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
-{
- pos_t i;
-
- /* free all the blocks of the file */
- for(i=0;iblockmax;++i) {
- pos_t block_pos = file->blockvec[i].parity_pos;
-
- /* adjust the first free position */
- if (disk->first_free_block > block_pos)
- disk->first_free_block = block_pos;
-
- tommy_array_set(&disk->blockarr, block_pos, 0);
- }
-
- /* remove the file from the file containers */
- tommy_hashdyn_remove_existing(&disk->fileset, &file->nodeset);
- tommy_list_remove_existing(&disk->filelist, &file->nodelist);
-
- /* deallocate */
- file_free(file);
-}
-
-/**
- * Insert the specified file in the data set.
- */
-static void state_scan_file_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
-{
- pos_t i;
- pos_t block_max;
- pos_t block_pos;
-
- /* allocate the blocks of the file */
- block_pos = disk->first_free_block;
- block_max = tommy_array_size(&disk->blockarr);
- for(i=0;iblockmax;++i) {
- /* find a free block */
- while (block_pos < block_max && tommy_array_get(&disk->blockarr, block_pos) != 0)
- ++block_pos;
-
- /* if not found, allocate a new one */
- if (block_pos == block_max) {
- ++block_max;
- tommy_array_grow(&disk->blockarr, block_max);
- }
-
- /* set it */
- file->blockvec[i].parity_pos = block_pos;
- tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]);
- }
- if (file->blockmax) {
- /* set the new free position, but only if allocated something */
- disk->first_free_block = block_pos + 1;
- }
-
- /* insert the file in the file containers */
- tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub));
- tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
-}
-
-static void state_scan_file(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* path, const char* sub)
-{
- struct stat st;
- struct snapraid_file* file;
-
- if (stat(path, &st) != 0) {
- fprintf(stderr, "Error in stat file '%s'\n", path);
- exit(EXIT_FAILURE);
- }
-
- /* check if the file already exists */
- file = tommy_hashdyn_search(&disk->fileset, file_compare, sub, file_hash(sub));
- if (file) {
- /* check if the file is the same */
- if (file->size == st.st_size && file->mtime == st.st_mtime) {
- /* mark as present */
- ++scan->count_equal;
- file->is_present = 1;
- return;
- } else {
- /* remove and reinsert it */
- ++scan->count_change;
- --scan->count_insert;
- state_scan_file_remove(state, disk, file);
- /* continue to insert it */
- }
- }
-
- /* create the new file */
- ++scan->count_insert;
- file = file_alloc(state->block_size, sub, st.st_size, st.st_mtime);
- file->is_present = 1;
-
- /* insert it */
- state_scan_file_insert(state, disk, file);
-}
-
-static void state_scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* dir, const char* sub)
-{
- DIR* d;
- struct dirent* dd;
-
- d = opendir(dir);
- if (!d) {
- fprintf(stderr, "Error accessing directory '%s'. %s.\n", dir, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- while ((dd = readdir(d)) != 0) {
- if (dd->d_type == DT_REG) {
- char path_next[PATH_MAX];
- char sub_next[PATH_MAX];
- snprintf(path_next, sizeof(path_next), "%s%s", dir, dd->d_name);
- snprintf(sub_next, sizeof(sub_next), "%s%s", sub, dd->d_name);
- state_scan_file(scan, state, disk, path_next, sub_next);
- } else if (dd->d_type == DT_DIR) {
- if (strcmp(dd->d_name, ".") != 0 && strcmp(dd->d_name, "..") != 0 && strcmp(dd->d_name, "lost+found") != 0
- ) {
- char dir_next[PATH_MAX];
- char sub_next[PATH_MAX];
- snprintf(dir_next, sizeof(dir_next), "%s%s/", dir, dd->d_name);
- snprintf(sub_next, sizeof(sub_next), "%s%s/", sub, dd->d_name);
- state_scan_dir(scan, state, disk, dir_next, sub_next);
- }
- } else {
- if (state->verbose) {
- printf("warning: File '%s/%s' not processed\n", dir, dd->d_name);
- }
- }
- }
-
- closedir(d);
-}
-
-/**
- * Update the internal state to represent the new state.
- */
-void state_scan(struct snapraid_state* state)
-{
- unsigned disk_count = tommy_array_size(&state->diskarr);
- unsigned i;
- struct snapraid_scan scan;
-
- scan.count_equal = 0;
- scan.count_change = 0;
- scan.count_remove = 0;
- scan.count_insert = 0;
-
- for(i=0;idiskarr, i);
- tommy_node* node;
-
- printf("Scanning disk %s...\n", disk->name);
-
- state_scan_dir(&scan, state, disk, disk->dir, "");
-
- /* check for removed files */
- node = disk->filelist;
- while (node) {
- struct snapraid_file* file = node->data;
-
- /* next node */
- node = node->next;
-
- /* remove if not present */
- if (!file->is_present) {
- state_scan_file_remove(state, disk, file);
- ++scan.count_remove;
- }
- }
- }
-
- if (state->verbose) {
- printf("\tequal %d\n", scan.count_equal);
- printf("\tchanged %d\n", scan.count_change);
- printf("\tremoved %d\n", scan.count_remove);
- printf("\tadded %d\n", scan.count_insert);
- }
-}
-
-/****************************************************************************/
-/* common */
-
-/**
- * Computes the size of the parity data in number of blocks.
- */
-static pos_t state_sync_resize(struct snapraid_state* state)
-{
- unsigned disk_count = tommy_array_size(&state->diskarr);
- pos_t parity_block;
- unsigned i;
-
- /* compute the size of the parity file */
- parity_block = 0;
- for(i=0;idiskarr, i);
-
- /* start from the declared size */
- pos_t block = tommy_array_size(&disk->blockarr);
-
- /* decrease the block until an allocated block */
- while (block > 0 && tommy_array_get(&disk->blockarr, block - 1) == 0)
- --block;
-
- if (i == 0 || parity_block < block)
- parity_block = block;
- }
-
- return parity_block;
-}
-
-static void state_xor_block(unsigned char* xor, const unsigned char* block, unsigned size)
-{
- while (size >= 16) {
- uint32_t* xor32 = (uint32_t*)xor;
- const uint32_t* block32 = (const uint32_t*)block;
- xor32[0] ^= block32[0];
- xor32[1] ^= block32[1];
- xor32[2] ^= block32[2];
- xor32[3] ^= block32[3];
-
- xor += 16;
- block += 16;
- size -= 16;
- }
-
- while (size >= 4) {
- uint32_t* xor32 = (uint32_t*)xor;
- const uint32_t* block32 = (const uint32_t*)block;
- xor32[0] ^= block32[0];
-
- xor += 4;
- block += 4;
- size -= 4;
- }
-
- while (size != 0) {
- xor[0] ^= block[0];
-
- xor += 1;
- block += 1;
- size -= 1;
- }
-}
-
-struct snapraid_sync_block {
- char path[PATH_MAX]; /**< Path of the file. */
- struct snapraid_disk* disk; /**< Disk of the file. */
- struct snapraid_file* file; /**< File opened. */
- int f; /**< Handle of the file. */
- off_t offset; /**< Position in the file. */
-};
-
-static int state_sync_read_block(int ret_on_error, struct snapraid_sync_block* sync, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size)
-{
- ssize_t read_ret;
- off_t offset;
- unsigned read_size;
-
- if (sync->file != block->file) {
- if (sync->f != -1) {
- int ret;
- ret = close(sync->f);
- if (ret != 0) {
- /* here we always abort, as close is supposed to never fail */
- fprintf(stderr, "Error closing file '%s'. %s.\n", sync->file->sub, strerror(errno));
- exit(EXIT_FAILURE);
- }
- }
-
- /* invalidate for error */
- sync->file = 0;
- sync->f = -1;
-
- snprintf(sync->path, sizeof(sync->path), "%s%s", sync->disk->dir, block->file->sub);
- sync->f = open(sync->path, O_RDONLY | O_BINARY);
- if (sync->f == -1) {
- if (ret_on_error)
- return ret_on_error;
- fprintf(stderr, "Error opening file '%s'. %s.\n", sync->path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- sync->file = block->file;
- sync->offset = 0;
- }
-
- offset = block_file_pos(block) * (off_t)block_size;
-
- if (sync->offset != offset) {
- off_t ret;
-
- /* invalidate for error */
- sync->offset = -1;
-
- ret = lseek(sync->f, offset, SEEK_SET);
- if (ret != offset) {
- if (ret_on_error)
- return ret_on_error;
- fprintf(stderr, "Error seeking file '%s'. %s.\n", sync->path, strerror(errno));
- exit(EXIT_FAILURE);
- }
- sync->offset = offset;
- }
-
- read_size = block_file_size(block, block_size);
-
- read_ret = read(sync->f, block_buffer, read_size);
- if (read_ret != read_size) {
- if (ret_on_error)
- return ret_on_error;
- fprintf(stderr, "Error reading file '%s'. %s.\n", sync->path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- sync->offset += read_size;
-
- return read_size;
-}
-
-static void state_sync_write_parity(const char* parity_path, int parity_f, off_t* parity_offset, pos_t pos, unsigned char* block_buffer, unsigned block_size)
-{
- ssize_t write_ret;
- off_t offset;
-
- offset = pos * (off_t)block_size;
-
- if (*parity_offset != offset) {
- off_t ret;
- ret = lseek(parity_f, offset, SEEK_SET);
- if (ret != offset) {
- fprintf(stderr, "Error seeking file '%s'. %s.\n", parity_path, strerror(errno));
- exit(EXIT_FAILURE);
- }
- *parity_offset = offset;
- }
-
- write_ret = write(parity_f, block_buffer, block_size);
- if (write_ret != block_size) {
- if (errno == ENOSPC) {
- fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", parity_path);
- } else {
- fprintf(stderr, "Error writing file '%s'. %s.\n", parity_path, strerror(errno));
- }
- exit(EXIT_FAILURE);
- }
-
- *parity_offset += block_size;
-}
-
-static int state_sync_read_parity(int ret_on_error, const char* parity_path, int parity_f, off_t* parity_offset, pos_t pos, unsigned char* block_buffer, unsigned block_size)
-{
- ssize_t read_ret;
- off_t offset;
-
- offset = pos * (off_t)block_size;
-
- if (*parity_offset != offset) {
- off_t ret;
-
- /* invalidate for error */
- *parity_offset = -1;
-
- ret = lseek(parity_f, offset, SEEK_SET);
- if (ret != offset) {
- if (ret_on_error)
- return ret_on_error;
- fprintf(stderr, "Error seeking file '%s'. %s.\n", parity_path, strerror(errno));
- exit(EXIT_FAILURE);
- }
- *parity_offset = offset;
- }
-
- read_ret = read(parity_f, block_buffer, block_size);
- if (read_ret != block_size) {
- if (ret_on_error)
- return ret_on_error;
- fprintf(stderr, "Error reading file '%s'. %s.\n", parity_path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- *parity_offset += block_size;
-
- return block_size;
-}
-
-/****************************************************************************/
-/* sync */
-
-static void state_sync_process(struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax)
-{
- struct snapraid_sync_block* sync;
- off_t parity_offset;
- unsigned diskmax = tommy_array_size(&state->diskarr);
- pos_t i;
- unsigned char* block_buffer;
- unsigned char* xor_buffer;
- int ret;
- uint64_t count_size;
- time_t start;
- time_t last;
-
- block_buffer = malloc_nofail(state->block_size);
- xor_buffer = malloc_nofail(state->block_size);
-
- sync = malloc_nofail(diskmax * sizeof(struct snapraid_sync_block));
- for(i=0;idiskarr, i);
- sync[i].file = 0;
- sync[i].f = -1;
- }
- parity_offset = -1;
-
- start = time(0);
- last = start;
- count_size = 0;
-
- for(i=blockstart;iis_hashed) {
- unhashed = 1;
- break;
- }
- }
-
- if (!unhashed)
- continue;
-
- /* start with 0 */
- memset(xor_buffer, 0, state->block_size);
-
- /* for each disk, process the block */
- for(j=0;jblock_size);
-
- /* now compute the hash */
- md5_init(&md5);
- md5_update(&md5, block_buffer, read_size);
- md5_final(&md5, hash);
-
- if (block->is_hashed) {
- /* compare the hash */
- if (memcmp(hash, block->hash, HASH_MAX) != 0) {
- fprintf(stderr, "Data error for file %s at position %u\n", block->file->sub, block_file_pos(block));
- fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u fix'\n", block_file_pos(block));
- exit(EXIT_FAILURE);
- }
- } else {
- /* copy the hash */
- memcpy(block->hash, hash, HASH_MAX);
-
- /* mark the block as hashed */
- block->is_hashed = 1;
- }
-
- /* compute the parity */
- state_xor_block(xor_buffer, block_buffer, read_size);
-
- count_size += read_size;
- }
-
- /* write the parity */
- state_sync_write_parity(state->parity, parity_f, &parity_offset, i, xor_buffer, state->block_size);
-
- /* progress */
- now = time(0);
- if (last != now) {
- printf("%u%%, %u MB", i * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
- if (now != start) {
- printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / (now - start)));
- }
- printf("\r");
- fflush(stdout);
- last = now;
- }
-
- /* stop if requested */
- if (global_interrupt) {
- printf("\rStopping at block %u\n", i);
- break;
- }
- }
-
- printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
-
- printf("Done\n");
-
- for(i=0;isub, strerror(errno));
- exit(EXIT_FAILURE);
- }
- }
- }
-
- free(sync);
- free(block_buffer);
- free(xor_buffer);
-}
-
-/**
- * Update the parity file.
- */
-void state_sync(struct snapraid_state* state, pos_t blockstart)
-{
- char path[PATH_MAX];
- pos_t blockmax;
- off_t size;
- int ret;
- int f;
- struct stat st;
-
- printf("Syncing...\n");
-
- blockmax = state_sync_resize(state);
- size = blockmax * (off_t)state->block_size;
-
- if (blockstart >= blockmax) {
- fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax);
- exit(EXIT_FAILURE);
- }
-
- snprintf(path, sizeof(path), "%s", state->parity);
- f = open(path, O_RDWR | O_CREAT, 0600);
- if (f == -1) {
- fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- ret = fstat(f, &st);
- if (ret != 0) {
- fprintf(stderr, "Error accessing parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- if (st.st_size < size) {
-#if HAVE_FALLOCATE
- /* allocate real space. This is Linux specific. */
- ret = fallocate(f, 0, 0, size);
-#else
- /* allocate using a sparse file */
- ret = ftruncate(f, size);
-#endif
- if (ret != 0) {
- if (errno == ENOSPC) {
- fprintf(stderr, "Failed to grow parity file '%s' due lack of space.\n", path);
- } else {
- fprintf(stderr, "Error growing parity file '%s'. %s.\n", path, strerror(errno));
- }
- exit(EXIT_FAILURE);
- }
- } else {
- ret = ftruncate(f, size);
- if (ret != 0) {
- fprintf(stderr, "Error truncating parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
- }
-
- /* ensure that the file size change is written to disk */
- /* this is not really required, but just to avoid to result in a wrong sized file */
- /* in case of a system crash */
- ret = fsync(f);
- if (ret != 0) {
- fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- state_sync_process(state, f, blockstart, blockmax);
-
- /* ensure that data changes are written to disk */
- /* this is required to ensure that parity is more updated than content */
- /* in case of a system crash */
- ret = fsync(f);
- if (ret != 0) {
- fprintf(stderr, "Error synching parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- ret = close(f);
- if (ret != 0) {
- fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-}
-
-/****************************************************************************/
-/* check */
-
-
-static int state_check_process(int ret_on_error, struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax)
-{
- struct snapraid_sync_block* sync;
- off_t parity_offset;
- unsigned diskmax = tommy_array_size(&state->diskarr);
- pos_t i;
- unsigned char* block_buffer;
- unsigned char* xor_buffer;
- int ret;
- uint64_t count_size;
- time_t start;
- time_t last;
- unsigned error;
- unsigned unrecoverable_error;
-
- block_buffer = malloc_nofail(state->block_size);
- xor_buffer = malloc_nofail(state->block_size);
-
- sync = malloc_nofail(diskmax * sizeof(struct snapraid_sync_block));
- for(i=0;idiskarr, i);
- sync[i].file = 0;
- sync[i].f = -1;
- }
- parity_offset = -1;
- error = 0;
- unrecoverable_error = 0;
-
- start = time(0);
- last = start;
- count_size = 0;
-
- for(i=blockstart;iis_hashed) {
- hashed = 1;
- break;
- }
- }
-
- /* if there is at least an hashed block we have to check */
- if (!hashed)
- continue;
-
- /* start with 0 */
- memset(xor_buffer, 0, state->block_size);
-
- /* for each disk, process the block */
- unhashed = 0;
- failed = 0;
- for(j=0;jis_hashed) {
- unhashed = 1;
- continue;
- }
-
- read_size = state_sync_read_block(-1, &sync[j], block, block_buffer, state->block_size);
-
- if (read_size == -1) {
- ++failed;
-
- /* save the hash for the parity check */
- memcpy(failed_hash, block->hash, HASH_MAX);
- failed_size = block_file_size(block, state->block_size);
-
- fprintf(stderr, "Read error for file %s at position %u\n", block->file->sub, block_file_pos(block));
- ++error;
- } else {
- /* now compute the hash */
- md5_init(&md5);
- md5_update(&md5, block_buffer, read_size);
- md5_final(&md5, hash);
-
- /* compare the hash */
- if (memcmp(hash, block->hash, HASH_MAX) != 0) {
- ++failed;
- fprintf(stderr, "Data error for file %s at position %u\n", block->file->sub, block_file_pos(block));
- ++error;
- }
-
- /* compute the parity */
- state_xor_block(xor_buffer, block_buffer, read_size);
-
- count_size += read_size;
- }
- }
-
- /* only if no unhashed block, we can check for parity */
- if (unhashed)
- continue;
-
- if (failed == 0) {
- int ret;
-
- /* read the parity */
- ret = state_sync_read_parity(-1, state->parity, parity_f, &parity_offset, i, block_buffer, state->block_size);
- if (ret == -1) {
- fprintf(stderr, "Parity read error for position %u\n", i);
- ++error;
- } else {
- /* compare it */
- if (memcmp(xor_buffer, block_buffer, state->block_size) != 0) {
- fprintf(stderr, "Parity data error for position %u\n", i);
- ++error;
- }
- }
- } else if (failed == 1) {
- struct md5_t md5;
- unsigned char hash[HASH_MAX];
- int ret;
-
- /* read the parity */
- ret = state_sync_read_parity(-1, state->parity, parity_f, &parity_offset, i, block_buffer, state->block_size);
- if (ret == -1) {
- fprintf(stderr, "UNRECOVERABLE parity read error for position %u\n", i);
- ++unrecoverable_error;
- }
-
- /* compute the failed block */
- state_xor_block(block_buffer, xor_buffer, state->block_size);
-
- /* now compute the hash */
- md5_init(&md5);
- md5_update(&md5, block_buffer, failed_size);
- md5_final(&md5, hash);
-
- /* compare the hash */
- if (memcmp(hash, failed_hash, HASH_MAX) != 0) {
- fprintf(stderr, "UNRECOVERABLE parity data error for position %u\n", i);
- ++unrecoverable_error;
- }
- } else {
- fprintf(stderr, "UNRECOVERABLE parity missing for position %u\n", i);
- ++unrecoverable_error;
- }
-
- /* progress */
- now = time(0);
- if (last != now) {
- printf("%u%%, %u MB", i * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
- if (now != start) {
- printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / (now - start)));
- }
- printf("\r");
- fflush(stdout);
- last = now;
- }
-
- /* stop if requested */
- if (global_interrupt) {
- printf("\rStopping at block %u\n", i);
- break;
- }
- }
-
- printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
-
- if (error || unrecoverable_error) {
- if (error)
- printf("%u recoverable errors\n", error);
- else
- printf("No recoverable errors\n");
- if (unrecoverable_error)
- printf("%u UNRECOVERABLE errors\n", unrecoverable_error);
- else
- printf("No unrecoverable errors\n");
- } else {
- printf("No error\n");
- }
-
- for(i=0;isub, strerror(errno));
- exit(EXIT_FAILURE);
- }
- }
- }
-
- free(sync);
- free(block_buffer);
- free(xor_buffer);
-
- return unrecoverable_error != 0 ? ret_on_error : 0;
-}
-
-
-/**
- * Check.
- */
-void state_check(struct snapraid_state* state, pos_t blockstart)
-{
- char path[PATH_MAX];
- pos_t blockmax;
- off_t size;
- int ret;
- int f;
-
- printf("Checking...\n");
-
- blockmax = state_sync_resize(state);
- size = blockmax * (off_t)state->block_size;
-
- if (blockstart >= blockmax) {
- fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax);
- exit(EXIT_FAILURE);
- }
-
- snprintf(path, sizeof(path), "%s", state->parity);
- f = open(path, O_RDONLY);
- if (f == -1) {
- fprintf(stderr, "Error opening parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-
- ret = state_check_process(-1, state, f, blockstart, blockmax);
- if (ret == -1) {
- /* no extra message */
- exit(EXIT_FAILURE);
- }
-
- ret = close(f);
- if (ret != 0) {
- fprintf(stderr, "Error closing parity file '%s'. %s.\n", path, strerror(errno));
- exit(EXIT_FAILURE);
- }
-}
+#include "elem.h"
+#include "state.h"
/****************************************************************************/
/* main */
@@ -1556,6 +35,8 @@ void usage(void) {
printf("\n");
printf("Options:\n");
printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file (default /etc/snapraid.conf)\n");
+ printf(" " SWITCH_GETOPT_LONG("-s, --start BLK ", "-s") " Start from the specified block number\n");
+ printf(" " SWITCH_GETOPT_LONG("-f, --force ", "-f") " Force dangerous operations\n");
printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n");
printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n");
printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n");
@@ -1565,6 +46,7 @@ void usage(void) {
struct option long_options[] = {
{ "conf", 1, 0, 'c' },
{ "start", 1, 0, 's' },
+ { "force", 0, 0, 'f' },
{ "verbose", 0, 0, 'v' },
{ "help", 0, 0, 'h' },
{ "version", 0, 0, 'V' },
@@ -1572,7 +54,9 @@ struct option long_options[] = {
};
#endif
-#define OPTIONS "c:s:vhV"
+#define OPTIONS "c:s:fvhV"
+
+volatile int global_interrupt = 0;
void signal_handler(int signal)
{
@@ -1590,7 +74,8 @@ void signal_handler(int signal)
int main(int argc, char* argv[])
{
int c;
- int verbose = 0;
+ int verbose;
+ int force;
const char* conf;
struct snapraid_state state;
int operation;
@@ -1599,6 +84,7 @@ int main(int argc, char* argv[])
/* defaults */
conf = "/etc/snapraid.conf";
verbose = 0;
+ force = 0;
blockstart = 0;
opterr = 0;
@@ -1619,6 +105,9 @@ int main(int argc, char* argv[])
exit(EXIT_FAILURE);
}
break;
+ case 'f' :
+ force = 1;
+ break;
case 'v' :
verbose = 1;
break;
@@ -1658,7 +147,7 @@ int main(int argc, char* argv[])
state_init(&state);
- state_config(&state, conf, verbose);
+ state_config(&state, conf, verbose, force);
if (operation == OPERATION_SYNC) {
state_read(&state);
@@ -1668,10 +157,10 @@ int main(int argc, char* argv[])
state_sync(&state, blockstart);
state_write(&state);
- } else if (operation == OPERATION_CHECK) {
+ } else if (operation == OPERATION_CHECK || operation == OPERATION_FIX) {
state_read(&state);
- state_check(&state, blockstart);
+ state_check(&state, operation == OPERATION_FIX, blockstart);
}
state_done(&state);
diff --git a/state.c b/state.c
new file mode 100644
index 00000000..73429315
--- /dev/null
+++ b/state.c
@@ -0,0 +1,490 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "elem.h"
+#include "state.h"
+#include "util.h"
+
+/**
+ * Max length of a line in the configuration and state files.
+ */
+#define LINE_MAX 1024
+
+void state_init(struct snapraid_state* state)
+{
+ state->verbose = 0;
+ state->block_size = 128*1024; /* default 128 kB */
+ state->content[0] = 0;
+ state->parity[0] = 0;
+ tommy_array_init(&state->diskarr);
+
+ if (sizeof(off_t) < sizeof(uint64_t)) {
+ fprintf(stderr, "Internal inconsistency in off_t type size\n");
+ exit(EXIT_FAILURE);
+ }
+}
+
+void state_done(struct snapraid_state* state)
+{
+ unsigned i;
+ for(i=0;idiskarr);++i)
+ disk_free(tommy_array_get(&state->diskarr, i));
+ tommy_array_done(&state->diskarr);
+}
+
+void state_config(struct snapraid_state* state, const char* path, int verbose, int force)
+{
+ FILE* f;
+ unsigned line;
+
+ state->verbose = verbose;
+ state->force = force;
+
+ f = fopen(path, "rt");
+ if (!f) {
+ if (errno == ENOENT) {
+ fprintf(stderr, "No configuration file found at '%s'\n", path);
+ } else if (errno == EACCES) {
+ fprintf(stderr, "You do not have rights to access the configuration file '%s'\n", path);
+ } else {
+ fprintf(stderr, "Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
+ }
+ exit(EXIT_FAILURE);
+ }
+
+ line = 0;
+ while (1) {
+ char buffer[LINE_MAX];
+ char* tag;
+ char* s;
+ int ret;
+
+ ++line;
+
+ ret = strgets(buffer, LINE_MAX, f);
+ if (ret < 0) {
+ fprintf(stderr, "Error reading the configuration file '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ if (ret == 0)
+ break;
+
+ /* start */
+ s = buffer;
+ s = strskip(s);
+
+ /* ignore commens and empty lines */
+ if (*s == '#' || *s == 0)
+ continue;
+
+ tag = s;
+ s = strtoken(s);
+
+ if (strcmp(tag, "block_size") == 0) {
+ ret = stru(s, &state->block_size);
+ if (ret != 0) {
+ fprintf(stderr, "Invalid 'block_size' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ if (state->block_size < 1) {
+ fprintf(stderr, "Too small 'block_size' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ if (state->block_size > 16*1024) {
+ fprintf(stderr, "Too big 'block_size' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ /* check if it's a power of 2 */
+ if ((state->block_size & (state->block_size - 1)) != 0) {
+ fprintf(stderr, "Not power of 2 'block_size' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ state->block_size *= 1024;
+ } else if (strcmp(tag, "parity") == 0) {
+ if (*state->parity) {
+ fprintf(stderr, "Multiple 'parity' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ snprintf(state->parity, sizeof(state->parity), "%s", s);
+ } else if (strcmp(tag, "content") == 0) {
+ if (*state->content) {
+ fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ snprintf(state->content, sizeof(state->content), "%s", s);
+ } else if (strcmp(tag, "disk") == 0) {
+ char* name = s;
+ s = strtoken(s);
+ tommy_array_insert(&state->diskarr, disk_alloc(name, s));
+ } else {
+ fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ fclose(f);
+
+ if (!state->parity) {
+ fprintf(stderr, "No 'parity' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ if (!state->content) {
+ fprintf(stderr, "No 'content' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+}
+
+void state_read(struct snapraid_state* state)
+{
+ FILE* f;
+ char path[PATH_MAX];
+ struct snapraid_disk* disk;
+ struct snapraid_file* file;
+ unsigned block_index;
+ unsigned line;
+ unsigned count_file;
+ unsigned count_block;
+
+ count_file = 0;
+ count_block = 0;
+
+ snprintf(path, sizeof(path), "%s", state->content);
+ f = fopen(state->content, "r");
+ if (!f) {
+ /* if not found, assume empty */
+ if (errno == ENOENT)
+ return;
+
+ fprintf(stderr, "Error opening the state file '%s'\n", path);
+ exit(EXIT_FAILURE);
+ }
+
+ printf("Loading state...\n");
+
+ disk = 0;
+ file = 0;
+ line = 0;
+ block_index = 0;
+ while (1) {
+ char buffer[LINE_MAX];
+ char* tag;
+ char* s;
+ int ret;
+
+ ++line;
+
+ ret = strgets(buffer, LINE_MAX, f);
+ if (ret < 0) {
+ fprintf(stderr, "Error reading the state file '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ if (ret == 0)
+ break;
+
+ /* start */
+ s = buffer;
+ s = strskip(s);
+
+ /* ignore commens and empty lines */
+ if (*s == '#' || *s == 0)
+ continue;
+
+ tag = s;
+ s = strtoken(s);
+
+ if (strcmp(tag, "file") == 0) {
+ char* name;
+ char* size;
+ char* mtime;
+ char* sub;
+ uint64_t v_size;
+ uint64_t v_mtime;
+ unsigned i;
+
+ if (file) {
+ fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ name = s;
+ s = strtoken(s);
+ size = s;
+ s = strtoken(s);
+ mtime = s;
+ s = strtoken(s);
+ sub = s;
+
+ ret = stru64(size, &v_size);
+ if (ret != 0) {
+ fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ ret = stru64(mtime, &v_mtime);
+ if (ret != 0) {
+ fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ if (!*sub) {
+ fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ /* find the disk */
+ for(i=0;idiskarr);++i) {
+ disk = tommy_array_get(&state->diskarr, i);
+ if (strcmp(disk->name, name) == 0)
+ break;
+ }
+ if (i == tommy_array_size(&state->diskarr)) {
+ fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", name, path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ /* allocate the file */
+ file = file_alloc(state->block_size, sub, v_size, v_mtime);
+
+ /* insert the file in the file containers */
+ tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub));
+ tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
+
+ /* start the block allocation of the file */
+ block_index = 0;
+
+ /* check for empty file */
+ if (block_index == file->blockmax) {
+ file = 0;
+ disk = 0;
+ }
+
+ /* stat */
+ ++count_file;
+ } else if (strcmp(tag, "blk") == 0) {
+ char* pos;
+ char* hash;
+ pos_t v_pos;
+ struct snapraid_block* block;
+ char* e;
+
+ if (!file) {
+ fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ pos = s;
+ s = strtoken(s);
+ hash = s;
+
+ ret = stru(pos, &v_pos);
+ if (ret != 0) {
+ fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ if (block_index >= file->blockmax) {
+ fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ block = &file->blockvec[block_index];
+
+ if (block->parity_pos != POS_INVALID) {
+ fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ block->parity_pos = v_pos;
+
+ /* set the hash only if present */
+ if (*hash != 0) {
+ e = strdechex(block->hash, HASH_MAX, hash);
+ if (e) {
+ fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+ block->is_hashed = 1;
+ }
+
+ /* insert the block in the block array */
+ tommy_array_grow(&disk->blockarr, block->parity_pos + 1);
+ tommy_array_set(&disk->blockarr, block->parity_pos, block);
+
+ /* check for termination of the block list */
+ ++block_index;
+ if (block_index == file->blockmax) {
+ file = 0;
+ disk = 0;
+ }
+
+ /* stat */
+ ++count_block;
+ } else {
+ fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ if (file) {
+ fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line);
+ exit(EXIT_FAILURE);
+ }
+
+ fclose(f);
+
+ if (state->verbose) {
+ printf("\tfile %u\n", count_file);
+ printf("\tblock %u\n", count_block);
+ }
+}
+
+void state_write(struct snapraid_state* state)
+{
+ FILE* f;
+ char path[PATH_MAX];
+ unsigned i;
+ unsigned count_file;
+ unsigned count_block;
+
+ count_file = 0;
+ count_block = 0;
+
+ printf("Saving state...\n");
+
+ snprintf(path, sizeof(path), "%s.tmp", state->content);
+ f = fopen(path, "w");
+ if (!f) {
+ fprintf(stderr, "Error opening for writing the state file '%s'\n", path);
+ exit(EXIT_FAILURE);
+ }
+
+ /* for each disk */
+ for(i=0;idiskarr);++i) {
+ tommy_node* j;
+ struct snapraid_disk* disk = tommy_array_get(&state->diskarr, i);
+
+ /* for each file */
+ for(j=disk->filelist;j!=0;j=j->next) {
+ pos_t k;
+ struct snapraid_file* file = j->data;
+ int ret;
+
+ ret = fprintf(f,"file %s %lld %ld %s\n", disk->name, file->size, file->mtime, file->sub);
+ if (ret < 0) {
+ fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ /* for each block */
+ for(k=0;kblockmax;++k) {
+ struct snapraid_block* block = &file->blockvec[k];
+
+ if (block->is_hashed) {
+ char s_hash[HASH_MAX*2+1];
+ strenchex(s_hash, block->hash, HASH_MAX);
+ s_hash[HASH_MAX*2] = 0;
+ ret = fprintf(f, "blk %u %s\n", block->parity_pos, s_hash);
+ } else {
+ ret = fprintf(f, "blk %u\n", block->parity_pos);
+ }
+ if (ret < 0) {
+ fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ++count_block;
+ }
+
+ ++count_file;
+ }
+ }
+
+ /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
+ /* than even in a system crash event we have one valid copy of the file. */
+
+ if (fflush(f) != 0) {
+ fprintf(stderr, "Error writing the state file '%s', in fflush(). %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (fsync(fileno(f)) != 0) {
+ fprintf(stderr, "Error writing the state file '%s' in fsync(). %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (fclose(f) != 0) {
+ fprintf(stderr, "Error writing the state file '%s' in close(). %s.\n", path, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (rename(path, state->content) != 0) {
+ fprintf(stderr, "Error renaming the state file '%s' to '%s' in rename(). %s.\n", path, state->content, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ if (state->verbose) {
+ printf("\tfile %u\n", count_file);
+ printf("\tblock %u\n", count_block);
+ }
+}
+
+#define PROGRESS_CLEAR " "
+
+int state_progress(time_t* start, time_t* last, pos_t blockpos, pos_t blockmax, uint64_t count_block, uint64_t count_size)
+{
+ time_t now;
+
+ now = time(0);
+
+ if (*last != now) {
+ time_t delta = now - *start;
+
+ printf("%u%%, %u MB", blockpos * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
+
+ if (delta != 0) {
+ printf(", %u MB/s", (unsigned)(count_size / (1024*1024) / delta));
+ }
+
+ if (delta > 5 && count_block > 0) {
+ unsigned m, h;
+ uint64_t todo = blockmax - blockpos;
+
+ m = todo * delta / (60 * count_block);
+
+ h = m / 60;
+ m = m % 60;
+
+ printf(", %u:%02u ETA%s", h, m, PROGRESS_CLEAR);
+ }
+ printf("\r");
+ fflush(stdout);
+ *last = now;
+ }
+
+
+ /* stop if requested */
+ if (global_interrupt) {
+ printf("\rStopping for interruption%s\n", PROGRESS_CLEAR);
+ return 1;
+ }
+
+ return 0;
+}
+
+
diff --git a/state.h b/state.h
new file mode 100644
index 00000000..cce6f20c
--- /dev/null
+++ b/state.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#ifndef __STATE_H
+#define __STATE_H
+
+/****************************************************************************/
+/* state */
+
+/**
+ * Global variable to identify if Ctrl+C is pressed.
+ */
+extern volatile int global_interrupt;
+
+struct snapraid_state {
+ int verbose; /**< Verbose output. */
+ int force; /**< Forced dangerous operations. */
+ unsigned block_size; /**< Block size in bytes. */
+ char content[PATH_MAX]; /**< Path of the content file. */
+ char parity[PATH_MAX]; /**< Path of the parity file. */
+ tommy_array diskarr; /**< Disk array. */
+};
+
+void state_init(struct snapraid_state* state);
+
+void state_done(struct snapraid_state* state);
+
+void state_config(struct snapraid_state* state, const char* path, int verbose, int force);
+
+/**
+ * Read the state.
+ */
+void state_read(struct snapraid_state* state);
+
+/**
+ * Write the new state.
+ */
+void state_write(struct snapraid_state* state);
+
+/**
+ * Update the internal state to represent the new state.
+ */
+void state_scan(struct snapraid_state* state);
+
+/**
+ * Sync.
+ */
+void state_sync(struct snapraid_state* state, pos_t blockstart);
+
+/**
+ * Check.
+ */
+void state_check(struct snapraid_state* state, int fix, pos_t blockstart);
+
+int state_progress(time_t* start, time_t* last, pos_t blockpos, pos_t blockmax, uint64_t count_block, uint64_t count_size);
+
+#endif
diff --git a/sync.c b/sync.c
new file mode 100644
index 00000000..67458a1d
--- /dev/null
+++ b/sync.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "util.h"
+#include "elem.h"
+#include "state.h"
+#include "parity.h"
+#include "handle.h"
+
+/****************************************************************************/
+/* sync */
+
+static void state_sync_process(struct snapraid_state* state, int parity_f, pos_t blockstart, pos_t blockmax)
+{
+ struct snapraid_handle* handle;
+ unsigned diskmax = tommy_array_size(&state->diskarr);
+ pos_t i;
+ unsigned j;
+ unsigned char* block_buffer;
+ unsigned char* xor_buffer;
+ uint64_t count_size;
+ uint64_t count_block;
+ time_t start;
+ time_t last;
+
+ block_buffer = malloc_nofail(state->block_size);
+ xor_buffer = malloc_nofail(state->block_size);
+
+ handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle));
+ for(i=0;idiskarr, i);
+ handle[i].file = 0;
+ handle[i].f = -1;
+ }
+
+ count_size = 0;
+ count_block = 0;
+ start = time(0);
+ last = start;
+
+ for(i=blockstart;iis_hashed) {
+ unhashed = 1;
+ break;
+ }
+ }
+
+ if (!unhashed)
+ continue;
+
+ /* start with 0 */
+ memset(xor_buffer, 0, state->block_size);
+
+ /* for each disk, process the block */
+ for(j=0;jfile);
+
+ read_size = handle_read(0, &handle[j], block, block_buffer, state->block_size);
+
+ /* now compute the hash */
+ md5_init(&md5);
+ md5_update(&md5, block_buffer, read_size);
+ md5_final(&md5, hash);
+
+ if (block->is_hashed) {
+ /* compare the hash */
+ if (memcmp(hash, block->hash, HASH_MAX) != 0) {
+ fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
+ fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u fix'\n", i);
+ exit(EXIT_FAILURE);
+ }
+ } else {
+ /* copy the hash */
+ memcpy(block->hash, hash, HASH_MAX);
+
+ /* mark the block as hashed */
+ block->is_hashed = 1;
+ }
+
+ /* compute the parity */
+ memxor(xor_buffer, block_buffer, read_size);
+
+ count_size += read_size;
+ }
+
+ /* write the parity */
+ parity_write(state->parity, parity_f, i, xor_buffer, state->block_size);
+
+ /* count the number of processed block */
+ ++count_block;
+
+ /* progress */
+ if (state_progress(&start, &last, i, blockmax, count_block, count_size))
+ break;
+ }
+
+ printf("%u%% completed, %u MB processed\n", i * 100 / blockmax, (unsigned)(count_size / (1024*1024)));
+
+ printf("Done\n");
+
+ for(j=0;jblock_size;
+
+ if (blockstart >= blockmax) {
+ fprintf(stderr, "The specified starting block %u is bigger than the parity size %u.\n", blockstart, blockmax);
+ exit(EXIT_FAILURE);
+ }
+
+ snprintf(path, sizeof(path), "%s", state->parity);
+ f = parity_create(path, size);
+
+ state_sync_process(state, f, blockstart, blockmax);
+
+ parity_close(path, f);
+}
diff --git a/tommy.c b/tommy.c
index 692a0f01..ec8a8a1f 100644
--- a/tommy.c
+++ b/tommy.c
@@ -17,13 +17,13 @@
#include "portable.h"
-/* redefine a safe malloc for tommy use */
+#include "util.h" /* for safe_malloc */
+
+/* redefine the malloc for tommy use */
#define tommy_malloc malloc_nofail
#define tommy_free free
-void* malloc_nofail(size_t size);
-
-/* direct include tommy source */
+/* include tommy source */
#include "tommylist.c"
#include "tommyhash.c"
#include "tommyhashdyn.c"
diff --git a/util.c b/util.c
new file mode 100644
index 00000000..e415fe2c
--- /dev/null
+++ b/util.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "util.h"
+
+void* malloc_nofail(size_t size)
+{
+ void* ptr = malloc(size);
+
+ if (!ptr) {
+ fprintf(stderr, "Low memory\n");
+ exit(EXIT_FAILURE);
+ }
+
+ return ptr;
+}
+
+void memxor(unsigned char* xor, const unsigned char* block, unsigned size)
+{
+ while (size >= 16) {
+ uint32_t* xor32 = (uint32_t*)xor;
+ const uint32_t* block32 = (const uint32_t*)block;
+ xor32[0] ^= block32[0];
+ xor32[1] ^= block32[1];
+ xor32[2] ^= block32[2];
+ xor32[3] ^= block32[3];
+
+ xor += 16;
+ block += 16;
+ size -= 16;
+ }
+
+ while (size >= 4) {
+ uint32_t* xor32 = (uint32_t*)xor;
+ const uint32_t* block32 = (const uint32_t*)block;
+ xor32[0] ^= block32[0];
+
+ xor += 4;
+ block += 4;
+ size -= 4;
+ }
+
+ while (size != 0) {
+ xor[0] ^= block[0];
+
+ xor += 1;
+ block += 1;
+ size -= 1;
+ }
+}
+
diff --git a/util.h b/util.h
index e59e14e0..380a9a1d 100644
--- a/util.h
+++ b/util.h
@@ -15,6 +15,9 @@
* along with this program. If not, see .
*/
+#ifndef __UTIL_H
+#define __UTIL_H
+
/****************************************************************************/
/* string */
@@ -168,3 +171,19 @@ static inline char* strdechex(void* void_data, unsigned data_len, char* str)
return 0;
}
+/****************************************************************************/
+/* memory */
+
+/**
+ * Safe malloc.
+ * If no memory is available, it aborts.
+ */
+void* malloc_nofail(size_t size);
+
+/**
+ * Xor two memory blocks.
+ */
+void memxor(unsigned char* xor, const unsigned char* block, unsigned size);
+
+#endif
+