diff --git a/HISTORY b/HISTORY index 1e8b5e75..330f0ba3 100644 --- a/HISTORY +++ b/HISTORY @@ -1,22 +1,26 @@ SnapRAID HISTORY ================ -4.0 2013/6 +4.0 2013/7 ========== - * Better selection of the RAID6 implementation for different CPUs. + * New 'scrub' command to periodically check the oldest blocks for silent + errors without the need to scan the whole array. + * New 'status' command to check the fragmentation, the last check time + distribution, and the silent error status of the array. * Added the new Spooky hash. It's faster in 64 bit architectures. To convert you can use the new 'rehash' command. + * Added a hash seed to make harder intentional collision attacks. + * When insering files for the first time, sort them by their physical + address to improve read performance. + * Better selection of the RAID6 implementation for different CPUs. * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4. They are a little faster than the previous unroll by 2. - * Added a hash seed to make harder intentional collision attacks. * Added a lock file to avoid multiple running instances on the same array. The file is named as parity file adding the .lock extension. There is also the undocumented --test-skip-lock to avoid to check it. * Automatically ignores, with warning, mount points inside the array directory tree. * Changes the 'dup' output format to include the size of each duplicate file. - * When insering files for the first time, sort them by their physical - address to improve read performance. 3.1 2013/5 ========== diff --git a/Makefile.am b/Makefile.am index 936f3266..ddfc53ac 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,6 +12,8 @@ snapraid_SOURCES = \ check.c \ dry.c \ rehash.c \ + scrub.c \ + status.c \ dup.c \ pool.c \ parity.c \ @@ -103,7 +105,9 @@ check-local: echo --- As empty, sync and check forcing murmur3 $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf --test-force-murmur3 sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status if HAVE_SYMLINK $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool endif @@ -114,6 +118,7 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf --test-force-order-physical sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status if HAVE_SYMLINK $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool endif @@ -140,6 +145,7 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf dup $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-skip-fallocate -v -c $(srcdir)/test-raid5.conf --test-force-order-inode sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-skip-sequential -c $(srcdir)/test-raid5.conf check if HAVE_SYMLINK $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool @@ -417,6 +423,7 @@ endif echo --- Sync after all the fixes $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status echo Success! # Manual testing diff --git a/TODO b/TODO index 4e166b48..839ea586 100644 --- a/TODO +++ b/TODO @@ -130,16 +130,3 @@ Use the "snapraid -T" comment to see the speed. Also, all the file operations are already done in background by the OS, so no improvement is expect from this side. -* The SSE2 RAID6 computation may be slower than the MMX version. -See for example this case on a AMD Zacate APU E-350 1.6GHz: -memset 1812 [MB/s] -MD5 252 [MB/s] -Murmur3 1116 [MB/s] -RAID5 int32x2 1692 [MB/s] -RAID5 mmxx2 1745 [MB/s] -RAID5 sse2x2 2481 [MB/s] -RAID6 int32x2 807 [MB/s] -RAID6 mmxx2 1304 [MB/s] -RAID6 sse2x2 787 [MB/s] -- It happens in some systems. At now not an high priority issue. - diff --git a/check.c b/check.c index 7d7b39ab..d04b3d16 100644 --- a/check.c +++ b/check.c @@ -596,7 +596,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* close the old one, if any */ ret = handle_close(&handle[j]); if (ret == -1) { - fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n"); + fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to check.\n"); printf("Stopping at block %u\n", i); ++unrecoverable_error; goto bail; @@ -1535,7 +1535,7 @@ bail: return 0; } -void state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount) +int state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount) { block_off_t blockmax; data_off_t size; @@ -1668,6 +1668,7 @@ void state_check(struct snapraid_state* state, int check, int fix, block_off_t b /* abort if error are present */ if (error != 0) - exit(EXIT_FAILURE); + return -1; + return 0; } diff --git a/elem.c b/elem.c index f45a14f6..96ed27af 100644 --- a/elem.c +++ b/elem.c @@ -542,3 +542,16 @@ void map_free(struct snapraid_map* map) free(map); } +int info_time_compare(const void* void_a, const void* void_b) +{ + const snapraid_info* info_a = void_a; + const snapraid_info* info_b = void_b; + time_t time_a = info_get_time(*info_a); + time_t time_b = info_get_time(*info_b); + if (time_a < time_b) + return -1; + if (time_a > time_b) + return 1; + return 0; +} + diff --git a/elem.h b/elem.h index 7c12c953..dae6fcec 100644 --- a/elem.h +++ b/elem.h @@ -698,10 +698,39 @@ void map_free(struct snapraid_map* map); /** * Makes an info. */ -static inline snapraid_info info_make(time_t last_access) +static inline snapraid_info info_make(time_t last_access, int error) { - /* clear the lowest bit as reserved for the hash type */ - return last_access & ~0x1; + /* clear the lowest bits as reserved for other information */ + snapraid_info info = last_access & ~0x1; + if (error != 0) + info |= 1; + return info; +} + +/** + * Extract the time information. + * This is the last time when the block was know to be correct. + */ +static inline time_t info_get_time(snapraid_info info) +{ + return info & ~0x1; +} + +/** + * Extract the error information. + * This report if the block address is reporting to have some problem. + */ +static inline int info_get_error(snapraid_info info) +{ + return (info & 0x1) != 0; +} + +/** + * Mark the block address as with error. + */ +static inline snapraid_info info_set_error(snapraid_info info) +{ + return info | 0x1; } /** @@ -731,5 +760,10 @@ static inline snapraid_info info_get(tommy_arrayof* array, unsigned pos) return info; } +/** + * Compares info by time. + */ +int info_time_compare(const void* void_a, const void* void_b); + #endif diff --git a/scrub.c b/scrub.c new file mode 100644 index 00000000..dd22a10c --- /dev/null +++ b/scrub.c @@ -0,0 +1,429 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "util.h" +#include "elem.h" +#include "state.h" +#include "parity.h" +#include "handle.h" +#include "raid.h" + +/****************************************************************************/ +/* scrub */ + +static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity* parity, struct snapraid_parity* qarity, block_off_t blockstart, block_off_t blockmax, time_t timelimit, block_off_t countlimit, time_t now) +{ + struct snapraid_handle* handle; + unsigned diskmax; + block_off_t i; + unsigned j; + void* buffer_alloc; + unsigned char* buffer_aligned; + unsigned char** buffer; + unsigned buffermax; + data_off_t countsize; + block_off_t countpos; + block_off_t countmax; + block_off_t blocklimit; + block_off_t autosavedone; + block_off_t autosavelimit; + block_off_t autosavemissing; + int ret; + unsigned error; + + /* maps the disks to handles */ + handle = handle_map(state, &diskmax); + + /* we need disk + 2 for each parity level buffers */ + buffermax = diskmax + state->level * 2; + + buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc); + buffer = malloc_nofail(buffermax * sizeof(void*)); + for(i=0;iblock_size; + } + + error = 0; + + /* first count the number of blocks to process */ + countmax = 0; + blocklimit = blockmax; /* block address for which we should stop */ + for(i=blockstart;iinfoarr, i); + if (info == 0) { + /* skip it */ + continue; + } + + /* if it's too new */ + blocktime = info_get_time(info); + if (blocktime > timelimit) { + /* skip it */ + continue; + } + if (countmax >= countlimit) { + blocklimit = i; + break; + } + + ++countmax; + } + + /* compute the autosave size for all disk, even if not read */ + /* this makes sense because the speed should be almost the same */ + /* if the disks are read in parallel */ + autosavelimit = state->autosave / (diskmax * state->block_size); + autosavemissing = countmax; /* blocks to do */ + autosavedone = 0; /* blocks done */ + + countsize = 0; + countpos = 0; + state_progress_begin(state, blockstart, blockmax, countmax); + for(i=blockstart;iinfoarr, i); + if (info == 0) { + /* skip it */ + continue; + } + + /* if it's too new */ + blocktime = info_get_time(info); + if (blocktime > timelimit) { + /* skip it */ + continue; + } + + /* one more block processed for autosave */ + ++autosavedone; + --autosavemissing; + + /* by default process the block, and skip it if something go wrong */ + error_on_this_block = 0; + silent_error_on_this_block = 0; + + /* for each disk, process the block */ + for(j=0;jblock_size); + continue; + } + + /* if the block is not used */ + block = disk_block_get(handle[j].disk, i); + if (!block_has_file(block)) { + /* use an empty block */ + memset(buffer[j], 0, state->block_size); + continue; + } + + /* if the file is different than the current one, close it */ + if (handle[j].file != block_file_get(block)) { + ret = handle_close(&handle[j]); + if (ret == -1) { + /* This one is really an unexpected error, because we are only reading */ + /* and closing a descriptor should never fail */ + fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to scrub.\n"); + printf("Stopping at block %u\n", i); + ++error; + goto bail; + } + } + + ret = handle_open(&handle[j], block_file_get(block), stderr, state->skip_sequential); + if (ret == -1) { + fprintf(stdlog, "error:%u:%s:%s: Open error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block)); + ++error; + error_on_this_block = 1; + continue; + } + + read_size = handle_read(&handle[j], block, buffer[j], state->block_size, stderr); + if (read_size == -1) { + fprintf(stdlog, "error:%u:%s:%s: Read error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block)); + ++error; + error_on_this_block = 1; + continue; + } + + countsize += read_size; + + /* now compute the hash */ + memhash(state->hash, state->hashseed, hash, buffer[j], read_size); + + if (block_has_hash(block)) { + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_SIZE) != 0) { + fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block)); + ++error; + silent_error_on_this_block = 1; + continue; + } + } + } + + /* if we have read all the data required, proceed with the parity */ + if (!error_on_this_block && !silent_error_on_this_block) { + unsigned char* buffer_parity; + unsigned char* buffer_qarity; + + /* buffers for parity read and not computed */ + if (state->level == 1) { + buffer_parity = buffer[diskmax + 1]; + buffer_qarity = 0; + } else { + buffer_parity = buffer[diskmax + 2]; + buffer_qarity = buffer[diskmax + 3]; + } + + /* read the parity */ + ret = parity_read(parity, i, buffer_parity, state->block_size, stdlog); + if (ret == -1) { + buffer_parity = 0; + fprintf(stdlog, "error:%u:parity: Read error\n", i); + ++error; + error_on_this_block = 1; + } + + /* read the qarity */ + if (state->level >= 2) { + ret = parity_read(qarity, i, buffer_qarity, state->block_size, stdlog); + if (ret == -1) { + buffer_qarity = 0; + fprintf(stdlog, "error:%u:qarity: Read error\n", i); + ++error; + error_on_this_block = 1; + } + } + + /* compute the parity */ + raid_gen(state->level, buffer, diskmax, state->block_size); + + /* compare the parity */ + if (buffer_parity && memcmp(buffer[diskmax], buffer_parity, state->block_size) != 0) { + fprintf(stdlog, "error:%u:parity: Data error\n", i); + ++error; + silent_error_on_this_block = 1; + } + if (state->level >= 2) { + if (buffer_qarity && memcmp(buffer[diskmax + 1], buffer_qarity, state->block_size) != 0) { + fprintf(stdlog, "error:%u:qarity: Data error\n", i); + ++error; + silent_error_on_this_block = 1; + } + } + } + + if (error_on_this_block) { + /* do nothing, as this is a generic error */ + /* maybe just caused by a not synched array */ + } else if (silent_error_on_this_block) { + /* set the error status keeping the existing time */ + snapraid_info info = info_get(&state->infoarr, i); + + info_set(&state->infoarr, i, info_set_error(info)); + } else { + /* update the time info of the block */ + info_set(&state->infoarr, i, info_make(now, 0)); + } + + /* mark the state as needing write */ + state->need_write = 1; + + /* count the number of processed block */ + ++countpos; + + /* progress */ + if (state_progress(state, i, countpos, countmax, countsize)) { + break; + } + + /* autosave */ + if (state->autosave != 0 + && autosavedone >= autosavelimit /* if we have reached the limit */ + && autosavemissing >= autosavelimit /* if we have at least a full step to do */ + ) { + autosavedone = 0; /* restart the counter */ + + state_progress_stop(state); + + printf("Autosaving...\n"); + state_write(state); + + state_progress_restart(state); + } + } + + state_progress_end(state, countpos, countmax, countsize); + +bail: + for(j=0;jinfoarr, i); + + /* skip unused blocks */ + if (info == 0) + continue; + + infomap[count++] = info; + } + + if (!count) { + fprintf(stderr, "The array appears to be empty.\n"); + exit(EXIT_FAILURE); + } + + /* sort it */ + qsort(infomap, count, sizeof(snapraid_info), info_time_compare); + + /* don't check more block than the available ones */ + if (countlimit >= count) + countlimit = count - 1; + + /* get the time limit */ + timelimit = info_get_time(infomap[countlimit]); + + /* don't scrub too recent blocks */ + if (timelimit > recentlimit) + timelimit = recentlimit; + + /* free the temp vector */ + free(infomap); + + /* open the file for reading */ + parity_ptr = &parity; + ret = parity_open(parity_ptr, state->parity, state->skip_sequential); + if (ret == -1) { + fprintf(stderr, "WARNING! Without an accessible Parity file, it isn't possible to scrub.\n"); + exit(EXIT_FAILURE); + } + + if (state->level >= 2) { + qarity_ptr = &qarity; + ret = parity_open(qarity_ptr, state->qarity, state->skip_sequential); + if (ret == -1) { + fprintf(stderr, "WARNING! Without an accessible Q-Parity file, it isn't possible to scrub.\n"); + exit(EXIT_FAILURE); + } + } else { + qarity_ptr = 0; + } + + printf("Scrubbing...\n"); + + error = 0; + + ret = state_scrub_process(state, parity_ptr, qarity_ptr, 0, blockmax, timelimit, countlimit, now); + if (ret == -1) { + ++error; + /* continue, as we are already exiting */ + } + + ret = parity_close(parity_ptr); + if (ret == -1) { + fprintf(stderr, "DANGER! Unexpected close error in Parity disk.\n"); + ++error; + /* continue, as we are already exiting */ + } + + if (state->level >= 2) { + ret = parity_close(qarity_ptr); + if (ret == -1) { + fprintf(stderr, "DANGER! Unexpected close error in Q-Parity disk.\n"); + ++error; + /* continue, as we are already exiting */ + } + } + + /* abort if required */ + if (error != 0) + return -1; + return 0; +} + diff --git a/snapraid.1 b/snapraid.1 index c47c5b22..b1d8ab90 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -30,7 +30,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] .PD 0 .PP .PD - sync|pool|diff|dup|check|fix + sync|status|scrub|diff|dup|pool|check|fix .PD 0 .PP .PD @@ -362,6 +362,37 @@ without losing the work already done. .PP The \(a"content\(a", \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary. The files in the array are NOT modified. +.SS status +Prints a summary of the state of the disk array. +.PP +It includes information about the parity fragmentation, how old +are the blocks without checking, and all the recorded silent +errors encoutered when scrubbing. +.PP +Nothing is modified. +.SS scrub +Scrubs the array, checking for silent errors. +.PP +For each command invocation, the 12% of the array is checked, +but nothing that it\(cqs more recent than 10 days. +.PP +The oldest blocks are scrubbed first ensuring an optimal check. +.PP +Any silent error identified is recorded in the content file, +and can be listed with the \(cqstatus\(cq command. +.PP +The \(a"content\(a" file is modified to update the time of the last check +of each block. +.SS diff +Lists all the files modified from the last \(a"sync\(a" command that +have to recompute their redundancy data. +.PP +Nothing is modified. +.SS dup +Lists all the duplicate files. Two files are assumed equal if their +hashes are matching. The effective data is not read. +.PP +Nothing is modified. .SS pool Creates or updates in the \(a"pooling\(a" directory a virtual view of all the files of your disk array. @@ -374,16 +405,6 @@ subdirectories are deleted and replaced with the new view of the array. Any othe regular file is left in place. .PP Nothing is modified outside the pool directory. -.SS diff -Lists all the files modified from the last \(a"sync\(a" command that -have to recompute their redundancy data. -.PP -Nothing is modified. -.SS dup -Lists all the duplicate files. Two files are assumed equal if their -hashes are matching. The effective data is not read. -.PP -Nothing is modified. .SS check Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved diff --git a/snapraid.c b/snapraid.c index 08b7fc4d..8f7657dd 100644 --- a/snapraid.c +++ b/snapraid.c @@ -45,13 +45,15 @@ void usage(void) { version(); - printf("Usage: " PACKAGE " sync|pool|diff|dup|check|fix [options]\n"); + printf("Usage: " PACKAGE " sync|status|scrub|diff|dup|pool|check|fix [options]\n"); printf("\n"); printf("Commands:\n"); printf(" sync Syncronize the state of the array of disks\n"); printf(" pool Create or update the virtual view of the array of disks\n"); printf(" diff Show the changes that needs to be syncronized\n"); printf(" dup Find duplicate files\n"); + printf(" scrub Scrub the array of disks\n"); + printf(" status Print the status of the array\n"); printf(" check Check the array of disks\n"); printf(" fix Fix the array of disks\n"); printf("\n"); @@ -177,6 +179,8 @@ void signal_handler(int signal) #define OPERATION_DUP 5 #define OPERATION_POOL 6 #define OPERATION_REHASH 7 +#define OPERATION_SCRUB 8 +#define OPERATION_STATUS 9 int main(int argc, char* argv[]) { @@ -399,16 +403,20 @@ int main(int argc, char* argv[]) operation = OPERATION_SYNC; } else if (strcmp(argv[optind], "check") == 0) { operation = OPERATION_CHECK; - } else if (strcmp(argv[optind], "fix") == 0) { + } else if (strcmp(argv[optind], "fix") == 0) { operation = OPERATION_FIX; - } else if (strcmp(argv[optind], "dry") == 0) { + } else if (strcmp(argv[optind], "dry") == 0) { operation = OPERATION_DRY; - } else if (strcmp(argv[optind], "dup") == 0) { + } else if (strcmp(argv[optind], "dup") == 0) { operation = OPERATION_DUP; - } else if (strcmp(argv[optind], "pool") == 0) { + } else if (strcmp(argv[optind], "pool") == 0) { operation = OPERATION_POOL; - } else if (strcmp(argv[optind], "rehash") == 0) { + } else if (strcmp(argv[optind], "rehash") == 0) { operation = OPERATION_REHASH; + } else if (strcmp(argv[optind], "scrub") == 0) { + operation = OPERATION_SCRUB; + } else if (strcmp(argv[optind], "status") == 0) { + operation = OPERATION_STATUS; } else { fprintf(stderr, "Unknown command '%s'\n", argv[optind]); exit(EXIT_FAILURE); @@ -548,7 +556,7 @@ int main(int argc, char* argv[]) if (!test_kill_after_sync && state.need_write) state_write(&state); - /* abort if required by the sync command */ + /* abort if required */ if (ret == -1) exit(EXIT_FAILURE); } else if (operation == OPERATION_DRY) { @@ -572,6 +580,25 @@ int main(int argc, char* argv[]) /* save the new state if required */ if (state.need_write) state_write(&state); + } else if (operation == OPERATION_SCRUB) { + state_read(&state); + + /* intercept Ctrl+C */ + signal(SIGINT, &signal_handler); + + ret = state_scrub(&state); + + /* save the new state if required */ + if (state.need_write) + state_write(&state); + + /* abort if required */ + if (ret != 0) + exit(EXIT_FAILURE); + } else if (operation == OPERATION_STATUS) { + state_read(&state); + + state_status(&state); } else if (operation == OPERATION_DUP) { state_read(&state); @@ -593,10 +620,14 @@ int main(int argc, char* argv[]) signal(SIGINT, &signal_handler); if (operation == OPERATION_CHECK) { - state_check(&state, !audit_only, 0, blockstart, blockcount); + ret = state_check(&state, !audit_only, 0, blockstart, blockcount); } else { /* it's fix */ - state_check(&state, 1, 1, blockstart, blockcount); + ret = state_check(&state, 1, 1, blockstart, blockcount); } + + /* abort if required */ + if (ret != 0) + exit(EXIT_FAILURE); } /* close log file */ diff --git a/snapraid.d b/snapraid.d index 184cb493..f6ec2f1b 100644 --- a/snapraid.d +++ b/snapraid.d @@ -9,7 +9,7 @@ Synopsis : [-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid] : [-s, --start BLKSTART] [-t, --count BLKCOUNT] : [-v, --verbose] [-l, --log FILE] - : sync|pool|diff|dup|check|fix + : sync|status|scrub|diff|dup|pool|check|fix :snapraid [-V, --version] [-h, --help] @@ -214,18 +214,28 @@ Commands The "content", "parity" and "q-parity" files are modified if necessary. The files in the array are NOT modified. - pool - Creates or updates in the "pooling" directory a virtual view of all - the files of your disk array. + status + Prints a summary of the state of the disk array. - The files are not really copied here, but just linked using - symbolic links. + It includes information about the parity fragmentation, how old + are the blocks without checking, and all the recorded silent + errors encoutered when scrubbing. - When updating, all the present symbolic links and empty - subdirectories are deleted and replaced with the new - view of the array. Any othe regular file is left in place. + Nothing is modified. - Nothing is modified outside the pool directory. + scrub + Scrubs the array, checking for silent errors. + + For each command invocation, the 12% of the array is checked, + but nothing that it's more recent than 10 days. + + The oldest blocks are scrubbed first ensuring an optimal check. + + Any silent error identified is recorded in the content file, + and can be listed with the 'status' command. + + The "content" file is modified to update the time of the last check + of each block. diff Lists all the files modified from the last "sync" command that @@ -239,6 +249,19 @@ Commands Nothing is modified. + pool + Creates or updates in the "pooling" directory a virtual view of all + the files of your disk array. + + The files are not really copied here, but just linked using + symbolic links. + + When updating, all the present symbolic links and empty + subdirectories are deleted and replaced with the new + view of the array. Any othe regular file is left in place. + + Nothing is modified outside the pool directory. + check Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved diff --git a/snapraid.txt b/snapraid.txt index b7f6dcf3..b1359cb8 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -13,7 +13,7 @@ snapraid [-c, --conf CONFIG] [-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid] [-s, --start BLKSTART] [-t, --count BLKCOUNT] [-v, --verbose] [-l, --log FILE] - sync|pool|diff|dup|check|fix + sync|status|scrub|diff|dup|pool|check|fix snapraid [-V, --version] [-h, --help] @@ -236,7 +236,50 @@ without losing the work already done. The "content", "parity" and "q-parity" files are modified if necessary. The files in the array are NOT modified. -5.2 pool +5.2 status +---------- + +Prints a summary of the state of the disk array. + +It includes information about the parity fragmentation, how old +are the blocks without checking, and all the recorded silent +errors encoutered when scrubbing. + +Nothing is modified. + +5.3 scrub +--------- + +Scrubs the array, checking for silent errors. + +For each command invocation, the 12% of the array is checked, +but nothing that it's more recent than 10 days. + +The oldest blocks are scrubbed first ensuring an optimal check. + +Any silent error identified is recorded in the content file, +and can be listed with the 'status' command. + +The "content" file is modified to update the time of the last check +of each block. + +5.4 diff +-------- + +Lists all the files modified from the last "sync" command that +have to recompute their redundancy data. + +Nothing is modified. + +5.5 dup +------- + +Lists all the duplicate files. Two files are assumed equal if their +hashes are matching. The effective data is not read. + +Nothing is modified. + +5.6 pool -------- Creates or updates in the "pooling" directory a virtual view of all @@ -251,23 +294,7 @@ view of the array. Any othe regular file is left in place. Nothing is modified outside the pool directory. -5.3 diff --------- - -Lists all the files modified from the last "sync" command that -have to recompute their redundancy data. - -Nothing is modified. - -5.4 dup -------- - -Lists all the duplicate files. Two files are assumed equal if their -hashes are matching. The effective data is not read. - -Nothing is modified. - -5.5 check +5.7 check --------- Checks all the files and the redundancy data. @@ -280,7 +307,7 @@ data is checked, and the redundandy data is ignored. Files are identified by path, and checked by content. Nothing is modified. -5.6 fix +5.8 fix ------- Checks and fix all the files. It's like "check" but it diff --git a/state.c b/state.c index dddbdbbf..5bee2eed 100644 --- a/state.c +++ b/state.c @@ -841,6 +841,8 @@ void state_read(struct snapraid_state* state) unsigned count_dir; tommy_node* node; oathash_t hash; + struct stat st; + int ret; count_file = 0; count_block = 0; @@ -891,6 +893,13 @@ void state_read(struct snapraid_state* state) return; } + /* get the date of the content file */ + ret = fstat(shandle(f), &st); + if (ret != 0) { + fprintf(stderr, "Error stating the content file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + /* start with a undefined default. */ /* it's for compatibility with version 1.0 where MD5 was implicit. */ state->hash = HASH_UNDEFINED; @@ -927,6 +936,7 @@ void state_read(struct snapraid_state* state) /* "blk"/"inv"/"new"/"chg" command */ block_off_t v_pos; struct snapraid_block* block; + snapraid_info info; if (!file) { fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line); @@ -1008,6 +1018,13 @@ void state_read(struct snapraid_state* state) disk = 0; } + /* set a fake info block, in case of upgrading from an old version */ + /* the real block info will overwrite this */ + info = info_make(st.st_mtime, 0); + + /* insert the block in the block array */ + info_set(&state->infoarr, v_pos, info); + /* stat */ ++count_block; } else if (strcmp(tag, "inf") == 0) { @@ -1038,7 +1055,7 @@ void state_read(struct snapraid_state* state) } hash = oathash32(hash, t); - info = info_make(t); + info = info_make(t, 0); /* insert the block in the block array */ info_set(&state->infoarr, v_pos, info); @@ -1472,6 +1489,38 @@ void state_read(struct snapraid_state* state) /* stat */ ++count_dir; + } else if (strcmp(tag, "bad") == 0) { + /* "bad" command */ + block_off_t v_pos; + snapraid_info info; + uint32_t t; + + hash = oathash8(hash, 'x'); + + ret = sgetu32(f, &v_pos); + if (ret < 0) { + fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + hash = oathash32(hash, v_pos); + + c = sgetc(f); + if (c != ' ') { + fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + ret = sgetu32(f, &t); + if (ret < 0) { + fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + hash = oathash32(hash, t); + + info = info_make(t, 1); + + /* insert the block in the block array */ + info_set(&state->infoarr, v_pos, info); } else if (strcmp(tag, "checksum") == 0) { hash = oathash8(hash, 'c'); @@ -1918,6 +1967,10 @@ void state_write(struct snapraid_state* state) /* maximum block */ blockmax = tommy_array_size(&disk->blockarr); + /* keep track of the maximum block for all disk */ + if (blockmax > infomax) + infomax = blockmax; + /* for each deleted block in the disk */ first_deleted = 1; for(b=0;b infomax) - infomax = b; - sputu32(b, f); hash = oathash32(hash, b); @@ -1960,19 +2009,36 @@ void state_write(struct snapraid_state* state) } } - /* note that here infomax contains the maximum block for which we wrote something */ - /* tha could be smaller than of blocks we have in memory, in case of deleted blocks */ - /* for each block */ for(b=0;binfoarr, b); - /* save only stuff different than 0 */ + /* check for each disk if block is really used */ + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* disk = i->data; + struct snapraid_block* block; + + block = disk_block_get(disk, b); + + if (block != BLOCK_EMPTY) + break; + } + + /* if processed all, it's empty for all */ + if (i == 0) + info = 0; + + /* save only stuffs different than 0 */ if (info != 0) { - sputsl("inf ", f); - hash = oathash8(hash, 'i'); + if (info_get_error(info)) { + sputsl("bad ", f); + hash = oathash8(hash, 'x'); + } else { + sputsl("inf ", f); + hash = oathash8(hash, 'i'); + } sputu32(b, f); hash = oathash32(hash, b); diff --git a/state.h b/state.h index 62954e19..4891e0ef 100644 --- a/state.h +++ b/state.h @@ -116,7 +116,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t * \param check If we have to check also the parity. * \param fix If we have to fix, after checking. It requires also check==1. */ -void state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount); +int state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount); /** * Dry the files. @@ -128,6 +128,16 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t */ void state_rehash(struct snapraid_state* state); +/** + * Scrub the files. + */ +int state_scrub(struct snapraid_state* state); + +/** + * Print the status. + */ +int state_status(struct snapraid_state* state); + /** * Finds duplicates. */ diff --git a/status.c b/status.c new file mode 100644 index 00000000..419b8833 --- /dev/null +++ b/status.c @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "util.h" +#include "elem.h" +#include "state.h" +#include "parity.h" +#include "handle.h" +#include "raid.h" + +/****************************************************************************/ +/* status */ + +unsigned day_ago(time_t time, time_t now) +{ + return (now - time) / (24 * 3600); +} + +#define GRAPH_COLUMN 70 +#define GRAPH_ROW 15 + +int state_status(struct snapraid_state* state) +{ + block_off_t blockmax; + block_off_t i; + snapraid_info* infomap; + time_t now; + unsigned error; + unsigned count; + unsigned dayoldest, daymedian, daynewest; + unsigned bar[GRAPH_COLUMN]; + unsigned barpos; + unsigned barmax; + time_t oldest, newest, median; + unsigned x, y; + tommy_node* node_disk; + unsigned file_count; + unsigned file_fragmented; + unsigned extra_fragment; + uint64_t file_size; + + /* get the present time */ + now = time(0); + + blockmax = parity_size(state); + + /* for each disk, recover empty files, symlinks and empty dirs */ + file_count = 0; + file_size = 0; + file_fragmented = 0; + extra_fragment = 0; + for(node_disk=state->disklist;node_disk!=0;node_disk=node_disk->next) { + struct snapraid_disk* disk = node_disk->data; + tommy_node* node; + + /* for each file in the disk */ + node = disk->filelist; + while (node) { + struct snapraid_file* file; + + file = node->data; + node = node->next; /* next node */ + + /* check fragmentation */ + if (file->size) { + block_off_t prev_pos; + int fragmented; + + fragmented = 0; + prev_pos = file->blockvec[0].parity_pos; + for(i=1;iblockmax;++i) { + block_off_t parity_pos = file->blockvec[i].parity_pos; + if (prev_pos + 1 != parity_pos) { + fragmented = 1; + ++extra_fragment; + } + prev_pos = parity_pos; + } + + if (fragmented) + ++file_fragmented; + } + + /* count files */ + ++file_count; + file_size += file->size; + } + } + + printf("\n"); + + printf("Files: %u\n", file_count); + printf("Fragmented files: %u\n", file_fragmented); + printf("Excess fragments: %u\n", extra_fragment); + printf("Files size: %" PRIu64 " GiB\n", file_size / (1024*1024*1024) ); + printf("Parity size: %" PRIu64 " GiB\n", blockmax * (uint64_t)state->block_size / (1024*1024*1024) ); + + printf("\n"); + + /* allocate a temp vector for all the blocks */ + infomap = malloc_nofail(blockmax * sizeof(snapraid_info)); + + /* copy the info in the temp vector */ + error = 0; + count = 0; + for(i=0;iinfoarr, i); + + /* skip unused blocks */ + if (info == 0) + continue; + + if (info_get_error(info)) + ++error; + + infomap[count++] = info; + } + + if (!count) { + fprintf(stderr, "The array appears to be empty.\n"); + free(infomap); + return 0; + } + + /* sort it */ + qsort(infomap, count, sizeof(snapraid_info), info_time_compare); + + oldest = info_get_time(infomap[0]); + median = info_get_time(infomap[count / 2]); + newest = info_get_time(infomap[count - 1]); + + barpos = 0; + barmax = 0; + for(i=0;i barmax) + barmax = step; + + bar[i] = step; + } + + dayoldest = day_ago(oldest, now); + daymedian = day_ago(median, now); + daynewest = day_ago(newest, now); + + printf("The oldest block was checked %u days ago, the median %u, and the newest %u days ago.\n", dayoldest, daymedian, daynewest); + + printf("\n"); + + /* print the graph */ + for(y=0;y= pivot) { + printf("*"); + } else if (bar[x] > 0) { + printf("+"); + } else { + printf("_"); + } + } else { + unsigned halfpivot = pivot - barmax / GRAPH_ROW / 2; + if (bar[x] >= pivot) { + printf("*"); + } else if (bar[x] >= halfpivot) { + printf("+"); + } else { + printf(" "); + } + } + } + printf("\n"); + } + printf(" %3u days ago of the last check %3u\n", dayoldest, daynewest); + + printf("\n"); + + if (error) + printf("DANGER! In the array there are %u failing blocks!\n", error); + else + printf("No silent error detected!\n"); + + /* free the temp vector */ + free(infomap); + + return 0; +} + diff --git a/sync.c b/sync.c index 48624a95..24b4145b 100644 --- a/sync.c +++ b/sync.c @@ -315,7 +315,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } /* update the time info of the block */ - info_set(&state->infoarr, i, info_make(now)); + info_set(&state->infoarr, i, info_make(now, 0)); } /* mark the state as needing write */ diff --git a/util.c b/util.c index 1dfbf5fe..787567a6 100644 --- a/util.c +++ b/util.c @@ -175,6 +175,14 @@ int sclose(STREAM* s) return 0; } +int shandle(STREAM* s) +{ + if (!s->handle_size) + return -1; + + return s->handle[0].f; +} + int sfill(STREAM* s) { ssize_t ret; diff --git a/util.h b/util.h index b3d1d318..8b230235 100644 --- a/util.h +++ b/util.h @@ -73,6 +73,12 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file); */ int sclose(STREAM* s); +/** + * Returns the handle of the file. + * In case of multi file, the first one is returned. + */ +int shandle(STREAM* s); + /** * Fills the read stream buffer and read a char. * \note Don't call this directly, but use sgetc().