rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds the new status and scrub commands.

Browse source
This commit is contained in:
Andrea Mazzoleni 2013-06-29 17:53:40 +02:00
parent edc885ac79
commit a32a1cd9a8
17 changed files with 978 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

14
HISTORY
View file

@ -1,22 +1,26 @@
SnapRAID HISTORY
================
4.0 2013/6
4.0 2013/7
==========
* Better selection of the RAID6 implementation for different CPUs.
* New 'scrub' command to periodically check the oldest blocks for silent
errors without the need to scan the whole array.
* New 'status' command to check the fragmentation, the last check time
distribution, and the silent error status of the array.
* Added the new Spooky hash. It's faster in 64 bit architectures.
To convert you can use the new 'rehash' command.
* Added a hash seed to make harder intentional collision attacks.
* When insering files for the first time, sort them by their physical
address to improve read performance.
* Better selection of the RAID6 implementation for different CPUs.
* Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4.
They are a little faster than the previous unroll by 2.
* Added a hash seed to make harder intentional collision attacks.
* Added a lock file to avoid multiple running instances on the same array.
The file is named as parity file adding the .lock extension.
There is also the undocumented --test-skip-lock to avoid to check it.
* Automatically ignores, with warning, mount points inside the array
directory tree.
* Changes the 'dup' output format to include the size of each duplicate file.
* When insering files for the first time, sort them by their physical
address to improve read performance.
3.1 2013/5
==========

7
Makefile.am
View file

@ -12,6 +12,8 @@ snapraid_SOURCES = \
check.c \
dry.c \
rehash.c \
scrub.c \
status.c \
dup.c \
pool.c \
parity.c \
@ -103,7 +105,9 @@ check-local:
echo --- As empty, sync and check forcing murmur3
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf --test-force-murmur3 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
if HAVE_SYMLINK
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
endif
@ -114,6 +118,7 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf --test-force-order-physical sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
if HAVE_SYMLINK
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
endif
@ -140,6 +145,7 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-skip-fallocate -v -c $(srcdir)/test-raid5.conf --test-force-order-inode sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-skip-sequential -c $(srcdir)/test-raid5.conf check
if HAVE_SYMLINK
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
@ -417,6 +423,7 @@ endif
echo --- Sync after all the fixes
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
echo Success!
# Manual testing

13
TODO
View file

@ -130,16 +130,3 @@ Use the "snapraid -T" comment to see the speed.
Also, all the file operations are already done in background by the OS,
so no improvement is expect from this side.
* The SSE2 RAID6 computation may be slower than the MMX version.
See for example this case on a AMD Zacate APU E-350 1.6GHz:
memset 1812 [MB/s]
MD5 252 [MB/s]
Murmur3 1116 [MB/s]
RAID5 int32x2 1692 [MB/s]
RAID5 mmxx2 1745 [MB/s]
RAID5 sse2x2 2481 [MB/s]
RAID6 int32x2 807 [MB/s]
RAID6 mmxx2 1304 [MB/s]
RAID6 sse2x2 787 [MB/s]
- It happens in some systems. At now not an high priority issue.

7
check.c
View file

@ -596,7 +596,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* close the old one, if any */
ret = handle_close(&handle[j]);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n");
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to check.\n");
printf("Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
@ -1535,7 +1535,7 @@ bail:
return 0;
}
void state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount)
int state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount)
{
block_off_t blockmax;
data_off_t size;
@ -1668,6 +1668,7 @@ void state_check(struct snapraid_state* state, int check, int fix, block_off_t b
/* abort if error are present */
if (error != 0)
exit(EXIT_FAILURE);
return -1;
return 0;
}

13
elem.c
View file

@ -542,3 +542,16 @@ void map_free(struct snapraid_map* map)
free(map);
}
int info_time_compare(const void* void_a, const void* void_b)
{
const snapraid_info* info_a = void_a;
const snapraid_info* info_b = void_b;
time_t time_a = info_get_time(*info_a);
time_t time_b = info_get_time(*info_b);
if (time_a < time_b)
return -1;
if (time_a > time_b)
return 1;
return 0;
}

40
elem.h
View file

@ -698,10 +698,39 @@ void map_free(struct snapraid_map* map);
/**
* Makes an info.
*/
static inline snapraid_info info_make(time_t last_access)
static inline snapraid_info info_make(time_t last_access, int error)
{
/* clear the lowest bit as reserved for the hash type */
return last_access & ~0x1;
/* clear the lowest bits as reserved for other information */
snapraid_info info = last_access & ~0x1;
if (error != 0)
info |= 1;
return info;
}
/**
* Extract the time information.
* This is the last time when the block was know to be correct.
*/
static inline time_t info_get_time(snapraid_info info)
{
return info & ~0x1;
}
/**
* Extract the error information.
* This report if the block address is reporting to have some problem.
*/
static inline int info_get_error(snapraid_info info)
{
return (info & 0x1) != 0;
}
/**
* Mark the block address as with error.
*/
static inline snapraid_info info_set_error(snapraid_info info)
{
return info | 0x1;
}
/**
@ -731,5 +760,10 @@ static inline snapraid_info info_get(tommy_arrayof* array, unsigned pos)
return info;
}
/**
* Compares info by time.
*/
int info_time_compare(const void* void_a, const void* void_b);
#endif

429
scrub.c Normal file
View file

@ -0,0 +1,429 @@
/*
* Copyright (C) 2013 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "portable.h"
#include "util.h"
#include "elem.h"
#include "state.h"
#include "parity.h"
#include "handle.h"
#include "raid.h"
/****************************************************************************/
/* scrub */
static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity* parity, struct snapraid_parity* qarity, block_off_t blockstart, block_off_t blockmax, time_t timelimit, block_off_t countlimit, time_t now)
{
struct snapraid_handle* handle;
unsigned diskmax;
block_off_t i;
unsigned j;
void* buffer_alloc;
unsigned char* buffer_aligned;
unsigned char** buffer;
unsigned buffermax;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
block_off_t blocklimit;
block_off_t autosavedone;
block_off_t autosavelimit;
block_off_t autosavemissing;
int ret;
unsigned error;
/* maps the disks to handles */
handle = handle_map(state, &diskmax);
/* we need disk + 2 for each parity level buffers */
buffermax = diskmax + state->level * 2;
buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc);
buffer = malloc_nofail(buffermax * sizeof(void*));
for(i=0;i<buffermax;++i) {
buffer[i] = buffer_aligned + i * state->block_size;
}
error = 0;
/* first count the number of blocks to process */
countmax = 0;
blocklimit = blockmax; /* block address for which we should stop */
for(i=blockstart;i<blockmax;++i) {
time_t blocktime;
snapraid_info info;
/* if it's unused */
info = info_get(&state->infoarr, i);
if (info == 0) {
/* skip it */
continue;
}
/* if it's too new */
blocktime = info_get_time(info);
if (blocktime > timelimit) {
/* skip it */
continue;
}
if (countmax >= countlimit) {
blocklimit = i;
break;
}
++countmax;
}
/* compute the autosave size for all disk, even if not read */
/* this makes sense because the speed should be almost the same */
/* if the disks are read in parallel */
autosavelimit = state->autosave / (diskmax * state->block_size);
autosavemissing = countmax; /* blocks to do */
autosavedone = 0; /* blocks done */
countsize = 0;
countpos = 0;
state_progress_begin(state, blockstart, blockmax, countmax);
for(i=blockstart;i<blocklimit;++i) {
time_t blocktime;
snapraid_info info;
int error_on_this_block;
int silent_error_on_this_block;
int ret;
/* if it's unused */
info = info_get(&state->infoarr, i);
if (info == 0) {
/* skip it */
continue;
}
/* if it's too new */
blocktime = info_get_time(info);
if (blocktime > timelimit) {
/* skip it */
continue;
}
/* one more block processed for autosave */
++autosavedone;
--autosavemissing;
/* by default process the block, and skip it if something go wrong */
error_on_this_block = 0;
silent_error_on_this_block = 0;
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
/* if the disk position is not used */
if (!handle[j].disk) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* if the block is not used */
block = disk_block_get(handle[j].disk, i);
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {
ret = handle_close(&handle[j]);
if (ret == -1) {
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to scrub.\n");
printf("Stopping at block %u\n", i);
++error;
goto bail;
}
}
ret = handle_open(&handle[j], block_file_get(block), stderr, state->skip_sequential);
if (ret == -1) {
fprintf(stdlog, "error:%u:%s:%s: Open error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
error_on_this_block = 1;
continue;
}
read_size = handle_read(&handle[j], block, buffer[j], state->block_size, stderr);
if (read_size == -1) {
fprintf(stdlog, "error:%u:%s:%s: Read error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
error_on_this_block = 1;
continue;
}
countsize += read_size;
/* now compute the hash */
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
if (block_has_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
silent_error_on_this_block = 1;
continue;
}
}
}
/* if we have read all the data required, proceed with the parity */
if (!error_on_this_block && !silent_error_on_this_block) {
unsigned char* buffer_parity;
unsigned char* buffer_qarity;
/* buffers for parity read and not computed */
if (state->level == 1) {
buffer_parity = buffer[diskmax + 1];
buffer_qarity = 0;
} else {
buffer_parity = buffer[diskmax + 2];
buffer_qarity = buffer[diskmax + 3];
}
/* read the parity */
ret = parity_read(parity, i, buffer_parity, state->block_size, stdlog);
if (ret == -1) {
buffer_parity = 0;
fprintf(stdlog, "error:%u:parity: Read error\n", i);
++error;
error_on_this_block = 1;
}
/* read the qarity */
if (state->level >= 2) {
ret = parity_read(qarity, i, buffer_qarity, state->block_size, stdlog);
if (ret == -1) {
buffer_qarity = 0;
fprintf(stdlog, "error:%u:qarity: Read error\n", i);
++error;
error_on_this_block = 1;
}
}
/* compute the parity */
raid_gen(state->level, buffer, diskmax, state->block_size);
/* compare the parity */
if (buffer_parity && memcmp(buffer[diskmax], buffer_parity, state->block_size) != 0) {
fprintf(stdlog, "error:%u:parity: Data error\n", i);
++error;
silent_error_on_this_block = 1;
}
if (state->level >= 2) {
if (buffer_qarity && memcmp(buffer[diskmax + 1], buffer_qarity, state->block_size) != 0) {
fprintf(stdlog, "error:%u:qarity: Data error\n", i);
++error;
silent_error_on_this_block = 1;
}
}
}
if (error_on_this_block) {
/* do nothing, as this is a generic error */
/* maybe just caused by a not synched array */
} else if (silent_error_on_this_block) {
/* set the error status keeping the existing time */
snapraid_info info = info_get(&state->infoarr, i);
info_set(&state->infoarr, i, info_set_error(info));
} else {
/* update the time info of the block */
info_set(&state->infoarr, i, info_make(now, 0));
}
/* mark the state as needing write */
state->need_write = 1;
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, i, countpos, countmax, countsize)) {
break;
}
/* autosave */
if (state->autosave != 0
&& autosavedone >= autosavelimit /* if we have reached the limit */
&& autosavemissing >= autosavelimit /* if we have at least a full step to do */
) {
autosavedone = 0; /* restart the counter */
state_progress_stop(state);
printf("Autosaving...\n");
state_write(state);
state_progress_restart(state);
}
}
state_progress_end(state, countpos, countmax, countsize);
bail:
for(j=0;j<diskmax;++j) {
ret = handle_close(&handle[j]);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
}
}
free(handle);
free(buffer_alloc);
free(buffer);
if (error != 0)
return -1;
return 0;
}
int state_scrub(struct snapraid_state* state)
{
block_off_t blockmax;
block_off_t countlimit;
block_off_t i;
time_t timelimit;
time_t recentlimit;
unsigned count;
int ret;
struct snapraid_parity parity;
struct snapraid_parity qarity;
struct snapraid_parity* parity_ptr;
struct snapraid_parity* qarity_ptr;
snapraid_info* infomap;
unsigned error;
time_t now;
/* get the present time */
now = time(0);
printf("Initializing...\n");
blockmax = parity_size(state);
/* by default scrub 1/12 of the array */
countlimit = blockmax / 12;
/* by default use a 10 day time limit */
recentlimit = now - 10 * 24 * 3600;
/* identify the time limit */
/* we sort all the block times, and we identify the time limit for which we reach the quota */
/* this allow to process first the oldest blocks */
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
/* copy the info in the temp vector */
count = 0;
for(i=0;i<blockmax;++i) {
snapraid_info info = info_get(&state->infoarr, i);
/* skip unused blocks */
if (info == 0)
continue;
infomap[count++] = info;
}
if (!count) {
fprintf(stderr, "The array appears to be empty.\n");
exit(EXIT_FAILURE);
}
/* sort it */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
/* don't check more block than the available ones */
if (countlimit >= count)
countlimit = count - 1;
/* get the time limit */
timelimit = info_get_time(infomap[countlimit]);
/* don't scrub too recent blocks */
if (timelimit > recentlimit)
timelimit = recentlimit;
/* free the temp vector */
free(infomap);
/* open the file for reading */
parity_ptr = &parity;
ret = parity_open(parity_ptr, state->parity, state->skip_sequential);
if (ret == -1) {
fprintf(stderr, "WARNING! Without an accessible Parity file, it isn't possible to scrub.\n");
exit(EXIT_FAILURE);
}
if (state->level >= 2) {
qarity_ptr = &qarity;
ret = parity_open(qarity_ptr, state->qarity, state->skip_sequential);
if (ret == -1) {
fprintf(stderr, "WARNING! Without an accessible Q-Parity file, it isn't possible to scrub.\n");
exit(EXIT_FAILURE);
}
} else {
qarity_ptr = 0;
}
printf("Scrubbing...\n");
error = 0;
ret = state_scrub_process(state, parity_ptr, qarity_ptr, 0, blockmax, timelimit, countlimit, now);
if (ret == -1) {
++error;
/* continue, as we are already exiting */
}
ret = parity_close(parity_ptr);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in Parity disk.\n");
++error;
/* continue, as we are already exiting */
}
if (state->level >= 2) {
ret = parity_close(qarity_ptr);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in Q-Parity disk.\n");
++error;
/* continue, as we are already exiting */
}
}
/* abort if required */
if (error != 0)
return -1;
return 0;
}

43
snapraid.1
View file

@ -30,7 +30,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG]
.PD 0
.PP
.PD
sync|pool|diff|dup|check|fix
sync|status|scrub|diff|dup|pool|check|fix
.PD 0
.PP
.PD
@ -362,6 +362,37 @@ without losing the work already done.
.PP
The \(a"content\(a", \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary.
The files in the array are NOT modified.
.SS status
Prints a summary of the state of the disk array.
.PP
It includes information about the parity fragmentation, how old
are the blocks without checking, and all the recorded silent
errors encoutered when scrubbing.
.PP
Nothing is modified.
.SS scrub
Scrubs the array, checking for silent errors.
.PP
For each command invocation, the 12% of the array is checked,
but nothing that it\(cqs more recent than 10 days.
.PP
The oldest blocks are scrubbed first ensuring an optimal check.
.PP
Any silent error identified is recorded in the content file,
and can be listed with the \(cqstatus\(cq command.
.PP
The \(a"content\(a" file is modified to update the time of the last check
of each block.
.SS diff
Lists all the files modified from the last \(a"sync\(a" command that
have to recompute their redundancy data.
.PP
Nothing is modified.
.SS dup
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
.PP
Nothing is modified.
.SS pool
Creates or updates in the \(a"pooling\(a" directory a virtual view of all
the files of your disk array.
@ -374,16 +405,6 @@ subdirectories are deleted and replaced with the new
view of the array. Any othe regular file is left in place.
.PP
Nothing is modified outside the pool directory.
.SS diff
Lists all the files modified from the last \(a"sync\(a" command that
have to recompute their redundancy data.
.PP
Nothing is modified.
.SS dup
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
.PP
Nothing is modified.
.SS check
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved

49
snapraid.c
View file

@ -45,13 +45,15 @@ void usage(void)
{
version();
printf("Usage: " PACKAGE " sync|pool|diff|dup|check|fix [options]\n");
printf("Usage: " PACKAGE " sync|status|scrub|diff|dup|pool|check|fix [options]\n");
printf("\n");
printf("Commands:\n");
printf(" sync Syncronize the state of the array of disks\n");
printf(" pool Create or update the virtual view of the array of disks\n");
printf(" diff Show the changes that needs to be syncronized\n");
printf(" dup Find duplicate files\n");
printf(" scrub Scrub the array of disks\n");
printf(" status Print the status of the array\n");
printf(" check Check the array of disks\n");
printf(" fix Fix the array of disks\n");
printf("\n");
@ -177,6 +179,8 @@ void signal_handler(int signal)
#define OPERATION_DUP 5
#define OPERATION_POOL 6
#define OPERATION_REHASH 7
#define OPERATION_SCRUB 8
#define OPERATION_STATUS 9
int main(int argc, char* argv[])
{
@ -399,16 +403,20 @@ int main(int argc, char* argv[])
operation = OPERATION_SYNC;
} else if (strcmp(argv[optind], "check") == 0) {
operation = OPERATION_CHECK;
} else if (strcmp(argv[optind], "fix") == 0) {
} else if (strcmp(argv[optind], "fix") == 0) {
operation = OPERATION_FIX;
} else if (strcmp(argv[optind], "dry") == 0) {
} else if (strcmp(argv[optind], "dry") == 0) {
operation = OPERATION_DRY;
} else if (strcmp(argv[optind], "dup") == 0) {
} else if (strcmp(argv[optind], "dup") == 0) {
operation = OPERATION_DUP;
} else if (strcmp(argv[optind], "pool") == 0) {
} else if (strcmp(argv[optind], "pool") == 0) {
operation = OPERATION_POOL;
} else if (strcmp(argv[optind], "rehash") == 0) {
} else if (strcmp(argv[optind], "rehash") == 0) {
operation = OPERATION_REHASH;
} else if (strcmp(argv[optind], "scrub") == 0) {
operation = OPERATION_SCRUB;
} else if (strcmp(argv[optind], "status") == 0) {
operation = OPERATION_STATUS;
} else {
fprintf(stderr, "Unknown command '%s'\n", argv[optind]);
exit(EXIT_FAILURE);
@ -548,7 +556,7 @@ int main(int argc, char* argv[])
if (!test_kill_after_sync && state.need_write)
state_write(&state);
/* abort if required by the sync command */
/* abort if required */
if (ret == -1)
exit(EXIT_FAILURE);
} else if (operation == OPERATION_DRY) {
@ -572,6 +580,25 @@ int main(int argc, char* argv[])
/* save the new state if required */
if (state.need_write)
state_write(&state);
} else if (operation == OPERATION_SCRUB) {
state_read(&state);
/* intercept Ctrl+C */
signal(SIGINT, &signal_handler);
ret = state_scrub(&state);
/* save the new state if required */
if (state.need_write)
state_write(&state);
/* abort if required */
if (ret != 0)
exit(EXIT_FAILURE);
} else if (operation == OPERATION_STATUS) {
state_read(&state);
state_status(&state);
} else if (operation == OPERATION_DUP) {
state_read(&state);
@ -593,10 +620,14 @@ int main(int argc, char* argv[])
signal(SIGINT, &signal_handler);
if (operation == OPERATION_CHECK) {
state_check(&state, !audit_only, 0, blockstart, blockcount);
ret = state_check(&state, !audit_only, 0, blockstart, blockcount);
} else { /* it's fix */
state_check(&state, 1, 1, blockstart, blockcount);
ret = state_check(&state, 1, 1, blockstart, blockcount);
}
/* abort if required */
if (ret != 0)
exit(EXIT_FAILURE);
}
/* close log file */

43
snapraid.d
View file

@ -9,7 +9,7 @@ Synopsis
: [-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid]
: [-s, --start BLKSTART] [-t, --count BLKCOUNT]
: [-v, --verbose] [-l, --log FILE]
: sync|pool|diff|dup|check|fix
: sync|status|scrub|diff|dup|pool|check|fix
:snapraid [-V, --version] [-h, --help]
@ -214,18 +214,28 @@ Commands
The "content", "parity" and "q-parity" files are modified if necessary.
The files in the array are NOT modified.
pool
Creates or updates in the "pooling" directory a virtual view of all
the files of your disk array.
status
Prints a summary of the state of the disk array.
The files are not really copied here, but just linked using
symbolic links.
It includes information about the parity fragmentation, how old
are the blocks without checking, and all the recorded silent
errors encoutered when scrubbing.
When updating, all the present symbolic links and empty
subdirectories are deleted and replaced with the new
view of the array. Any othe regular file is left in place.
Nothing is modified.
Nothing is modified outside the pool directory.
scrub
Scrubs the array, checking for silent errors.
For each command invocation, the 12% of the array is checked,
but nothing that it's more recent than 10 days.
The oldest blocks are scrubbed first ensuring an optimal check.
Any silent error identified is recorded in the content file,
and can be listed with the 'status' command.
The "content" file is modified to update the time of the last check
of each block.
diff
Lists all the files modified from the last "sync" command that
@ -239,6 +249,19 @@ Commands
Nothing is modified.
pool
Creates or updates in the "pooling" directory a virtual view of all
the files of your disk array.
The files are not really copied here, but just linked using
symbolic links.
When updating, all the present symbolic links and empty
subdirectories are deleted and replaced with the new
view of the array. Any othe regular file is left in place.
Nothing is modified outside the pool directory.
check
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved

67
snapraid.txt
View file

@ -13,7 +13,7 @@ snapraid [-c, --conf CONFIG]
[-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid]
[-s, --start BLKSTART] [-t, --count BLKCOUNT]
[-v, --verbose] [-l, --log FILE]
sync|pool|diff|dup|check|fix
sync|status|scrub|diff|dup|pool|check|fix
snapraid [-V, --version] [-h, --help]
@ -236,7 +236,50 @@ without losing the work already done.
The "content", "parity" and "q-parity" files are modified if necessary.
The files in the array are NOT modified.
5.2 pool
5.2 status
----------
Prints a summary of the state of the disk array.
It includes information about the parity fragmentation, how old
are the blocks without checking, and all the recorded silent
errors encoutered when scrubbing.
Nothing is modified.
5.3 scrub
---------
Scrubs the array, checking for silent errors.
For each command invocation, the 12% of the array is checked,
but nothing that it's more recent than 10 days.
The oldest blocks are scrubbed first ensuring an optimal check.
Any silent error identified is recorded in the content file,
and can be listed with the 'status' command.
The "content" file is modified to update the time of the last check
of each block.
5.4 diff
--------
Lists all the files modified from the last "sync" command that
have to recompute their redundancy data.
Nothing is modified.
5.5 dup
-------
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
Nothing is modified.
5.6 pool
--------
Creates or updates in the "pooling" directory a virtual view of all
@ -251,23 +294,7 @@ view of the array. Any othe regular file is left in place.
Nothing is modified outside the pool directory.
5.3 diff
--------
Lists all the files modified from the last "sync" command that
have to recompute their redundancy data.
Nothing is modified.
5.4 dup
-------
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
Nothing is modified.
5.5 check
5.7 check
---------
Checks all the files and the redundancy data.
@ -280,7 +307,7 @@ data is checked, and the redundandy data is ignored.
Files are identified by path, and checked by content.
Nothing is modified.
5.6 fix
5.8 fix
-------
Checks and fix all the files. It's like "check" but it

88
state.c
View file

@ -841,6 +841,8 @@ void state_read(struct snapraid_state* state)
unsigned count_dir;
tommy_node* node;
oathash_t hash;
struct stat st;
int ret;
count_file = 0;
count_block = 0;
@ -891,6 +893,13 @@ void state_read(struct snapraid_state* state)
return;
}
/* get the date of the content file */
ret = fstat(shandle(f), &st);
if (ret != 0) {
fprintf(stderr, "Error stating the content file '%s'. %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);
}
/* start with a undefined default. */
/* it's for compatibility with version 1.0 where MD5 was implicit. */
state->hash = HASH_UNDEFINED;
@ -927,6 +936,7 @@ void state_read(struct snapraid_state* state)
/* "blk"/"inv"/"new"/"chg" command */
block_off_t v_pos;
struct snapraid_block* block;
snapraid_info info;
if (!file) {
fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line);
@ -1008,6 +1018,13 @@ void state_read(struct snapraid_state* state)
disk = 0;
}
/* set a fake info block, in case of upgrading from an old version */
/* the real block info will overwrite this */
info = info_make(st.st_mtime, 0);
/* insert the block in the block array */
info_set(&state->infoarr, v_pos, info);
/* stat */
++count_block;
} else if (strcmp(tag, "inf") == 0) {
@ -1038,7 +1055,7 @@ void state_read(struct snapraid_state* state)
}
hash = oathash32(hash, t);
info = info_make(t);
info = info_make(t, 0);
/* insert the block in the block array */
info_set(&state->infoarr, v_pos, info);
@ -1472,6 +1489,38 @@ void state_read(struct snapraid_state* state)
/* stat */
++count_dir;
} else if (strcmp(tag, "bad") == 0) {
/* "bad" command */
block_off_t v_pos;
snapraid_info info;
uint32_t t;
hash = oathash8(hash, 'x');
ret = sgetu32(f, &v_pos);
if (ret < 0) {
fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, v_pos);
c = sgetc(f);
if (c != ' ') {
fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
ret = sgetu32(f, &t);
if (ret < 0) {
fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, t);
info = info_make(t, 1);
/* insert the block in the block array */
info_set(&state->infoarr, v_pos, info);
} else if (strcmp(tag, "checksum") == 0) {
hash = oathash8(hash, 'c');
@ -1918,6 +1967,10 @@ void state_write(struct snapraid_state* state)
/* maximum block */
blockmax = tommy_array_size(&disk->blockarr);
/* keep track of the maximum block for all disk */
if (blockmax > infomax)
infomax = blockmax;
/* for each deleted block in the disk */
first_deleted = 1;
for(b=0;b<blockmax;++b) {
@ -1940,10 +1993,6 @@ void state_write(struct snapraid_state* state)
sputsl("off ", f);
hash = oathash8(hash, 'o');
/* keep track of the maximum block */
if (b > infomax)
infomax = b;
sputu32(b, f);
hash = oathash32(hash, b);
@ -1960,19 +2009,36 @@ void state_write(struct snapraid_state* state)
}
}
/* note that here infomax contains the maximum block for which we wrote something */
/* tha could be smaller than of blocks we have in memory, in case of deleted blocks */
/* for each block */
for(b=0;b<infomax;++b) {
snapraid_info info;
info = info_get(&state->infoarr, b);
/* save only stuff different than 0 */
/* check for each disk if block is really used */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block;
block = disk_block_get(disk, b);
if (block != BLOCK_EMPTY)
break;
}
/* if processed all, it's empty for all */
if (i == 0)
info = 0;
/* save only stuffs different than 0 */
if (info != 0) {
sputsl("inf ", f);
hash = oathash8(hash, 'i');
if (info_get_error(info)) {
sputsl("bad ", f);
hash = oathash8(hash, 'x');
} else {
sputsl("inf ", f);
hash = oathash8(hash, 'i');
}
sputu32(b, f);
hash = oathash32(hash, b);

12
state.h
View file

@ -116,7 +116,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
* \param check If we have to check also the parity.
* \param fix If we have to fix, after checking. It requires also check==1.
*/
void state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount);
int state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount);
/**
* Dry the files.
@ -128,6 +128,16 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t
*/
void state_rehash(struct snapraid_state* state);
/**
* Scrub the files.
*/
int state_scrub(struct snapraid_state* state);
/**
* Print the status.
*/
int state_status(struct snapraid_state* state);
/**
* Finds duplicates.
*/

224
status.c Normal file
View file

@ -0,0 +1,224 @@
/*
* Copyright (C) 2013 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "portable.h"
#include "util.h"
#include "elem.h"
#include "state.h"
#include "parity.h"
#include "handle.h"
#include "raid.h"
/****************************************************************************/
/* status */
unsigned day_ago(time_t time, time_t now)
{
return (now - time) / (24 * 3600);
}
#define GRAPH_COLUMN 70
#define GRAPH_ROW 15
int state_status(struct snapraid_state* state)
{
block_off_t blockmax;
block_off_t i;
snapraid_info* infomap;
time_t now;
unsigned error;
unsigned count;
unsigned dayoldest, daymedian, daynewest;
unsigned bar[GRAPH_COLUMN];
unsigned barpos;
unsigned barmax;
time_t oldest, newest, median;
unsigned x, y;
tommy_node* node_disk;
unsigned file_count;
unsigned file_fragmented;
unsigned extra_fragment;
uint64_t file_size;
/* get the present time */
now = time(0);
blockmax = parity_size(state);
/* for each disk, recover empty files, symlinks and empty dirs */
file_count = 0;
file_size = 0;
file_fragmented = 0;
extra_fragment = 0;
for(node_disk=state->disklist;node_disk!=0;node_disk=node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
tommy_node* node;
/* for each file in the disk */
node = disk->filelist;
while (node) {
struct snapraid_file* file;
file = node->data;
node = node->next; /* next node */
/* check fragmentation */
if (file->size) {
block_off_t prev_pos;
int fragmented;
fragmented = 0;
prev_pos = file->blockvec[0].parity_pos;
for(i=1;i<file->blockmax;++i) {
block_off_t parity_pos = file->blockvec[i].parity_pos;
if (prev_pos + 1 != parity_pos) {
fragmented = 1;
++extra_fragment;
}
prev_pos = parity_pos;
}
if (fragmented)
++file_fragmented;
}
/* count files */
++file_count;
file_size += file->size;
}
}
printf("\n");
printf("Files: %u\n", file_count);
printf("Fragmented files: %u\n", file_fragmented);
printf("Excess fragments: %u\n", extra_fragment);
printf("Files size: %" PRIu64 " GiB\n", file_size / (1024*1024*1024) );
printf("Parity size: %" PRIu64 " GiB\n", blockmax * (uint64_t)state->block_size / (1024*1024*1024) );
printf("\n");
/* allocate a temp vector for all the blocks */
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
/* copy the info in the temp vector */
error = 0;
count = 0;
for(i=0;i<blockmax;++i) {
snapraid_info info = info_get(&state->infoarr, i);
/* skip unused blocks */
if (info == 0)
continue;
if (info_get_error(info))
++error;
infomap[count++] = info;
}
if (!count) {
fprintf(stderr, "The array appears to be empty.\n");
free(infomap);
return 0;
}
/* sort it */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
oldest = info_get_time(infomap[0]);
median = info_get_time(infomap[count / 2]);
newest = info_get_time(infomap[count - 1]);
barpos = 0;
barmax = 0;
for(i=0;i<GRAPH_COLUMN;++i) {
time_t limit;
unsigned step;
limit = oldest + (newest - oldest) * (i+1) / GRAPH_COLUMN;
step = 0;
while (barpos < count && info_get_time(infomap[barpos]) <= limit) {
++barpos;
++step;
}
if (step > barmax)
barmax = step;
bar[i] = step;
}
dayoldest = day_ago(oldest, now);
daymedian = day_ago(median, now);
daynewest = day_ago(newest, now);
printf("The oldest block was checked %u days ago, the median %u, and the newest %u days ago.\n", dayoldest, daymedian, daynewest);
printf("\n");
/* print the graph */
for(y=0;y<GRAPH_ROW;++y) {
if (y == 0)
printf("%3u%% ", barmax * 100 / count);
else if (y == GRAPH_ROW - 1)
printf(" 0%% ");
else if (y == GRAPH_ROW/2)
printf("%3u%% ", barmax * 50 / count);
else
printf(" ");
for(x=0;x<GRAPH_COLUMN;++x) {
unsigned pivot = barmax * (GRAPH_ROW-y) / GRAPH_ROW;
/* if it's the baseline */
if (y == GRAPH_ROW-1) {
if (bar[x] >= pivot) {
printf("*");
} else if (bar[x] > 0) {
printf("+");
} else {
printf("_");
}
} else {
unsigned halfpivot = pivot - barmax / GRAPH_ROW / 2;
if (bar[x] >= pivot) {
printf("*");
} else if (bar[x] >= halfpivot) {
printf("+");
} else {
printf(" ");
}
}
}
printf("\n");
}
printf(" %3u days ago of the last check %3u\n", dayoldest, daynewest);
printf("\n");
if (error)
printf("DANGER! In the array there are %u failing blocks!\n", error);
else
printf("No silent error detected!\n");
/* free the temp vector */
free(infomap);
return 0;
}

2
sync.c
View file

@ -315,7 +315,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
/* update the time info of the block */
info_set(&state->infoarr, i, info_make(now));
info_set(&state->infoarr, i, info_make(now, 0));
}
/* mark the state as needing write */

8
util.c
View file

@ -175,6 +175,14 @@ int sclose(STREAM* s)
return 0;
}
int shandle(STREAM* s)
{
if (!s->handle_size)
return -1;
return s->handle[0].f;
}
int sfill(STREAM* s)
{
ssize_t ret;

6
util.h
View file

@ -73,6 +73,12 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file);
*/
int sclose(STREAM* s);
/**
* Returns the handle of the file.
* In case of multi file, the first one is returned.
*/
int shandle(STREAM* s);
/**
* Fills the read stream buffer and read a char.
* \note Don't call this directly, but use sgetc().