Converts prehash to be a command line option.

This commit is contained in:
Andrea Mazzoleni 2014-02-12 22:04:14 +01:00
parent c420391493
commit 857c8eba63
11 changed files with 109 additions and 125 deletions

View file

@ -13,9 +13,9 @@ SnapRAID HISTORY
* Detect copied files from one disk to another and reuse the already
computed hash information to validate them in "sync".
Files are assumed copied if they matches the name, size and timestamp.
* Added a new "prehash" configuration option to run a preliminary hashing
step for all the new files to ensure to detect silent errors caused
by the heavy machine usage of the parity computation.
* For "sync", added a new -h, --pre-hash option to run a preliminary
hashing step for all the new files to ensure to detect silent errors
caused by the heavy machine usage of the parity computation.
* In "fix", if a previous attempt was made, resulting in a
.unrecoverable file, uses this file as starting point for the
new attempt.

View file

@ -138,7 +138,7 @@ check-local:
mkdir test/disk6
mkdir test/pool
echo --- Help
$(TESTENV) ./snapraid$(EXEEXT) --test-skip-device -h
$(TESTENV) ./snapraid$(EXEEXT) --test-skip-device -H
$(TESTENV) ./mktest$(EXEEXT)
echo --- Run the self test only this time
$(TESTENV) ./snapraid$(EXEEXT) --test-skip-device -c $(CONF) status
@ -402,7 +402,7 @@ endif
cp -pR test/disk1/a test/disk1/a_copy
cp -pR test/disk2/a test/disk2/a_copy
cp -pR test/disk3/a test/disk3/a_copy
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-kill-after-sync --test-prehash -c $(CONF) sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-kill-after-sync -c $(CONF) -h sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check
rm -r test/disk1/a_copy
rm -r test/disk2/a_copy

29
check.c
View file

@ -479,7 +479,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
return -1;
}
static int state_check_process(struct snapraid_state* state, int check, int fix, struct snapraid_parity** parity, block_off_t blockstart, block_off_t blockmax)
static int state_check_process(struct snapraid_state* state, int fix, struct snapraid_parity** parity, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_handle* handle;
unsigned diskmax;
@ -647,7 +647,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
}
/* if we are only hashing, we can skip excluded files and don't event read them */
if (!check && file_flag_has(block_file_get(block), FILE_IS_EXCLUDED)) {
if (state->opt.auditonly && file_flag_has(block_file_get(block), FILE_IS_EXCLUDED)) {
/* use an empty block */
/* in true, this is unnecessary, becase we are not checking any parity */
/* but we keep it for completeness */
@ -806,7 +806,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
}
/* now read and check the parity if requested */
if (check) {
if (!state->opt.auditonly) {
void* buffer_recov[LEV_MAX];
void* buffer_zero;
@ -1108,7 +1108,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* we are not fixing, but only checking */
/* print just the final status */
if (file_flag_has(file, FILE_IS_DAMAGED)) {
if (!check) {
if (state->opt.auditonly) {
fprintf(stdlog, "status:damaged:%s:%s\n", handle[j].disk->name, file->sub);
if (!state->opt.quiet) {
printf("damaged %s\n", path);
@ -1597,7 +1597,7 @@ bail:
printf("DANGER! There are unrecoverable errors!\n");
} else {
/* without checking, we don't know if they are really recoverable or not */
if (check)
if (!state->opt.auditonly)
printf("%8u unrecoverable errors\n", unrecoverable_error);
if (fix)
printf("Everything OK\n");
@ -1611,7 +1611,7 @@ bail:
fprintf(stdlog, "summary:error:%u\n", error);
if (fix)
fprintf(stdlog, "summary:error_recovered:%u\n", recovered_error);
if (check)
if (!state->opt.auditonly)
fprintf(stdlog, "summary:error_unrecoverable:%u\n", unrecoverable_error);
if (fix) {
if (error + recovered_error + unrecoverable_error == 0)
@ -1620,7 +1620,7 @@ bail:
fprintf(stdlog, "summary:exit:recovered\n");
else
fprintf(stdlog, "summary:exit:unrecoverable\n");
} else if (check) {
} else if (!state->opt.auditonly) {
if (error + unrecoverable_error == 0)
fprintf(stdlog, "summary:exit:ok\n");
else if (unrecoverable_error == 0)
@ -1666,7 +1666,7 @@ bail:
return 0;
}
int state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount)
int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount)
{
block_off_t blockmax;
data_off_t size;
@ -1679,13 +1679,6 @@ int state_check(struct snapraid_state* state, int check, int fix, block_off_t bl
printf("Initializing...\n");
if (!check && fix) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error in calling, you cannot fix without checking parity.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
blockmax = parity_size(state);
size = blockmax * (data_off_t)state->block_size;
@ -1722,7 +1715,7 @@ int state_check(struct snapraid_state* state, int check, int fix, block_off_t bl
/* LCOV_EXCL_STOP */
}
}
} else if (check) {
} else if (!state->opt.auditonly) {
/* if checking, open the file for reading */
/* it may fail if the file doesn't exist, in this case we continue to check the files */
for(l=0;l<state->level;++l) {
@ -1742,7 +1735,7 @@ int state_check(struct snapraid_state* state, int check, int fix, block_off_t bl
if (fix)
printf("Fixing...\n");
else if (check)
else if (!state->opt.auditonly)
printf("Checking...\n");
else
printf("Hashing...\n");
@ -1751,7 +1744,7 @@ int state_check(struct snapraid_state* state, int check, int fix, block_off_t bl
/* skip degenerated cases of empty parity, or skipping all */
if (blockstart < blockmax) {
ret = state_check_process(state, check, fix, parity_ptr, blockstart, blockmax);
ret = state_check_process(state, fix, parity_ptr, blockstart, blockmax);
if (ret == -1) {
/* LCOV_EXCL_START */
++error;

View file

@ -14,7 +14,7 @@ snapraid [\-c, \-\-conf CONFIG]
.PD 0
.PP
.PD
[\-a, \-\-audit\-only] [\-i, \-\-import DIR]
[\-a, \-\-audit\-only] [\-h, \-\-pre\-hash] [\-i, \-\-import DIR]
.PD 0
.PP
.PD
@ -43,7 +43,7 @@ snapraid [\-c, \-\-conf CONFIG]
.PP
.PD
.PP
snapraid [\-V, \-\-version] [\-h, \-\-help] [\-C, \-\-gen\-conf CONTENT]
snapraid [\-V, \-\-version] [\-H, \-\-help] [\-C, \-\-gen\-conf CONTENT]
.PD 0
.PP
.PD
@ -737,12 +737,28 @@ Blocks marked as bad are always scrubbed despite this option.
This option can be used only with the \(a"scrub\(a" command.
.TP
.B \-a, \-\-audit\-only
Verifies the hash of the files, with the \(a"check\(a" command,
without doing any kind of check on the parity data.
In the \(a"check\(a" command verifies the hash of the files without
doing any kind of check on the parity data.
If you are interested in checking only the file data this
option can speedup a lot the checking process.
This option can be used only with the \(a"check\(a" command.
.TP
.B \-h, \-\-pre\-hash
In the \(a"sync\(a" command run a preliminary hashing phase of all
the new data to ensure that during the parity computation
the data read is correct.
Normally in \(a"sync\(a" no preliminary hashing is done, and the new
data is hashed directly during the parity computation when it\(cqs
read for the first time,
This happens when the system is under heavy pressure due all
disks spinning and CPU busy, and in such extreme conditions,
if your machine has a latent hardware problem, it\(cqs possible to
encouter silent errors what cannot be detected because the data
is not yet hashed.
To avoid this risk, you can enable the \(a"pre\-hash\(a" mode and have
all the data hashed two times to ensure its integrity.
This option can be used only with the \(a"sync\(a" command.
.TP
.B \-i, \-\-import DIR
Imports from the specified directory any file that you deleted
from the array after the last \(a"sync\(a" command.
@ -974,17 +990,6 @@ commands interrupted by a machine crash, or any other event that
may interrupt SnapRAID.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
.SS prehash
Enables a double pass \(a"sync\(a" process with an initial hashing phase.
In the default \(a"sync\(a" mode no preliminary hashing is done, and the new
data is hashed directly during the parity computation when it\(cqs read
for the first time,
This happens when the system is under heavy pressure due all disks spinning
and CPU busy, and in such extreme conditions, if your machine has a latent
hardware problem, it\(cqs possible to encouter silent errors what cannot be
detected because the data is not yet hashed.
To avoid this risk, you can enable the \(a"prehash\(a" mode and have all the
data hashed two times to ensure its integrity.
.SS pool DIR
Defines the pooling directory where the virtual view of the disk
array is created using the \(a"pool\(a" command.

View file

@ -67,7 +67,8 @@ void usage(void)
printf(" " SWITCH_GETOPT_LONG("-o, --older-than DAYS ", "-o") " Process only the older part of the array\n");
printf(" " SWITCH_GETOPT_LONG("-i, --import DIR ", "-i") " Import deleted files\n");
printf(" " SWITCH_GETOPT_LONG("-l, --log FILE ", "-l") " Log file. Default none\n");
printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-A") " Check only file data and not parity\n");
printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-a") " Check only file data and not parity\n");
printf(" " SWITCH_GETOPT_LONG("-h, --pre-hash ", "-h") " Pre hash all the new data\n");
printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n");
printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force synching of disks that get empty\n");
printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n");
@ -75,7 +76,7 @@ void usage(void)
printf(" " SWITCH_GETOPT_LONG("-s, --start BLKSTART ", "-s") " Start from the specified block number\n");
printf(" " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT ", "-t") " Count of block to process\n");
printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n");
printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n");
printf(" " SWITCH_GETOPT_LONG("-H, --help ", "-H") " Help\n");
printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n");
}
@ -104,7 +105,6 @@ void memory(void)
#define OPT_TEST_FORCE_SCRUB_EVEN 272
#define OPT_TEST_FORCE_CONTENT_WRITE 273
#define OPT_TEST_FORCE_CONTENT_TEXT 274
#define OPT_TEST_PREHASH 275
#if HAVE_GETOPT_LONG
struct option long_options[] = {
@ -125,12 +125,13 @@ struct option long_options[] = {
{ "force-device", 0, 0, 'D' },
{ "find-by-name", 0, 0, 'N' }, /* deprecated in SnapRAID 4.0 */
{ "audit-only", 0, 0, 'a' },
{ "pre-hash", 0, 0, 'h' },
{ "speed-test", 0, 0, 'T' },
{ "gen-conf", 1, 0, 'C' },
{ "verbose", 0, 0, 'v' },
{ "quiet", 0, 0, 'q' },
{ "gui", 0, 0, 'G' }, /* undocumented GUI interface command */
{ "help", 0, 0, 'h' },
{ "help", 0, 0, 'H' },
{ "version", 0, 0, 'V' },
/* The following are test specific options, DO NOT USE! */
@ -186,14 +187,11 @@ struct option long_options[] = {
/* Force the use of text content file . */
{ "test-force-content-text", 0, 0, OPT_TEST_FORCE_CONTENT_TEXT },
/* Like the prehash configuration option. */
{ "test-prehash", 0, 0, OPT_TEST_PREHASH },
{ 0, 0, 0, 0 }
};
#endif
#define OPTIONS "c:f:d:mep:o:s:t:i:l:ZEUDNaTC:vqhVG"
#define OPTIONS "c:f:d:mep:o:s:t:i:l:ZEUDNahTC:vqHVG"
volatile int global_interrupt = 0;
@ -225,7 +223,6 @@ int main(int argc, char* argv[])
{
int c;
struct snapraid_option opt;
int audit_only;
const char* conf;
struct snapraid_state state;
int operation;
@ -253,7 +250,6 @@ int main(int argc, char* argv[])
/* defaults */
conf = CONF;
memset(&opt, 0, sizeof(opt));
audit_only = 0;
blockstart = 0;
blockcount = 0;
tommy_list_init(&filterlist_file);
@ -376,7 +372,10 @@ int main(int argc, char* argv[])
fprintf(stderr, "WARNING! Option --find-by-name, -N is deprecated and does nothing!\n");
break;
case 'a' :
audit_only = 1;
opt.auditonly = 1;
break;
case 'h' :
opt.prehash = 1;
break;
case 'v' :
opt.verbose = 1;
@ -387,7 +386,7 @@ int main(int argc, char* argv[])
case 'G' :
opt.gui = 1;
break;
case 'h' :
case 'H' :
usage();
exit(EXIT_SUCCESS);
case 'V' :
@ -457,9 +456,6 @@ int main(int argc, char* argv[])
case OPT_TEST_FORCE_CONTENT_TEXT :
opt.force_content_text = 1;
break;
case OPT_TEST_PREHASH :
opt.prehash = 1;
break;
default:
/* LCOV_EXCL_START */
fprintf(stderr, "Unknown option '%c'\n", (char)c);
@ -516,9 +512,21 @@ int main(int argc, char* argv[])
case OPERATION_CHECK :
break;
default:
if (audit_only) {
if (opt.auditonly) {
/* LCOV_EXCL_START */
fprintf(stderr, "You cannot use -A, --audit-only with the '%s' command\n", command);
fprintf(stderr, "You cannot use -a, --audit-only with the '%s' command\n", command);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
switch (operation) {
case OPERATION_SYNC :
break;
default:
if (opt.prehash) {
/* LCOV_EXCL_START */
fprintf(stderr, "You cannot use -h, --pre-hash with the '%s' command\n", command);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@ -767,9 +775,9 @@ int main(int argc, char* argv[])
signal(SIGINT, &signal_handler);
if (operation == OPERATION_CHECK) {
ret = state_check(&state, !audit_only, 0, blockstart, blockcount);
ret = state_check(&state, 0, blockstart, blockcount);
} else { /* it's fix */
ret = state_check(&state, 1, 1, blockstart, blockcount);
ret = state_check(&state, 1, blockstart, blockcount);
}
/* abort if required */

View file

@ -63,20 +63,6 @@ exclude /lost+found/
# Format: "autosave SIZE_IN_GiB"
#autosave 500
# Enables a double pass "sync" process with an initial hashing phase.
# In the default "sync" mode no preliminary hashing is done, and the new
# data is hashed directly during the parity computation when it's read
# for the first time,
# This happens when the system is under heavy pressure due all disks spinning
# and CPU busy, and in such extreme conditions, if your machine has a latent
# hardware problem, it's possible to encouter silent errors what cannot be
# detected because the data is not yet hashed.
# To avoid this risk, you can enable the "prehash" mode and have all the
# data hashed two times to ensure its integrity.
#
# Format: prehash
#prehash
# Defines the pooling directory where the virtual view of the disk
# array is created using the "pool" command (uncomment to enable).
# The files are not really copied here, but just linked using

View file

@ -67,20 +67,6 @@ exclude \Windows\
# Format: "autosave SIZE_IN_GiB"
#autosave 500
# Enables a double pass "sync" process with an initial hashing phase.
# In the default "sync" mode no preliminary hashing is done, and the new
# data is hashed directly during the parity computation when it's read
# for the first time,
# This happens when the system is under heavy pressure due all disks spinning
# and CPU busy, and in such extreme conditions, if your machine has a latent
# hardware problem, it's possible to encouter silent errors what cannot be
# detected because the data is not yet hashed.
# To avoid this risk, you can enable the "prehash" mode and have all the
# data hashed two times to ensure its integrity.
#
# Format: prehash
#prehash
# Defines the pooling directory where the virtual view of the disk
# array is created using the "pool" command (uncomment to enable).
# The files are not really copied here, but just linked using

View file

@ -5,7 +5,7 @@ Synopsis
:snapraid [-c, --conf CONFIG]
: [-f, --filter PATTERN] [-d, --filter-disk NAME]
: [-m, --filter-missing] [-e, --filter-error]
: [-a, --audit-only] [-i, --import DIR]
: [-a, --audit-only] [-h, --pre-hash] [-i, --import DIR]
: [-p, --percentage PERC] [-o, --older-than DAYS]
: [-Z, --force-zero] [-E, --force-empty]
: [-U, --force-uuid] [-D, --force-device]
@ -13,7 +13,7 @@ Synopsis
: [-s, --start BLKSTART] [-t, --count BLKCOUNT]
: status|sync|scrub|fix|check|list|diff|dup|pool|rehash
:snapraid [-V, --version] [-h, --help] [-C, --gen-conf CONTENT]
:snapraid [-V, --version] [-H, --help] [-C, --gen-conf CONTENT]
Description
SnapRAID is a backup program for disk arrays. It stores parity
@ -503,12 +503,28 @@ Options
This option can be used only with the "scrub" command.
-a, --audit-only
Verifies the hash of the files, with the "check" command,
without doing any kind of check on the parity data.
In the "check" command verifies the hash of the files without
doing any kind of check on the parity data.
If you are interested in checking only the file data this
option can speedup a lot the checking process.
This option can be used only with the "check" command.
-h, --pre-hash
In the "sync" command run a preliminary hashing phase of all
the new data to ensure that during the parity computation
the data read is correct.
Normally in "sync" no preliminary hashing is done, and the new
data is hashed directly during the parity computation when it's
read for the first time,
This happens when the system is under heavy pressure due all
disks spinning and CPU busy, and in such extreme conditions,
if your machine has a latent hardware problem, it's possible to
encouter silent errors what cannot be detected because the data
is not yet hashed.
To avoid this risk, you can enable the "pre-hash" mode and have
all the data hashed two times to ensure its integrity.
This option can be used only with the "sync" command.
-i, --import DIR
Imports from the specified directory any file that you deleted
from the array after the last "sync" command.
@ -742,18 +758,6 @@ Configuration
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
prehash
Enables a double pass "sync" process with an initial hashing phase.
In the default "sync" mode no preliminary hashing is done, and the new
data is hashed directly during the parity computation when it's read
for the first time,
This happens when the system is under heavy pressure due all disks spinning
and CPU busy, and in such extreme conditions, if your machine has a latent
hardware problem, it's possible to encouter silent errors what cannot be
detected because the data is not yet hashed.
To avoid this risk, you can enable the "prehash" mode and have all the
data hashed two times to ensure its integrity.
pool DIR
Defines the pooling directory where the virtual view of the disk
array is created using the "pool" command.

View file

@ -9,7 +9,7 @@
snapraid [-c, --conf CONFIG]
[-f, --filter PATTERN] [-d, --filter-disk NAME]
[-m, --filter-missing] [-e, --filter-error]
[-a, --audit-only] [-i, --import DIR]
[-a, --audit-only] [-h, --pre-hash] [-i, --import DIR]
[-p, --percentage PERC] [-o, --older-than DAYS]
[-Z, --force-zero] [-E, --force-empty]
[-U, --force-uuid] [-D, --force-device]
@ -17,7 +17,7 @@ snapraid [-c, --conf CONFIG]
[-s, --start BLKSTART] [-t, --count BLKCOUNT]
status|sync|scrub|fix|check|list|diff|dup|pool|rehash
snapraid [-V, --version] [-h, --help] [-C, --gen-conf CONTENT]
snapraid [-V, --version] [-H, --help] [-C, --gen-conf CONTENT]
2 DESCRIPTION
@ -548,12 +548,28 @@ SnapRAID provides the following options:
This option can be used only with the "scrub" command.
-a, --audit-only
Verifies the hash of the files, with the "check" command,
without doing any kind of check on the parity data.
In the "check" command verifies the hash of the files without
doing any kind of check on the parity data.
If you are interested in checking only the file data this
option can speedup a lot the checking process.
This option can be used only with the "check" command.
-h, --pre-hash
In the "sync" command run a preliminary hashing phase of all
the new data to ensure that during the parity computation
the data read is correct.
Normally in "sync" no preliminary hashing is done, and the new
data is hashed directly during the parity computation when it's
read for the first time,
This happens when the system is under heavy pressure due all
disks spinning and CPU busy, and in such extreme conditions,
if your machine has a latent hardware problem, it's possible to
encouter silent errors what cannot be detected because the data
is not yet hashed.
To avoid this risk, you can enable the "pre-hash" mode and have
all the data hashed two times to ensure its integrity.
This option can be used only with the "sync" command.
-i, --import DIR
Imports from the specified directory any file that you deleted
from the array after the last "sync" command.
@ -806,25 +822,13 @@ may interrupt SnapRAID.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
---- 7.8.2 prehash ----
Enables a double pass "sync" process with an initial hashing phase.
In the default "sync" mode no preliminary hashing is done, and the new
data is hashed directly during the parity computation when it's read
for the first time,
This happens when the system is under heavy pressure due all disks spinning
and CPU busy, and in such extreme conditions, if your machine has a latent
hardware problem, it's possible to encouter silent errors what cannot be
detected because the data is not yet hashed.
To avoid this risk, you can enable the "prehash" mode and have all the
data hashed two times to ensure its integrity.
---- 7.8.3 pool DIR ----
---- 7.8.2 pool DIR ----
Defines the pooling directory where the virtual view of the disk
array is created using the "pool" command.
The directory must already exist.
---- 7.8.4 share UNC_DIR ----
---- 7.8.3 share UNC_DIR ----
Defines the Windows UNC path required to access the disks remotely.
If this option is specified, the symbolic links created in the pool

View file

@ -681,8 +681,6 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
tommy_list_insert_tail(&state->disklist, &disk->node, disk);
} else if (strcmp(tag, "nohidden") == 0) {
state->filter_hidden = 1;
} else if (strcmp(tag, "prehash") == 0) {
state->opt.prehash = 1;
} else if (strcmp(tag, "exclude") == 0) {
struct snapraid_filter* filter;

View file

@ -59,7 +59,8 @@ struct snapraid_option {
int verbose; /**< Verbose output. */
int quiet; /**< Quiet output. */
int gui; /**< Gui output. */
int prehash; /**< Enables the prehash mode. */
int auditonly; /**< Enable the auditonly mode for check. */
int prehash; /**< Enables the prehash mode for sync. */
int force_zero; /**< Forced dangerous operations of synching files now with zero size. */
int force_empty; /**< Forced dangerous operations of synching disks now empty. */
int force_uuid; /**< Forced dangerous operations of synching disks with uuid changed. */
@ -170,10 +171,9 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
/**
* Checks (and fixes) all the files and parity data.
* \param check If we have to check also the parity.
* \param fix If we have to fix, after checking. It requires also check==1.
* \param fix If we have to fix, after checking.
*/
int state_check(struct snapraid_state* state, int check, int fix, block_off_t blockstart, block_off_t blockcount);
int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount);
/**
* Dry the files.