Adds the -i, --import options to import deleted files for fix.

This commit is contained in:
Andrea Mazzoleni 2013-02-27 21:07:16 +01:00
parent 94ba44c0e7
commit 6da21cdbd9
11 changed files with 183 additions and 82 deletions

View file

@ -5,11 +5,14 @@ SnapRAID HISTORY
==========
* Added pooling support with the new 'pool' command. It creates a virtual
view of the array using symbolic links pointing at the original files.
* Added a new -m, -.filter-missing options that allow to undelete files,
* Added a new -m, --filter-missing option that allow to undelete files,
without checking/fixing the others.
* Added a new -i, --import option to automatically import deleted files
when fixing.
* Added support also for hardlinks and empty directories.
* Added symlinks support to Windows. Note that Windows junctions are still
not supported.
* Added support to save symlinks in Windows. Note that only the symlink
is saved and not the linked file or directory.
Junctions are still not supported.
* Files without read permission generates now an error instead of a warning.
You now must explicitely exclude them in the configuration file with
exclusion rules.

6
TODO
View file

@ -3,12 +3,6 @@ SnapRAID TODO
This is the list of TODO items for SnapRAID.
- v3.0
* We can extend the fix command to read know deleted files from a specific directory,
even outside the array, compute the hash of all the blocks, and use it as a bag of data,
indexed by hash, to find missing data for DELETED and CHG blocks.
- v3.1
* If a content file is missing, sync should rewrite it, even if no change was done at the content.

163
check.c
View file

@ -19,6 +19,7 @@
#include "util.h"
#include "elem.h"
#include "import.h"
#include "state.h"
#include "parity.h"
#include "handle.h"
@ -234,35 +235,59 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
int error;
unsigned j;
int n;
int has_bad;
int something_to_recover;
error = 0;
/* here we have to try two different strategies to recover, because in case the 'sync' */
/* process is aborted, we don't know if the parity data is really updated or */
/* still represents the state before the 'sync' */
/* Here we have to try two different strategies to recover, because in case the 'sync' */
/* process is aborted, we don't know if the parity data is really updated just like after 'sync', */
/* or if it still represents the state before the 'sync'. */
/* now we assume that the parity is already computed for the current state */
/* As first, we assume that the parity is already computed for the current state */
/* and that we are going to recover the state after the last 'sync'. */
/* we need to put in the recovering process only the bad blocks, because all the */
/* others already contains the correct data, and the parity is correctly computed for them. */
/* this is the normal condition. */
/* This is the normal condition. */
/* In this case, parity contains correct info for BLK, CHG (new version) and NEW blocks, */
/* and not for DELETED ones. */
/* We need to put in the recovering process only the bad blocks, because all the */
/* others already contains the correct data read frin disk, and the parity is correctly computed for them. */
/* We are interested to recover BLK, CHG and NEW blocks if they are marked as bad, */
/* but we are not interested in DELETED ones. */
n = 0;
has_bad = 0; /* keep track if there is at least one block to fix */
something_to_recover = 0; /* keep track if there is at least one block to fix */
for(j=0;j<failed_count;++j) {
if (failed[j].is_bad) {
has_bad = 1;
unsigned block_state = block_state_get(failed[j].block);
/* we never set a DELETED block as bad */
assert(block_state_get(failed[j].block) != BLOCK_STATE_DELETED);
assert(block_state != BLOCK_STATE_DELETED); /* we cannot have bad DELETED blocks */
failed_map[n] = j;
++n;
if (block_state == BLOCK_STATE_BLK) { /* if we have the hash for it */
/* try to fetch the block using the hash */
if (state_import_fetch(state, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* we have corrected it! */
} else {
/* otherwise try to recover it */
failed_map[n] = j;
++n;
/* we have something to try to recover */
something_to_recover = 1;
}
} else {
assert(block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_NEW);
/* otherwise it's CHG or NEW and we try to recover it */
failed_map[n] = j;
++n;
/* we have something to try to recover */
something_to_recover = 1;
}
}
}
/* if nothing to fix */
if (!has_bad) {
if (!something_to_recover) {
/* recompute only parity and qarity */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
@ -289,9 +314,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
/* it may contain garbage */
failed[j].is_outofdate = 1;
}
}
if (block_state == BLOCK_STATE_CHG) {
} else if (block_state == BLOCK_STATE_CHG) {
/* if the hash is a bogus value we cannot check the result */
/* this could happen if we have lost this information */
/* after an aborted sync */
@ -299,7 +322,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
/* it may contain garbage */
failed[j].is_outofdate = 1;
} else
/* if the hash is different the previous one, we are sure to have */
/* if the hash is different than the previous one, we are sure to have */
/* restored it to the state after the 'sync' */
/* instead, if the hash matches, it could be either that the */
/* block after the sync has this hash, or that */
@ -317,31 +340,51 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
if (ret > 0)
error += ret;
/* now assume that the parity computation was not updated at the current state */
/* Now assume that the parity computation was not updated at the current state, */
/* but still represent the state before the last 'sync' process. */
/* this may happen only if a 'sync' is aborted by a crash or manually */
/* This may happen only if a 'sync' is aborted by a crash or manually. */
/* In this case, parity contains info for BLK, CHG (old version) and DELETED blocks, */
/* but not for CHG (new version) and NEW ones. */
/* We are interested to recover BLK ones marked as bad, */
/* but we are not interested to recover NEW and CHG (new version) blocks, even if marked as bad, */
/* because we don't have parity for them and it's just impossible, */
/* and we are not interested to recover DELETED ones. */
n = 0;
has_bad = 0; /* keep track if there is at least one block to fix */
something_to_recover = 0; /* keep track if there is at least one block to fix */
for(j=0;j<failed_count;++j) {
unsigned block_state;
unsigned block_state = block_state_get(failed[j].block);
if (failed[j].is_bad)
has_bad = 1;
block_state = block_state_get(failed[j].block);
if (failed[j].is_bad /* if the block is bad we don't know its content */
|| block_state == BLOCK_STATE_DELETED /* we don't know the original content of deleted blocks */
|| block_state == BLOCK_STATE_CHG /* we don't know the original content of changed blocks */
if (block_state == BLOCK_STATE_DELETED
|| block_state == BLOCK_STATE_CHG
) {
failed_map[n] = j;
++n;
}
/* If the block is CHG or DELETED, we don't have the original content of block, */
/* and we must try to recover it. */
/* This apply to CHG blocks even if they are not marked bad, */
/* because the parityis computed with old content, and not with the new one. */
/* Note that this recovering is done just to make possible to recover any other BLK one, */
/* we are not really interseted in DELETED and CHG (old version) ones. */
/* now adjust the state as before the last 'sync' */
/* try to fetch the old block using the old hash */
if (state_import_fetch(state, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
} else {
/* otherwise try to recover it */
failed_map[n] = j;
++n;
/* note that we don't set something_to_recover, because we are */
/* not really interested to recover *only* old blocks. */
}
/* mark that we have restored or we will restore an old state */
/* note that if the block is not marked as is_bad, */
/* we are not going to write it in the disk */
failed[j].is_outofdate = 1;
} else if (block_state == BLOCK_STATE_NEW) {
/* If the block is NEW, restore it to the original 0 as before the 'sync' */
/* Also in this case, we do this to just allow recovering of other BLK ones */
/* if the block is new, restore it to the original 0 as before the 'sync' */
if (block_state == BLOCK_STATE_NEW) {
memset(buffer[failed[j].index], 0, state->block_size);
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
@ -350,36 +393,31 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
/* note that if the block is not marked as is_bad, */
/* we are not going to write it in the disk */
failed[j].is_outofdate = 1;
} else if (failed[j].is_bad) {
/* If the block is bad we don't know its content, and we try to recover it */
/* At this point, we can have only BLK ones */
assert(block_state == BLOCK_STATE_BLK);
/* we have something we are interested to recover */
something_to_recover = 1;
/* we try to recover it */
failed_map[n] = j;
++n;
}
}
/* if nothing to fix */
if (!has_bad) {
/* recompute only parity and qarity */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
ret = repair_step(state, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret == 0) {
/* reprocess the blocks for NEW and CHG ones, for which we don't have a hash to check */
/* we know for sure that, even if we have restored them, that we have restored an old state */
/* so, we mark all of them as outofdate */
for(j=0;j<failed_count;++j) {
/* we take care only of BAD blocks we have to write back */
if (failed[j].is_bad) {
unsigned block_state = block_state_get(failed[j].block);
if (block_state == BLOCK_STATE_NEW || block_state == BLOCK_STATE_CHG) {
/* for sure we have recovered the state before the sync */
failed[j].is_outofdate = 1;
}
}
/* if nothing to fix, we just don't try */
if (something_to_recover) {
ret = repair_step(state, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret == 0) {
/* we alreay marked as outdated NEW and CHG blocks, we don't need to do it again */
return 0;
}
return 0;
if (ret > 0)
error += ret;
}
if (ret > 0)
error += ret;
/* return the number of failed attempts, or -1 if no strategy */
if (error)
@ -641,7 +679,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
if (block_state == BLOCK_STATE_NEW || block_state == BLOCK_STATE_CHG) {
/* store it in the failed set, because potentially */
/* the parity may be still computed with the previous content */
/* note that these blocks are never recovered, because we cannot check them */
failed[failed_count].is_bad = 0;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;

2
elem.h
View file

@ -435,7 +435,7 @@ struct snapraid_deleted* deleted_alloc(void);
struct snapraid_deleted* deleted_dup(struct snapraid_block* block);
/**
* Frees a deleted block.
* Frees a deleted block.
*/
void deleted_free(struct snapraid_deleted* deleted);

2
scan.c
View file

@ -573,7 +573,7 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
fprintf(stderr, "You can exclude it in the config file with:\n\texclude /%s\n", sub);
exit(EXIT_FAILURE);
}
if (dd == 0 && errno == 0) {
if (dd == 0) {
break; /* finished */
}

View file

@ -10,7 +10,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG]
.PD 0
.PP
.PD
[\(hya, \(hy\(hyaudit\(hyonly]
[\(hya, \(hy\(hyaudit\(hyonly] [\(hyi, \(hy\(hyimport DIR]
.PD 0
.PP
.PD
@ -388,6 +388,7 @@ Nothing is modified.
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
in the previous \(a"sync\(a" command.
.PP
If you use the \(hya, \(hy\(hyaudit\(hyonly option, only the file
data is checked, and the redundandy data is ignored.
.PP
@ -442,7 +443,7 @@ This option can be used only with the \(a"check\(a" and \(a"fix\(a" commands.
Note that it cannot be used with \(a"sync\(a", because \(a"sync\(a" always
process the whole array.
.TP
.B \(hyn, \(hy\(hyfileter\(hymissing
.B \(hyn, \(hy\(hyfilter\(hymissing
Filters the files to process in the \(a"check\(a" and \(a"fix\(a"
commands.
Only the files missing/deleted from the array are processed.
@ -456,8 +457,20 @@ process the whole array.
.B \(hya, \(hy\(hyaudit\(hyonly
When checking, only verify the hash of the files, without
doing any kind of check on the redundancy data.
If you are interested in checking only the file data this
option can speedup a lot the checking process.
This option can be used only with the \(a"check\(a" command.
.TP
.B \(hyi, \(hy\(hyimport DIR
When fixing imports from the specified directory any file
that you deleted from the array after the last \(a"sync\(a"
commmand.
If you still have such files, they could be used by the \(a"fix\(a"
command to improve the recover process.
The files are read also in subdirectories and they are
identified regardless of their name.
This option can be used only with the \(a"check\(a" and \(a"fix\(a" command.
.TP
.B \(hyN, \(hy\(hyfind\(hyby\(hyname
When syncing finds the files by path instead than by inode,
and ignores the nanosecond part of timestamp.

View file

@ -20,9 +20,9 @@
#include "snapraid.h"
#include "util.h"
#include "elem.h"
#include "import.h"
#include "state.h"
#include "raid.h"
#include "os.h"
/****************************************************************************/
/* main */
@ -60,6 +60,7 @@ void usage(void)
printf(" " SWITCH_GETOPT_LONG("-f, --filter PATTERN ", "-f") " Process only files matching the pattern\n");
printf(" " SWITCH_GETOPT_LONG("-d, --filter-dist NAME", "-f") " Process only files in the disk\n");
printf(" " SWITCH_GETOPT_LONG("-m, --filter-missing ", "-m") " Process only missing/delete files\n");
printf(" " SWITCH_GETOPT_LONG("-i, --import DIR ", "-i") " Import deleted files\n");
printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-A") " Check only file data and not parity\n");
printf(" " SWITCH_GETOPT_LONG("-N, --find-by-name ", "-N") " Find the files by name instead than by inode\n");
printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n");
@ -86,6 +87,7 @@ struct option long_options[] = {
{ "filter-missing", 0, 0, 'm' },
{ "start", 1, 0, 's' },
{ "count", 1, 0, 't' },
{ "import", 1, 0, 'i' },
{ "force-zero", 0, 0, 'Z' },
{ "force-empty", 0, 0, 'E' },
{ "force-second", 0, 0, 'S' },
@ -106,7 +108,7 @@ struct option long_options[] = {
};
#endif
#define OPTIONS "c:f:d:ms:t:ZESNaTvhVG"
#define OPTIONS "c:f:d:ms:t:i:ZESNaTvhVG"
volatile int global_interrupt = 0;
@ -152,6 +154,7 @@ int main(int argc, char* argv[])
int filter_missing;
char* e;
const char* command;
char* import;
os_init();
@ -173,6 +176,7 @@ int main(int argc, char* argv[])
tommy_list_init(&filterlist_file);
tommy_list_init(&filterlist_disk);
filter_missing = 0;
import = 0;
opterr = 0;
while ((c =
@ -219,6 +223,13 @@ int main(int argc, char* argv[])
exit(EXIT_FAILURE);
}
break;
case 'i' :
if (import) {
fprintf(stderr, "Import directory '%s' already specified as '%s'\n", optarg, import);
exit(EXIT_FAILURE);
}
import = strdup_nofail(optarg);
break;
case 'Z' :
force_zero = 1;
break;
@ -324,6 +335,10 @@ int main(int argc, char* argv[])
fprintf(stderr, "You cannot filter with the '%s' command\n", command);
exit(EXIT_FAILURE);
}
if (import != 0) {
fprintf(stderr, "You cannot import with the '%s' command\n", command);
exit(EXIT_FAILURE);
}
}
raid_init();
@ -335,6 +350,11 @@ int main(int argc, char* argv[])
state_config(&state, conf, verbose, gui, force_zero, force_empty, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_device);
if (import != 0) {
printf("%s\n", import);
state_import(&state, import);
}
if (operation == OPERATION_DIFF) {
state_read(&state);
@ -420,6 +440,7 @@ int main(int argc, char* argv[])
state_done(&state);
tommy_list_foreach(&filterlist_file, (tommy_foreach_func*)filter_free);
tommy_list_foreach(&filterlist_disk, (tommy_foreach_func*)filter_free);
free(import);
os_done();

View file

@ -4,7 +4,7 @@ Name{number}
Synopsis
:snapraid [-c, --conf CONFIG]
: [-f, --filter PATTERN] [-d, --filter-disk NAME] [-m, --filter-missing]
: [-a, --audit-only]
: [-a, --audit-only] [-i, --import DIR]
: [-N, --find-by-name]
: [-Z, --force-zero] [-E, --force-empty]
: [-s, --start BLKSTART] [-t, --count BLKCOUNT]
@ -243,6 +243,7 @@ Commands
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
in the previous "sync" command.
If you use the -a, --audit-only option, only the file
data is checked, and the redundandy data is ignored.
@ -299,7 +300,7 @@ Options
Note that it cannot be used with "sync", because "sync" always
process the whole array.
-n, --fileter-missing
-n, --filter-missing
Filters the files to process in the "check" and "fix"
commands.
Only the files missing/deleted from the array are processed.
@ -313,8 +314,20 @@ Options
-a, --audit-only
When checking, only verify the hash of the files, without
doing any kind of check on the redundancy data.
If you are interested in checking only the file data this
option can speedup a lot the checking process.
This option can be used only with the "check" command.
-i, --import DIR
When fixing imports from the specified directory any file
that you deleted from the array after the last "sync"
commmand.
If you still have such files, they could be used by the "fix"
command to improve the recover process.
The files are read also in subdirectories and they are
identified regardless of their name.
This option can be used only with the "check" and "fix" command.
-N, --find-by-name
When syncing finds the files by path instead than by inode,
and ignores the nanosecond part of timestamp.

View file

@ -8,7 +8,7 @@
snapraid [-c, --conf CONFIG]
[-f, --filter PATTERN] [-d, --filter-disk NAME] [-m, --filter-missing]
[-a, --audit-only]
[-a, --audit-only] [-i, --import DIR]
[-N, --find-by-name]
[-Z, --force-zero] [-E, --force-empty]
[-s, --start BLKSTART] [-t, --count BLKCOUNT]
@ -273,6 +273,7 @@ Nothing is modified.
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
in the previous "sync" command.
If you use the -a, --audit-only option, only the file
data is checked, and the redundandy data is ignored.
@ -334,7 +335,7 @@ SnapRAID provides the following options:
Note that it cannot be used with "sync", because "sync" always
process the whole array.
-n, --fileter-missing
-n, --filter-missing
Filters the files to process in the "check" and "fix"
commands.
Only the files missing/deleted from the array are processed.
@ -348,8 +349,20 @@ SnapRAID provides the following options:
-a, --audit-only
When checking, only verify the hash of the files, without
doing any kind of check on the redundancy data.
If you are interested in checking only the file data this
option can speedup a lot the checking process.
This option can be used only with the "check" command.
-i, --import DIR
When fixing imports from the specified directory any file
that you deleted from the array after the last "sync"
commmand.
If you still have such files, they could be used by the "fix"
command to improve the recover process.
The files are read also in subdirectories and they are
identified regardless of their name.
This option can be used only with the "check" and "fix" command.
-N, --find-by-name
When syncing finds the files by path instead than by inode,
and ignores the nanosecond part of timestamp.

View file

@ -18,6 +18,7 @@
#include "portable.h"
#include "elem.h"
#include "import.h"
#include "state.h"
#include "util.h"
@ -46,6 +47,8 @@ void state_init(struct snapraid_state* state)
tommy_list_init(&state->maplist);
tommy_list_init(&state->contentlist);
tommy_list_init(&state->filterlist);
tommy_list_init(&state->importlist);
tommy_hashdyn_init(&state->importset);
state->loaded_blockmax = 0;
}
@ -55,6 +58,8 @@ void state_done(struct snapraid_state* state)
tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
tommy_hashdyn_done(&state->importset);
}
/**

View file

@ -50,6 +50,8 @@ struct snapraid_state {
tommy_list disklist; /**< List of all the disks. */
tommy_list maplist; /**< List of all the disk mappings. */
tommy_list filterlist; /**< List of inclusion/exclusion. */
tommy_list importlist; /**< List of import file. */
tommy_hashdyn importset; /**< Hashtable by hash of all the import blocks. */
block_off_t loaded_blockmax; /**< Previous size of the parity file, computed from the loaded state. */
time_t progress_start; /**< Start of processing for progress visualization. */
time_t progress_last; /**< Last update of progress visualization. */