From 2a50ec747d47f68e5d385b38c8621d3f1aeb0715 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 13 Mar 2012 23:59:59 +0100 Subject: [PATCH] Adds the new autosave configuration option. Checks the parity validity invariant after reading the content file. --- elem.h | 6 +++ parity.c | 4 +- snapraid.1 | 26 +++++++--- snapraid.c | 4 +- snapraid.conf.example | 9 ++++ snapraid.conf.example.windows | 9 ++++ snapraid.d | 27 +++++++--- snapraid.txt | 27 +++++++--- state.c | 93 ++++++++++++++++++++++++++++++++++- state.h | 13 +++++ sync.c | 19 +++++++ 11 files changed, 209 insertions(+), 28 deletions(-) diff --git a/elem.h b/elem.h index 634531c0..8a5a5bfe 100644 --- a/elem.h +++ b/elem.h @@ -80,6 +80,9 @@ struct snapraid_file; /** * The block has the hash computed, but the parity is invalid. * This happens when the parity is invalidated by a change in another block. + * Note that this state is almost reduntant because we can detect that the parity is invalid + * also checking the other disks, where for sure exists a block the BLOCK_STATE_CHG/NEW state + * or a BLOCK_DELETED block. */ #define BLOCK_STATE_INV 2 @@ -300,6 +303,9 @@ block_off_t block_file_pos(struct snapraid_block* block); */ int block_is_last(struct snapraid_block* block); +/** + * Checks if the block is a real allocated block. + */ static inline int block_is_valid(struct snapraid_block* block) { return block != BLOCK_EMPTY && block != BLOCK_DELETED; diff --git a/parity.c b/parity.c index e9765ca9..9dce0f67 100644 --- a/parity.c +++ b/parity.c @@ -37,7 +37,9 @@ block_off_t parity_resize(struct snapraid_state* state) /* start from the declared size */ block_off_t block = tommy_array_size(&disk->blockarr); - /* decrease the block until an allocated or deleted block */ + /* decrease the block until an allocated one */ + /* we don't stop at deleted blocks, because we want to have them cleared */ + /* if they are at the end of the parity */ while (block > parity_block && !block_is_valid(tommy_array_get(&disk->blockarr, block - 1))) --block; diff --git a/snapraid.1 b/snapraid.1 index 4609003b..1fefd45c 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -353,7 +353,7 @@ This option can be used many times. In Unix, ensure to quote globbing chars if used. .TP .B \(hyH, \(hy\(hyfilter\(hynohidden -Filter out hidden files and directory. In Unix hidden files are +Filters out hidden files and directory. In Unix hidden files are the ones starting with \(cq.\(cq. In Windows they are the ones with the hidden attribute. .TP @@ -482,8 +482,8 @@ The default is 256 and it should work for most conditions. You could increase this value if you do not have enough RAM memory to run SnapRAID. .PP -As a rule of thumb, with 4GB or more memory use the default 256, -with 2GB use 512, and with 1GB use 1024. +As a rule of thumb, with 4 GiB or more memory use the default 256, +with 2 GiB use 512, and with 1 GiB use 1024. .PP In more details SnapRAID requires about TS*24/BS bytes of RAM memory to run. Where TS is the total size in bytes of @@ -506,6 +506,13 @@ wasted for each file. .PP For example, with 10000 files and a 256 KiB block size, you are going to waste 1.2 GiB. +.SS autosave SIZE_IN_GIBIBYTES +Automatically save the state when synching after the specied amount +of GiB processed. +This option is useful to avoid to restart from scratch long \(cqsync\(cq +commands interrupted by a machine crash. +The SIZE argument is specified in gibibytes. Where one gibi bytes +is 1073741824 bytes. .SS Examples An example of a typical configuration for Unix is: .PP @@ -765,7 +772,7 @@ snapraid fix 2>fix.log .PP This command will take a long time. .PP -Take care that you need also few GB free to store the fix.log file, so run it +Take care that you need also few gigabytes free to store the fix.log file, so run it from a disk with some free space. .PP Now you have recovered all the recoverable. If some file is partially or totally @@ -844,7 +851,7 @@ to recompute the parity. The PATH information is used in the \(a"check\(a" and \(a"fix\(a" commands to identify the file. .SS blk BLOCK HASH -Defines the ordered parity block list used by the last defined file. +Defines an ordered parity block, part of the last defined file. .PP BLOCK is the block position in the \(a"parity\(a" file. 0 for the first block, 1 for the second one and so on. @@ -858,13 +865,15 @@ This field is used only when you interrupt manually the \(a"sync\(a" command. .SS new BLOCK Like \(a"blk\(a", but for new allocated blocks for which the hash is not -yet computed. +yet computed, and the stored parity doesn\(cqt take into account this +new block. .PP This field is used only when you interrupt manually the \(a"sync\(a" command. .SS chg BLOCK Like \(a"blk\(a", but for reallocated blocks for which the hash is not -yet computed. +yet computed, and the parity is computed using the previous value +of the block. .PP This field is used only when you interrupt manually the \(a"sync\(a" command. @@ -874,7 +883,8 @@ Defines the list of blocks that are deleted from a disk. This field is used only when you interrupt manually the \(a"sync\(a" command. .SS off BLOCK -Defines the block deleted from a disk. +Defines a block deleted from a disk, part of the last defined hole, +for which the parity is computed using the previous value. .PP This field is used only when you interrupt manually the \(a"sync\(a" command. diff --git a/snapraid.c b/snapraid.c index 991f3762..a793a8e6 100644 --- a/snapraid.c +++ b/snapraid.c @@ -124,8 +124,8 @@ int main(int argc, char* argv[]) int gui; int force_zero; int force_empty; - int find_by_name; int filter_hidden; + int find_by_name; int test_expect_unrecoverable; int test_expect_recoverable; int test_kill_after_sync; @@ -146,8 +146,8 @@ int main(int argc, char* argv[]) gui = 0; force_zero = 0; force_empty = 0; - find_by_name = 0; filter_hidden = 0; + find_by_name = 0; test_expect_unrecoverable = 0; test_expect_recoverable = 0; test_kill_after_sync = 0; diff --git a/snapraid.conf.example b/snapraid.conf.example index b9367f28..9148d1f9 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -47,3 +47,12 @@ exclude /lost+found/ # Format: "block_size SIZE_IN_KiB" block_size 256 +# Automatically save the state when synching after the specied amount +# of GiB processed. +# This option is useful to avoid to restart from scratch long 'sync' +# commands interrupted by a machine crash. +# The SIZE argument is specified in gibi bytes -> 1073741824 bytes +# Default value is 0, meaning disabled. +# Format: "autosave_size SIZE_IN_GiB" +# autosize 500 + diff --git a/snapraid.conf.example.windows b/snapraid.conf.example.windows index 3088dbff..0fb51ea2 100644 --- a/snapraid.conf.example.windows +++ b/snapraid.conf.example.windows @@ -48,3 +48,12 @@ exclude \System Volume Information # Format: "block_size SIZE_IN_KiB" block_size 256 +# Automatically save the state when synching after the specied amount +# of GiB processed. +# This option is useful to avoid to restart from scratch long 'sync' +# commands interrupted by a machine crash. +# The SIZE argument is specified in gibi bytes -> 1073741824 bytes +# Default value is 0, meaning disabled. +# Format: "autosave_size SIZE_IN_GiB" +# autosize 500 + diff --git a/snapraid.d b/snapraid.d index 50b4f408..a0ba6d59 100644 --- a/snapraid.d +++ b/snapraid.d @@ -238,7 +238,7 @@ Options In Unix, ensure to quote globbing chars if used. -H, --filter-nohidden - Filter out hidden files and directory. In Unix hidden files are + Filters out hidden files and directory. In Unix hidden files are the ones starting with '.'. In Windows they are the ones with the hidden attribute. @@ -375,8 +375,8 @@ Configuration You could increase this value if you do not have enough RAM memory to run SnapRAID. - As a rule of thumb, with 4GB or more memory use the default 256, - with 2GB use 512, and with 1GB use 1024. + As a rule of thumb, with 4 GiB or more memory use the default 256, + with 2 GiB use 512, and with 1 GiB use 1024. In more details SnapRAID requires about TS*24/BS bytes of RAM memory to run. Where TS is the total size in bytes of @@ -397,6 +397,14 @@ Configuration For example, with 10000 files and a 256 KiB block size, you are going to waste 1.2 GiB. + autosave SIZE_IN_GIBIBYTES + Automatically save the state when synching after the specied amount + of GiB processed. + This option is useful to avoid to restart from scratch long 'sync' + commands interrupted by a machine crash. + The SIZE argument is specified in gibibytes. Where one gibi bytes + is 1073741824 bytes. + Examples An example of a typical configuration for Unix is: @@ -523,7 +531,7 @@ Recovering This command will take a long time. - Take care that you need also few GB free to store the fix.log file, so run it + Take care that you need also few gigabytes free to store the fix.log file, so run it from a disk with some free space. Now you have recovered all the recoverable. If some file is partially or totally @@ -601,7 +609,7 @@ Content to identify the file. blk BLOCK HASH - Defines the ordered parity block list used by the last defined file. + Defines an ordered parity block, part of the last defined file. BLOCK is the block position in the "parity" file. 0 for the first block, 1 for the second one and so on. @@ -617,14 +625,16 @@ Content new BLOCK Like "blk", but for new allocated blocks for which the hash is not - yet computed. + yet computed, and the stored parity doesn't take into account this + new block. This field is used only when you interrupt manually the "sync" command. chg BLOCK Like "blk", but for reallocated blocks for which the hash is not - yet computed. + yet computed, and the parity is computed using the previous value + of the block. This field is used only when you interrupt manually the "sync" command. @@ -636,7 +646,8 @@ Content command. off BLOCK - Defines the block deleted from a disk. + Defines a block deleted from a disk, part of the last defined hole, + for which the parity is computed using the previous value. This field is used only when you interrupt manually the "sync" command. diff --git a/snapraid.txt b/snapraid.txt index 9c1568e0..1ef43795 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -267,7 +267,7 @@ SnapRAID provides the following options: In Unix, ensure to quote globbing chars if used. -H, --filter-nohidden - Filter out hidden files and directory. In Unix hidden files are + Filters out hidden files and directory. In Unix hidden files are the ones starting with '.'. In Windows they are the ones with the hidden attribute. @@ -421,8 +421,8 @@ The default is 256 and it should work for most conditions. You could increase this value if you do not have enough RAM memory to run SnapRAID. -As a rule of thumb, with 4GB or more memory use the default 256, -with 2GB use 512, and with 1GB use 1024. +As a rule of thumb, with 4 GiB or more memory use the default 256, +with 2 GiB use 512, and with 1 GiB use 1024. In more details SnapRAID requires about TS*24/BS bytes of RAM memory to run. Where TS is the total size in bytes of @@ -443,6 +443,14 @@ wasted for each file. For example, with 10000 files and a 256 KiB block size, you are going to waste 1.2 GiB. +---- 7.7.1 autosave SIZE_IN_GIBIBYTES ---- +Automatically save the state when synching after the specied amount +of GiB processed. +This option is useful to avoid to restart from scratch long 'sync' +commands interrupted by a machine crash. +The SIZE argument is specified in gibibytes. Where one gibi bytes +is 1073741824 bytes. + 7.8 Examples ------------ @@ -577,7 +585,7 @@ Run the fix command, storing the log in an external file with: This command will take a long time. -Take care that you need also few GB free to store the fix.log file, so run it +Take care that you need also few gigabytes free to store the fix.log file, so run it from a disk with some free space. Now you have recovered all the recoverable. If some file is partially or totally @@ -670,7 +678,7 @@ to identify the file. 10.6 blk BLOCK HASH ------------------- -Defines the ordered parity block list used by the last defined file. +Defines an ordered parity block, part of the last defined file. BLOCK is the block position in the "parity" file. 0 for the first block, 1 for the second one and so on. @@ -690,7 +698,8 @@ command. -------------- Like "blk", but for new allocated blocks for which the hash is not -yet computed. +yet computed, and the stored parity doesn't take into account this +new block. This field is used only when you interrupt manually the "sync" command. @@ -699,7 +708,8 @@ command. -------------- Like "blk", but for reallocated blocks for which the hash is not -yet computed. +yet computed, and the parity is computed using the previous value +of the block. This field is used only when you interrupt manually the "sync" command. @@ -715,7 +725,8 @@ command. 10.11 off BLOCK --------------- -Defines the block deleted from a disk. +Defines a block deleted from a disk, part of the last defined hole, +for which the parity is computed using the previous value. This field is used only when you interrupt manually the "sync" command. diff --git a/state.c b/state.c index 17c12bf2..592bda8d 100644 --- a/state.c +++ b/state.c @@ -29,6 +29,7 @@ void state_init(struct snapraid_state* state) state->force_empty = 0; state->filter_hidden = 0; state->find_by_name = 0; + state->autosave = 0; state->expect_unrecoverable = 0; state->expect_recoverable = 0; state->need_write = 0; @@ -387,6 +388,24 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i exit(EXIT_FAILURE); } tommy_list_insert_tail(&state->filterlist, &filter->node, filter); + } else if (strcmp(tag, "autosave") == 0) { + char* e; + + ret = sgetlasttok(f, buffer, sizeof(buffer)); + if (ret < 0) { + fprintf(stderr, "Invalid 'autosave' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + state->autosave = strtoul(buffer, &e, 0); + + if (!e || *e) { + fprintf(stderr, "Invalid 'autosave' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + /* convert to GiB */ + state->autosave *= 1024 * 1024 * 1024; } else if (tag[0] == 0) { /* allow empty lines */ } else if (tag[0] == '#') { @@ -525,6 +544,7 @@ static void state_map(struct snapraid_state* state) static void state_content_check(struct snapraid_state* state, const char* path) { tommy_node* i; + block_off_t j; /* checks that any map has different name and position */ for(i=state->maplist;i!=0;i=i->next) { @@ -542,6 +562,48 @@ static void state_content_check(struct snapraid_state* state, const char* path) } } } + + /* check the parity validity invariant */ + for(j=0;1;++j) { /* for all the blocks */ + int at_least_one = 0; + int parity_is_valid = 0; + int parity_is_invalid = 0; + + /* for all the disks */ + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* disk = i->data; + + /* if the block is present */ + block_off_t size = tommy_array_size(&disk->blockarr); + if (j < size) { + struct snapraid_block* block = tommy_array_get(&disk->blockarr, j); + + at_least_one = 1; + + if (block == BLOCK_EMPTY) { + /* not relevant */ + } else if (block == BLOCK_DELETED) { + parity_is_invalid = 1; + } else { + switch (block_state_get(block)) { + case BLOCK_STATE_BLK : parity_is_valid = 1; break; + case BLOCK_STATE_NEW : parity_is_invalid = 1; break; + case BLOCK_STATE_INV : parity_is_invalid = 1; break; + case BLOCK_STATE_CHG : parity_is_invalid = 1; break; + } + } + } + } + + if (parity_is_valid && parity_is_invalid) { + fprintf(stderr, "Internal inconsistency in parity validity at block %u\n", j); + exit(EXIT_FAILURE); + } + + /* stop if reached the end of all the arrays */ + if (!at_least_one) + break; + } } void state_read(struct snapraid_state* state) @@ -735,6 +797,13 @@ void state_read(struct snapraid_state* state) /* insert the block in the block array */ tommy_array_grow(&disk->blockarr, v_pos + 1); + + /* deleted block must be empty blocks */ + if (tommy_array_get(&disk->blockarr, v_pos) != 0) { + fprintf(stderr, "Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + tommy_array_set(&disk->blockarr, v_pos, BLOCK_DELETED); } else if (strcmp(tag, "file") == 0) { /* file */ @@ -1442,6 +1511,7 @@ void state_progress_begin(struct snapraid_state* state, block_off_t blockstart, state->progress_start = now; state->progress_last = now; + state->progress_subtract = 0; } } @@ -1462,6 +1532,27 @@ void state_progress_end(struct snapraid_state* state, block_off_t countpos, bloc #define PROGRESS_CLEAR " " +void state_progress_stop(struct snapraid_state* state) +{ + time_t now; + + now = time(0); + + printf("\n"); + + state->progress_interruption = now; +} + +void state_progress_restart(struct snapraid_state* state) +{ + time_t now; + + now = time(0); + + if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */ + state->progress_subtract += now - state->progress_interruption; +} + int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize) { if (state->gui) { @@ -1473,7 +1564,7 @@ int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off now = time(0); if (state->progress_last != now) { - time_t delta = now - state->progress_start; + time_t delta = now - state->progress_start - state->progress_subtract; printf("%u%%, %u MiB", countpos * 100 / countmax, (unsigned)(countsize / (1024*1024))); diff --git a/state.h b/state.h index cb7ea40e..946be1fe 100644 --- a/state.h +++ b/state.h @@ -36,6 +36,7 @@ struct snapraid_state { int force_empty; /**< Forced dangerous operations of synching disk now empty. */ int filter_hidden; /**< Filter out hidden files. */ int find_by_name; /**< Forced dangerous operations of synching a rewritten disk. */ + uint64_t autosave; /**< Autosave after the specified amount of data. 0 to disable. */ int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */ int expect_recoverable; /**< Expect presence of recoverable error in checking. */ int need_write; /**< If the state is changed. */ @@ -52,6 +53,8 @@ struct snapraid_state { tommy_list filterlist; /**< List of inclusion/exclusion. */ time_t progress_start; /**< Start of processing for progress visualization. */ time_t progress_last; /**< Last update of progress visualization. */ + time_t progress_interruption; /**< Start of the measure interruption. */ + time_t progress_subtract; /**< Time to subtract for the interruptions. */ }; /** @@ -124,5 +127,15 @@ void state_progress_end(struct snapraid_state* state, block_off_t countpos, bloc */ int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize); +/** + * Stop temporarely the progress. + */ +void state_progress_stop(struct snapraid_state* state); + +/** + * Restart the progress. + */ +void state_progress_restart(struct snapraid_state* state); + #endif diff --git a/sync.c b/sync.c index cf12eefc..04478011 100644 --- a/sync.c +++ b/sync.c @@ -40,6 +40,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari data_off_t countsize; block_off_t countpos; block_off_t countmax; + uint64_t autosavesize; int ret; unsigned unrecoverable_error; @@ -84,6 +85,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } countsize = 0; + autosavesize = 0; countpos = 0; state_progress_begin(state, blockstart, blockmax, countmax); for(i=blockstart;iblock_size; + /* for each disk, process the block */ for(j=0;jautosave != 0 && autosavesize >= state->autosave) { + autosavesize = 0; + + state_progress_stop(state); + + printf("Autosaving...\n"); + state_write(state); + + state_progress_restart(state); + } } state_progress_end(state, countpos, countmax, countsize);