diff --git a/HISTORY b/HISTORY index d8693d9f..d75ab4d5 100644 --- a/HISTORY +++ b/HISTORY @@ -3,8 +3,14 @@ SnapRAID HISTORY 8.2 2015/08 =========== + * The synced blocks are now scrubbed with priority over all + the already scrubbed ones. + This allows to verify as early as possible that the written + parity during sync is really correct. + The "status" command reports the amount of scrub necessary to + ensure to scrub all the synched blocks. * Update 'best' RAID functions for recent Atom CPUs. - * Validate filters specificatinos rejecting relative paths. + * Validate filters specifications rejecting relative paths. 8.1 2015/05 =========== diff --git a/TODO b/TODO index 2b7d7654..e3be8bf2 100644 --- a/TODO +++ b/TODO @@ -5,15 +5,6 @@ This is the list of TODO items for SnapRAID. - Next -* Allow to scrub/check/verify the just synched data. -Like an option to scrub only recent data. -Or more simply, we can add a new option at sync that avoid -to write the time info for just synched data, to ensure -that is scrubbed before any already scrubbed one. -Maybe we can also make this the new default behaviour, -as this is something that make sense, and make the next -version 9.0. - * Improvement in fix, due missing disks, when using -d DISK. A. Don't halt due to missing content files as long as there is one valid content file B. Don't halt due to missing disks when enough parity exist to replace them. diff --git a/cmdline/elem.c b/cmdline/elem.c index 7cffbc33..b79076a6 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -800,17 +800,14 @@ void map_free(struct snapraid_map* map) free(map); } -int info_time_compare(const void* void_a, const void* void_b) +int time_compare(const void* void_a, const void* void_b) { - const snapraid_info* info_a = void_a; - const snapraid_info* info_b = void_b; - time_t time_a = info_get_time(*info_a); - time_t time_b = info_get_time(*info_b); + const time_t* time_a = void_a; + const time_t* time_b = void_b; - if (time_a < time_b) + if (*time_a < *time_b) return -1; - if (time_a > time_b) + if (*time_a > *time_b) return 1; return 0; } - diff --git a/cmdline/elem.h b/cmdline/elem.h index fa6a95f4..51f9496e 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -975,6 +975,19 @@ static inline int info_get_justsynced(snapraid_info info) return (info & 0x4) != 0; } +/** + * Get the latest scrub time. + * + * If the block was not scrubbed, 0 is returned. + */ +static inline time_t info_get_scrubtime(snapraid_info info) +{ + if (info_get_justsynced(info)) + return 0; + else + return info_get_time(info); +} + /** * Marks the block address as with error. */ @@ -1019,9 +1032,9 @@ static inline snapraid_info info_get(tommy_arrayblkof* array, unsigned pos) } /** - * Compares info by time. + * Compare times */ -int info_time_compare(const void* void_a, const void* void_b); +int time_compare(const void* void_a, const void* void_b); #endif diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 12745e5b..c90f148c 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -54,7 +54,7 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, time_t if (!info_get_bad(info)) { /* if it's too new */ - blocktime = info_get_time(info); + blocktime = info_get_scrubtime(info); if (blocktime > timelimit) { /* skip it */ return 0; @@ -561,7 +561,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) int ret; struct snapraid_parity_handle parity[LEV_MAX]; struct snapraid_parity_handle* parity_ptr[LEV_MAX]; - snapraid_info* infomap; + time_t* timemap; unsigned error; time_t now; unsigned l; @@ -598,7 +598,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) /* identify the time limit */ /* we sort all the block times, and we identify the time limit for which we reach the quota */ /* this allow to process first the oldest blocks */ - infomap = malloc_nofail(blockmax * sizeof(snapraid_info)); + timemap = malloc_nofail(blockmax * sizeof(time_t)); /* copy the info in the temp vector */ count = 0; @@ -610,7 +610,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) if (info == 0) continue; - infomap[count++] = info; + timemap[count++] = info_get_scrubtime(info); } if (!count) { @@ -621,16 +621,16 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) } /* sort it */ - qsort(infomap, count, sizeof(snapraid_info), info_time_compare); + qsort(timemap, count, sizeof(time_t), time_compare); /* output the info map */ i = 0; log_tag("info_count:%u\n", count); while (i < count) { unsigned j = i + 1; - while (j < count && info_get_time(infomap[i]) == info_get_time(infomap[j])) + while (j < count && timemap[i] == timemap[j]) ++j; - log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)info_get_time(infomap[i]), j - i); + log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)timemap[i], j - i); i = j; } @@ -639,18 +639,18 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) countlimit = count; /* decrease until we reach the specific recentlimit */ - while (countlimit > 0 && info_get_time(infomap[countlimit - 1]) > recentlimit) + while (countlimit > 0 && timemap[countlimit - 1] > recentlimit) --countlimit; /* if there is something to scrub */ if (countlimit > 0) { /* get the most recent time we want to scrub */ - timelimit = info_get_time(infomap[countlimit - 1]); + timelimit = timemap[countlimit - 1]; /* count how many entries for this exact time we have to scrub */ /* if the blocks have all the same time, we end with countlimit == lastlimit */ lastlimit = 1; - while (countlimit > lastlimit && info_get_time(infomap[countlimit - lastlimit - 1]) == timelimit) + while (countlimit > lastlimit && timemap[countlimit - lastlimit - 1] == timelimit) ++lastlimit; } else { /* if nothing to scrub, disable also other limits */ @@ -663,7 +663,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) log_tag("last_limit:%u\n", lastlimit); /* free the temp vector */ - free(infomap); + free(timemap); /* open the file for reading */ for (l = 0; l < state->level; ++l) { diff --git a/cmdline/status.c b/cmdline/status.c index 6060855f..dc3c2c6e 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -47,7 +47,7 @@ int state_status(struct snapraid_state* state) { block_off_t blockmax; block_off_t i; - snapraid_info* infomap; + time_t* timemap; time_t now; block_off_t bad; block_off_t bad_first; @@ -70,6 +70,7 @@ int state_status(struct snapraid_state* state) uint64_t file_block_free; block_off_t parity_block_free; unsigned unsynced_blocks; + unsigned unscrubbed_blocks; uint64_t all_wasted; int free_not_zero; @@ -258,13 +259,14 @@ int state_status(struct snapraid_state* state) log_flush(); /* copy the info a temp vector, and count bad/rehash/unsynced blocks */ - infomap = malloc_nofail(blockmax * sizeof(snapraid_info)); + timemap = malloc_nofail(blockmax * sizeof(time_t)); bad = 0; bad_first = 0; bad_last = 0; count = 0; rehash = 0; unsynced_blocks = 0; + unscrubbed_blocks = 0; log_tag("block_count:%u\n", blockmax); for (i = 0; i < blockmax; ++i) { int one_invalid; @@ -302,7 +304,10 @@ int state_status(struct snapraid_state* state) if (info_get_rehash(info)) ++rehash; - infomap[count++] = info; + if (info_get_justsynced(info)) + ++unscrubbed_blocks; + + timemap[count++] = info_get_time(info); } if (state->opt.gui) { @@ -314,33 +319,34 @@ int state_status(struct snapraid_state* state) } log_tag("summary:has_unsynced:%u\n", unsynced_blocks); + log_tag("summary:has_unscrubbed:%u\n", unscrubbed_blocks); log_tag("summary:has_rehash:%u\n", rehash); log_tag("summary:has_bad:%u:%u:%u\n", bad, bad_first, bad_last); log_flush(); if (!count) { log_fatal("The array is empty.\n"); - free(infomap); + free(timemap); return 0; } /* sort the info to get the time info */ - qsort(infomap, count, sizeof(snapraid_info), info_time_compare); + qsort(timemap, count, sizeof(time_t), time_compare); /* output the info map */ i = 0; log_tag("info_count:%u\n", count); while (i < count) { unsigned j = i + 1; - while (j < count && info_get_time(infomap[i]) == info_get_time(infomap[j])) + while (j < count && timemap[i] == timemap[j]) ++j; - log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)info_get_time(infomap[i]), j - i); + log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)timemap[i], j - i); i = j; } - oldest = info_get_time(infomap[0]); - median = info_get_time(infomap[count / 2]); - newest = info_get_time(infomap[count - 1]); + oldest = info_get_time(timemap[0]); + median = info_get_time(timemap[count / 2]); + newest = info_get_time(timemap[count - 1]); dayoldest = day_ago(oldest, now); daymedian = day_ago(median, now); daynewest = day_ago(newest, now); @@ -355,7 +361,7 @@ int state_status(struct snapraid_state* state) limit = oldest + (newest - oldest) * (i + 1) / GRAPH_COLUMN; step = 0; - while (barpos < count && info_get_time(infomap[barpos]) <= limit) { + while (barpos < count && timemap[barpos] <= limit) { ++barpos; ++step; } @@ -402,7 +408,7 @@ int state_status(struct snapraid_state* state) } printf("\n"); } - printf(" %3u days ago of the last scrub %3u\n", dayoldest, daynewest); + printf(" %3u days ago of the last scrub/sync %3u\n", dayoldest, daynewest); printf("\n"); @@ -417,6 +423,12 @@ int state_status(struct snapraid_state* state) printf("No sync is in progress.\n"); } + if (unscrubbed_blocks) { + printf("You need to scrub at least the %u%% to check the latest sync.\n", (unscrubbed_blocks * 100 + blockmax - 1) / blockmax); + } else { + printf("The full array was scrubbed at least one time.\n"); + } + if (rehash) { printf("You have a rehash in progress at %u%%.\n", (count - rehash) * 100 / count); } else { @@ -463,7 +475,7 @@ int state_status(struct snapraid_state* state) } /* free the temp vector */ - free(infomap); + free(timemap); return 0; } diff --git a/snapraid.1 b/snapraid.1 index dce6ff5f..4c667a96 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -780,7 +780,9 @@ You can use the \-p, \-\-percentage option to specify a different amount, and the \-o, \-\-older\-than option to specify a different age in days. You can have a full scrub with \[dq]\-p 100 \-o 0\[dq]. .PP -The oldest blocks are scrubbed first ensuring an optimal check. +The just synched blocks are scrubbed first, then are scrubbed the blocks +with the oldest scrub time. +To get some stats of the scrub times, you can use the \[dq]status\[dq] command. .PP For any silent or input/output error found the corresponding blocks are marked as bad in the \[dq]content\[dq] file. diff --git a/snapraid.d b/snapraid.d index 57a5a0b9..b517279b 100644 --- a/snapraid.d +++ b/snapraid.d @@ -489,7 +489,9 @@ Commands and the -o, --older-than option to specify a different age in days. You can have a full scrub with "-p 100 -o 0". - The oldest blocks are scrubbed first ensuring an optimal check. + The just synched blocks never scrubbed, are scrubbed first, then are + scrubbed the blocks with the oldest scrub time. + To get some stats of the scrub times, you can use the "status" command. For any silent or input/output error found the corresponding blocks are marked as bad in the "content" file. @@ -498,8 +500,8 @@ Commands corrected, the bad mark will be removed. To scrub only the bad blocks, you can use the "scrub -p 0" command. - It's recommended to run "scrub" on a synced array, to avoid to have - reported error caused by unsynced data. These errors are recognized + It's recommended to run "scrub" only on a synced array, to avoid to + have reported error caused by unsynced data. These errors are recognized as not being silent errors, and the blocks are not marked as bad, but such errors are reported in the output of the command. diff --git a/snapraid.txt b/snapraid.txt index 9061443d..70045689 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -527,7 +527,9 @@ You can use the -p, --percentage option to specify a different amount, and the -o, --older-than option to specify a different age in days. You can have a full scrub with "-p 100 -o 0". -The oldest blocks are scrubbed first ensuring an optimal check. +The just synched blocks are scrubbed first, then are scrubbed the blocks +with the oldest scrub time. +To get some stats of the scrub times, you can use the "status" command. For any silent or input/output error found the corresponding blocks are marked as bad in the "content" file.