rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Scrub first the just synced blocks

Browse source
This commit is contained in:
Andrea Mazzoleni 2015-08-27 18:58:25 +02:00
parent 9a5f30cd32
commit cca2cbb23f
9 changed files with 74 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

8
HISTORY
View file

@ -3,8 +3,14 @@ SnapRAID HISTORY
8.2 2015/08
===========
* The synced blocks are now scrubbed with priority over all
the already scrubbed ones.
This allows to verify as early as possible that the written
parity during sync is really correct.
The "status" command reports the amount of scrub necessary to
ensure to scrub all the synched blocks.
* Update 'best' RAID functions for recent Atom CPUs.
* Validate filters specificatinos rejecting relative paths.
* Validate filters specifications rejecting relative paths.
8.1 2015/05
===========

9
TODO
View file

@ -5,15 +5,6 @@ This is the list of TODO items for SnapRAID.
- Next
* Allow to scrub/check/verify the just synched data.
Like an option to scrub only recent data.
Or more simply, we can add a new option at sync that avoid
to write the time info for just synched data, to ensure
that is scrubbed before any already scrubbed one.
Maybe we can also make this the new default behaviour,
as this is something that make sense, and make the next
version 9.0.
* Improvement in fix, due missing disks, when using -d DISK.
A. Don't halt due to missing content files as long as there is one valid content file
B. Don't halt due to missing disks when enough parity exist to replace them.

13
cmdline/elem.c
View file

@ -800,17 +800,14 @@ void map_free(struct snapraid_map* map)
free(map);
}
int info_time_compare(const void* void_a, const void* void_b)
int time_compare(const void* void_a, const void* void_b)
{
const snapraid_info* info_a = void_a;
const snapraid_info* info_b = void_b;
time_t time_a = info_get_time(*info_a);
time_t time_b = info_get_time(*info_b);
const time_t* time_a = void_a;
const time_t* time_b = void_b;
if (time_a < time_b)
if (*time_a < *time_b)
return -1;
if (time_a > time_b)
if (*time_a > *time_b)
return 1;
return 0;
}

17
cmdline/elem.h
View file

@ -975,6 +975,19 @@ static inline int info_get_justsynced(snapraid_info info)
return (info & 0x4) != 0;
}
/**
* Get the latest scrub time.
*
* If the block was not scrubbed, 0 is returned.
*/
static inline time_t info_get_scrubtime(snapraid_info info)
{
if (info_get_justsynced(info))
return 0;
else
return info_get_time(info);
}
/**
* Marks the block address as with error.
*/
@ -1019,9 +1032,9 @@ static inline snapraid_info info_get(tommy_arrayblkof* array, unsigned pos)
}
/**
* Compares info by time.
* Compare times
*/
int info_time_compare(const void* void_a, const void* void_b);
int time_compare(const void* void_a, const void* void_b);
#endif

22
cmdline/scrub.c
View file

@ -54,7 +54,7 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, time_t
if (!info_get_bad(info)) {
/* if it's too new */
blocktime = info_get_time(info);
blocktime = info_get_scrubtime(info);
if (blocktime > timelimit) {
/* skip it */
return 0;
@ -561,7 +561,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
int ret;
struct snapraid_parity_handle parity[LEV_MAX];
struct snapraid_parity_handle* parity_ptr[LEV_MAX];
snapraid_info* infomap;
time_t* timemap;
unsigned error;
time_t now;
unsigned l;
@ -598,7 +598,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
/* identify the time limit */
/* we sort all the block times, and we identify the time limit for which we reach the quota */
/* this allow to process first the oldest blocks */
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
timemap = malloc_nofail(blockmax * sizeof(time_t));
/* copy the info in the temp vector */
count = 0;
@ -610,7 +610,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
if (info == 0)
continue;
infomap[count++] = info;
timemap[count++] = info_get_scrubtime(info);
}
if (!count) {
@ -621,16 +621,16 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
}
/* sort it */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
qsort(timemap, count, sizeof(time_t), time_compare);
/* output the info map */
i = 0;
log_tag("info_count:%u\n", count);
while (i < count) {
unsigned j = i + 1;
while (j < count && info_get_time(infomap[i]) == info_get_time(infomap[j]))
while (j < count && timemap[i] == timemap[j])
++j;
log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)info_get_time(infomap[i]), j - i);
log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)timemap[i], j - i);
i = j;
}
@ -639,18 +639,18 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
countlimit = count;
/* decrease until we reach the specific recentlimit */
while (countlimit > 0 && info_get_time(infomap[countlimit - 1]) > recentlimit)
while (countlimit > 0 && timemap[countlimit - 1] > recentlimit)
--countlimit;
/* if there is something to scrub */
if (countlimit > 0) {
/* get the most recent time we want to scrub */
timelimit = info_get_time(infomap[countlimit - 1]);
timelimit = timemap[countlimit - 1];
/* count how many entries for this exact time we have to scrub */
/* if the blocks have all the same time, we end with countlimit == lastlimit */
lastlimit = 1;
while (countlimit > lastlimit && info_get_time(infomap[countlimit - lastlimit - 1]) == timelimit)
while (countlimit > lastlimit && timemap[countlimit - lastlimit - 1] == timelimit)
++lastlimit;
} else {
/* if nothing to scrub, disable also other limits */
@ -663,7 +663,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
log_tag("last_limit:%u\n", lastlimit);
/* free the temp vector */
free(infomap);
free(timemap);
/* open the file for reading */
for (l = 0; l < state->level; ++l) {

38
cmdline/status.c
View file

@ -47,7 +47,7 @@ int state_status(struct snapraid_state* state)
{
block_off_t blockmax;
block_off_t i;
snapraid_info* infomap;
time_t* timemap;
time_t now;
block_off_t bad;
block_off_t bad_first;
@ -70,6 +70,7 @@ int state_status(struct snapraid_state* state)
uint64_t file_block_free;
block_off_t parity_block_free;
unsigned unsynced_blocks;
unsigned unscrubbed_blocks;
uint64_t all_wasted;
int free_not_zero;
@ -258,13 +259,14 @@ int state_status(struct snapraid_state* state)
log_flush();
/* copy the info a temp vector, and count bad/rehash/unsynced blocks */
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
timemap = malloc_nofail(blockmax * sizeof(time_t));
bad = 0;
bad_first = 0;
bad_last = 0;
count = 0;
rehash = 0;
unsynced_blocks = 0;
unscrubbed_blocks = 0;
log_tag("block_count:%u\n", blockmax);
for (i = 0; i < blockmax; ++i) {
int one_invalid;
@ -302,7 +304,10 @@ int state_status(struct snapraid_state* state)
if (info_get_rehash(info))
++rehash;
infomap[count++] = info;
if (info_get_justsynced(info))
++unscrubbed_blocks;
timemap[count++] = info_get_time(info);
}
if (state->opt.gui) {
@ -314,33 +319,34 @@ int state_status(struct snapraid_state* state)
}
log_tag("summary:has_unsynced:%u\n", unsynced_blocks);
log_tag("summary:has_unscrubbed:%u\n", unscrubbed_blocks);
log_tag("summary:has_rehash:%u\n", rehash);
log_tag("summary:has_bad:%u:%u:%u\n", bad, bad_first, bad_last);
log_flush();
if (!count) {
log_fatal("The array is empty.\n");
free(infomap);
free(timemap);
return 0;
}
/* sort the info to get the time info */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
qsort(timemap, count, sizeof(time_t), time_compare);
/* output the info map */
i = 0;
log_tag("info_count:%u\n", count);
while (i < count) {
unsigned j = i + 1;
while (j < count && info_get_time(infomap[i]) == info_get_time(infomap[j]))
while (j < count && timemap[i] == timemap[j])
++j;
log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)info_get_time(infomap[i]), j - i);
log_tag("info_time:%" PRIu64 ":%u\n", (uint64_t)timemap[i], j - i);
i = j;
}
oldest = info_get_time(infomap[0]);
median = info_get_time(infomap[count / 2]);
newest = info_get_time(infomap[count - 1]);
oldest = info_get_time(timemap[0]);
median = info_get_time(timemap[count / 2]);
newest = info_get_time(timemap[count - 1]);
dayoldest = day_ago(oldest, now);
daymedian = day_ago(median, now);
daynewest = day_ago(newest, now);
@ -355,7 +361,7 @@ int state_status(struct snapraid_state* state)
limit = oldest + (newest - oldest) * (i + 1) / GRAPH_COLUMN;
step = 0;
while (barpos < count && info_get_time(infomap[barpos]) <= limit) {
while (barpos < count && timemap[barpos] <= limit) {
++barpos;
++step;
}
@ -402,7 +408,7 @@ int state_status(struct snapraid_state* state)
}
printf("\n");
}
printf(" %3u days ago of the last scrub %3u\n", dayoldest, daynewest);
printf(" %3u days ago of the last scrub/sync %3u\n", dayoldest, daynewest);
printf("\n");
@ -417,6 +423,12 @@ int state_status(struct snapraid_state* state)
printf("No sync is in progress.\n");
}
if (unscrubbed_blocks) {
printf("You need to scrub at least the %u%% to check the latest sync.\n", (unscrubbed_blocks * 100 + blockmax - 1) / blockmax);
} else {
printf("The full array was scrubbed at least one time.\n");
}
if (rehash) {
printf("You have a rehash in progress at %u%%.\n", (count - rehash) * 100 / count);
} else {
@ -463,7 +475,7 @@ int state_status(struct snapraid_state* state)
}
/* free the temp vector */
free(infomap);
free(timemap);
return 0;
}

4
snapraid.1
View file

@ -780,7 +780,9 @@ You can use the \-p, \-\-percentage option to specify a different amount,
and the \-o, \-\-older\-than option to specify a different age in days.
You can have a full scrub with \[dq]\-p 100 \-o 0\[dq].
.PP
The oldest blocks are scrubbed first ensuring an optimal check.
The just synched blocks are scrubbed first, then are scrubbed the blocks
with the oldest scrub time.
To get some stats of the scrub times, you can use the \[dq]status\[dq] command.
.PP
For any silent or input/output error found the corresponding blocks
are marked as bad in the \[dq]content\[dq] file.

8
snapraid.d
View file

@ -489,7 +489,9 @@ Commands
and the -o, --older-than option to specify a different age in days.
You can have a full scrub with "-p 100 -o 0".
The oldest blocks are scrubbed first ensuring an optimal check.
The just synched blocks never scrubbed, are scrubbed first, then are
scrubbed the blocks with the oldest scrub time.
To get some stats of the scrub times, you can use the "status" command.
For any silent or input/output error found the corresponding blocks
are marked as bad in the "content" file.
@ -498,8 +500,8 @@ Commands
corrected, the bad mark will be removed.
To scrub only the bad blocks, you can use the "scrub -p 0" command.
It's recommended to run "scrub" on a synced array, to avoid to have
reported error caused by unsynced data. These errors are recognized
It's recommended to run "scrub" only on a synced array, to avoid to
have reported error caused by unsynced data. These errors are recognized
as not being silent errors, and the blocks are not marked as bad,
but such errors are reported in the output of the command.

4
snapraid.txt
View file

@ -527,7 +527,9 @@ You can use the -p, --percentage option to specify a different amount,
and the -o, --older-than option to specify a different age in days.
You can have a full scrub with "-p 100 -o 0".
The oldest blocks are scrubbed first ensuring an optimal check.
The just synched blocks are scrubbed first, then are scrubbed the blocks
with the oldest scrub time.
To get some stats of the scrub times, you can use the "status" command.
For any silent or input/output error found the corresponding blocks
are marked as bad in the "content" file.