rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Log new block_unsynced and block_unscrubbed counter

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-02-02 15:39:39 +01:00
parent c09bdc572e
commit 60bd71d184
9 changed files with 154 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

33
cmdline/parity.c
View file

@ -84,39 +84,6 @@ block_off_t parity_used_size(struct snapraid_state* state)
return parity_block;
}
int parity_is_invalid(struct snapraid_state* state)
{
block_off_t blockmax;
block_off_t i;
blockmax = parity_allocated_size(state);
for (i = 0; i < blockmax; ++i) {
tommy_node* node_disk;
int one_invalid;
int one_valid;
/* for each disk */
one_invalid = 0;
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = fs_par2block_find(disk, i);
if (block_has_file(block))
one_valid = 1;
if (block_has_invalid_parity(block))
one_invalid = 1;
}
/* if both valid and invalid, we need to update */
if (one_invalid && one_valid)
return 1;
}
return 0;
}
void parity_overflow(struct snapraid_state* state, data_off_t size)
{
tommy_node* i;

9
cmdline/parity.h
View file

@ -84,15 +84,6 @@ block_off_t parity_allocated_size(struct snapraid_state* state);
*/
block_off_t parity_used_size(struct snapraid_state* state);
/**
* Check if the parity needs to be updated with a "sync".
*
* This is the same logic used in "status" to detect an incomplete "sync",
* that ignores invalid block, if they are not used by a file in any disk.
* This means that DELETED blocks won't necessarily imply an invalid parity.
*/
int parity_is_invalid(struct snapraid_state* state);
/**
* Report all the files outside the specified parity size.
*/

2
cmdline/scan.c
View file

@ -2009,7 +2009,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
no_difference = !total.count_move && !total.count_copy && !total.count_restore
&& !total.count_change && !total.count_remove && !total.count_insert;
int parity_invalid = parity_is_invalid(state);
int parity_invalid = state->parity_is_invalid;
if (is_diff) {
if (!no_difference) {
msg_status("There are differences!\n");

95
cmdline/state.c
View file

@ -181,6 +181,7 @@ void state_init(struct snapraid_state* state)
state->no_conf = 0;
for (i = 0; i < SMART_IGNORE_MAX; ++i)
state->smartignore[i] = 0;
state->parity_is_invalid = 0;
state->thermal_stop_gathering = 0;
state->thermal_ambient_temperature = 0;
state->thermal_highest_temperature = 0;
@ -1864,6 +1865,34 @@ static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
return block_state_get(block) == BLOCK_STATE_DELETED;
}
/**
* Check if a block parity needs sync
*
* This is the same logic used in "status" to detect an incomplete "sync",
* that ignores invalid block, if they are not used by a file in any disk.
*
* This means that DELETED blocks won't necessarily imply an invalid parity.
*/
static int fs_is_block_unsynced(struct snapraid_state* state, block_off_t pos)
{
int one_invalid = 0;
int one_valid = 0;
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = fs_par2block_find(disk, pos);
if (block_has_file(block))
one_valid = 1;
if (block_has_invalid_parity(block))
one_invalid = 1;
if (one_invalid && one_valid)
return 1;
}
return 0;
}
/**
* Flush the file checking the final CRC.
* We exploit the fact that the CRC is always stored in the last 4 bytes.
@ -1925,6 +1954,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
int crc_checked;
char buffer[PATH_MAX];
int ret;
@ -1938,6 +1969,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
count_dir = 0;
count_bad = 0;
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
crc_checked = 0;
mapping_max = 0;
tommy_array_init(&disk_mapping);
@ -2277,9 +2310,11 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
justsynced = (flag & 8) != 0;
if (bad)
++count_bad;
count_bad += v_count;
if (rehash)
++count_rehash;
count_rehash += v_count;
if (justsynced)
count_unscrubbed += v_count;
if (rehash && state->prevhash == HASH_UNDEFINED) {
/* LCOV_EXCL_START */
@ -2312,6 +2347,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* they are discarded at the first write */
}
if (fs_is_block_unsynced(state, v_pos))
++count_unsynced;
/* go to next block */
++v_pos;
--v_count;
@ -3084,9 +3122,15 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
log_tag("content_info:hardlink:%u\n", count_hardlink);
log_tag("content_info:symlink:%u\n", count_symlink);
log_tag("content_info:dir_empty:%u\n", count_dir);
log_tag("content_info:block:%u\n", blockmax);
log_tag("content_info:block_bad:%u\n", count_bad);
log_tag("content_info:block_rehash:%u\n", count_rehash);
log_tag("content_info:block:%u\n", blockmax);
log_tag("content_info:block_unsynced:%u\n", count_unsynced);
log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
/* set the parity validity */
state->parity_is_invalid = count_unsynced != 0;
}
struct state_write_thread_context {
@ -3109,6 +3153,8 @@ struct state_write_thread_context {
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
};
static void* state_write_thread(void* arg)
@ -3128,6 +3174,8 @@ static void* state_write_thread(void* arg)
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
tommy_node* i;
block_off_t idx;
block_off_t begin;
@ -3139,6 +3187,8 @@ static void* state_write_thread(void* arg)
count_dir = 0;
count_bad = 0;
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
/* force version 3 a we want to always store the parity size */
/* write header */
@ -3497,20 +3547,28 @@ static void* state_write_thread(void* arg)
while (begin < blockmax) {
snapraid_info info;
block_off_t end;
block_off_t count;
time_t t;
unsigned flag;
info = info_get(&state->infoarr, begin);
/* avoid this slow operation if not needed */
if (context->first && fs_is_block_unsynced(state, begin))
++count_unsynced;
/* find the end of run of blocks */
end = begin + 1;
while (end < blockmax
&& info == info_get(&state->infoarr, end)
) {
if (context->first && fs_is_block_unsynced(state, end))
++count_unsynced;
++end;
}
sputb32(end - begin, f);
count = end - begin;
sputb32(count, f);
/* if there is info */
if (info) {
@ -3518,14 +3576,19 @@ static void* state_write_thread(void* arg)
flag = 1; /* info is present */
if (info_get_bad(info)) {
flag |= 2;
++count_bad;
if (context->first)
count_bad += count;
}
if (info_get_rehash(info)) {
flag |= 4;
++count_rehash;
if (context->first)
count_rehash += count;
}
if (info_get_justsynced(info))
if (info_get_justsynced(info)) {
flag |= 8;
if (context->first)
count_unscrubbed += count;
}
sputb32(flag, f);
t = info_get_time(info);
@ -3599,6 +3662,8 @@ static void* state_write_thread(void* arg)
context->count_dir = count_dir;
context->count_bad = count_bad;
context->count_rehash = count_rehash;
context->count_unsynced = count_unsynced;
context->count_unscrubbed = count_unscrubbed;
return 0;
}
@ -3629,6 +3694,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
/* blocks of all array */
blockmax = parity_allocated_size(state);
@ -3754,6 +3821,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
count_dir = 0;
count_bad = 0;
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
@ -3803,6 +3872,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
count_dir = context->count_dir;
cound_bad = context->count_bad;
count_rehash = context->count_rehash;
count_unsynced = context->count_unsynced;
count_unscrubbed = context->count_unscrubbed;
} else {
if (crc != context->crc) {
/* LCOV_EXCL_START */
@ -3927,6 +3998,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
count_dir = context->count_dir;
count_bad = context->count_bad;
count_rehash = context->count_rehash;
count_unsynced = context->count_unsynced;
count_unscrubbed = context->count_unscrubbed;
free(context);
#endif
@ -3940,9 +4013,15 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
log_tag("content_info:hardlink:%u\n", count_hardlink);
log_tag("content_info:symlink:%u\n", count_symlink);
log_tag("content_info:dir_empty:%u\n", count_dir);
log_tag("content_info:block:%u\n", blockmax);
log_tag("content_info:block_bad:%u\n", count_bad);
log_tag("content_info:block_rehash:%u\n", count_rehash);
log_tag("content_info:block:%u\n", blockmax);
log_tag("content_info:block_unsynced:%u\n", count_unsynced);
log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
/* set the parity validity */
state->parity_is_invalid = count_unsynced != 0;
*out_crc = crc;
}

2
cmdline/state.h
View file

@ -142,6 +142,8 @@ struct snapraid_state {
unsigned besthash; /**< Best hash suggested. */
const char* command; /**< Command running. */
int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for all devices. */
int parity_is_invalid; /**< If the parity is invalid, and a sync is needed. */
tommy_list contentlist; /**< List of content files. */
tommy_list disklist; /**< List of all the disks. */
tommy_list maplist; /**< List of all the disk mappings. */

4
cmdline/status.c
View file

@ -358,10 +358,6 @@ int state_status(struct snapraid_state* state)
}
}
log_tag("summary:has_unsynced:%u\n", unsynced_blocks);
log_tag("summary:has_unscrubbed:%u\n", unscrubbed_blocks);
log_tag("summary:has_rehash:%u\n", rehash);
log_tag("summary:has_bad:%u:%u:%u\n", bad, bad_first, bad_last);
log_flush();
if (!count) {

54
doc/snapraid_log.1
View file

@ -300,16 +300,42 @@ The size of the parity file (uint).
.RE
.TP
.B content_info:<kind>:<counter>
The counters in the content file.
The counters in the content file. The \`kind\` is
one of the following:
.RS 4
.PD 0
.HP 4
.I <kind>
One of file, hardlink, symlink, dir_empty,
block_bad, block_rehash, block.
.I file
The count of files.
.HP 4
.I <counter>
The number of elements (uint)
.I hardlink
The counter of hardlinks.
.HP 4
.I symlink
The counter of symlinks.
.HP 4
.I dir_empty
The counter of empty directories.
.HP 4
.I block_bad
The counter of bad blocks. Bad blocks indicate data
or parity corruption.
.HP 4
.I block_rehash
The counter of blocks needing a rehash, usually
due to hash algorithm change.
.HP 4
.I block_unscrubbed
The counter of blocks that are unscrubbed
(just synced), meaning they have never been verified
since their first sync.
.HP 4
.I block_unsynced
The counter of blocks that are unsynced (need a sync),
meaning their parity data is out of date.
.HP 4
.I block
The total counter of blocks.
.PD
.RE
.SH DIAGNOSTICS TAGS
@ -825,22 +851,6 @@ capacity.
.B summary:total_use_percent:<uint>
Overall array use percentage.
.TP
.B summary:has_unsynced:<uint>
Count of blocks that are unsynced (need a sync), meaning their
parity data is out of date.
.TP
.B summary:has_unscrubbed:<uint>
Count of blocks that are unscrubbed (just synced), meaning they
have never been verified since the last sync.
.TP
.B summary:has_rehash:<uint>
Count of blocks needing a rehash, usually due to a content file
update or hash algorithm change.
.TP
.B summary:has_bad:<count>:<first>:<last>
Count of bad blocks, along with the first and last bad block index.
Bad blocks indicate data or parity corruption.
.TP
.B summary:scrub_oldest_days:<uint>
Days ago the oldest block was last scrubbed.
.TP

32
doc/snapraid_log.d
View file

@ -192,11 +192,23 @@ Content Tags
<size> - The size of the parity file (uint).
=content_info:<kind>:<counter>
The counters in the content file.
The counters in the content file. The `kind` is
one of the following:
<kind> - One of file, hardlink, symlink, dir_empty,
block_bad, block_rehash, block.
<counter> - The number of elements (uint)
file - The count of files.
hardlink - The counter of hardlinks.
symlink - The counter of symlinks.
dir_empty - The counter of empty directories.
block_bad - The counter of bad blocks. Bad blocks indicate data
or parity corruption.
block_rehash - The counter of blocks needing a rehash, usually
due to hash algorithm change.
block_unscrubbed - The counter of blocks that are unscrubbed
(just synced), meaning they have never been verified
since their first sync.
block_unsynced - The counter of blocks that are unsynced (need a sync),
meaning their parity data is out of date.
block - The total counter of blocks.
Diagnostics Tags
=version:<version>
@ -536,18 +548,6 @@ Command Status Tags
capacity.
=summary:total_use_percent:<uint>
Overall array use percentage.
=summary:has_unsynced:<uint>
Count of blocks that are unsynced (need a sync), meaning their
parity data is out of date.
=summary:has_unscrubbed:<uint>
Count of blocks that are unscrubbed (just synced), meaning they
have never been verified since the last sync.
=summary:has_rehash:<uint>
Count of blocks needing a rehash, usually due to a content file
update or hash algorithm change.
=summary:has_bad:<count>:<first>:<last>
Count of bad blocks, along with the first and last bad block index.
Bad blocks indicate data or parity corruption.
=summary:scrub_oldest_days:<uint>
Days ago the oldest block was last scrubbed.
=summary:scrub_median_days:<uint>

32
doc/snapraid_log.txt
View file

@ -202,11 +202,23 @@ These tags report the information stored in the content file.
<size> - The size of the parity file (uint).
content_info:<kind>:<counter>
The counters in the content file.
The counters in the content file. The `kind` is
one of the following:
<kind> - One of file, hardlink, symlink, dir_empty,
block_bad, block_rehash, block.
<counter> - The number of elements (uint)
file - The count of files.
hardlink - The counter of hardlinks.
symlink - The counter of symlinks.
dir_empty - The counter of empty directories.
block_bad - The counter of bad blocks. Bad blocks indicate data
or parity corruption.
block_rehash - The counter of blocks needing a rehash, usually
due to hash algorithm change.
block_unscrubbed - The counter of blocks that are unscrubbed
(just synced), meaning they have never been verified
since their first sync.
block_unsynced - The counter of blocks that are unsynced (need a sync),
meaning their parity data is out of date.
block - The total counter of blocks.
4 DIAGNOSTICS TAGS
@ -561,18 +573,6 @@ and overall state.
capacity.
summary:total_use_percent:<uint>
Overall array use percentage.
summary:has_unsynced:<uint>
Count of blocks that are unsynced (need a sync), meaning their
parity data is out of date.
summary:has_unscrubbed:<uint>
Count of blocks that are unscrubbed (just synced), meaning they
have never been verified since the last sync.
summary:has_rehash:<uint>
Count of blocks needing a rehash, usually due to a content file
update or hash algorithm change.
summary:has_bad:<count>:<first>:<last>
Count of bad blocks, along with the first and last bad block index.
Bad blocks indicate data or parity corruption.
summary:scrub_oldest_days:<uint>
Days ago the oldest block was last scrubbed.
summary:scrub_median_days:<uint>