diff --git a/cmdline/parity.c b/cmdline/parity.c index 15e94f9a..59d798fa 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -84,39 +84,6 @@ block_off_t parity_used_size(struct snapraid_state* state) return parity_block; } -int parity_is_invalid(struct snapraid_state* state) -{ - block_off_t blockmax; - block_off_t i; - - blockmax = parity_allocated_size(state); - - for (i = 0; i < blockmax; ++i) { - tommy_node* node_disk; - int one_invalid; - int one_valid; - - /* for each disk */ - one_invalid = 0; - one_valid = 0; - for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) { - struct snapraid_disk* disk = node_disk->data; - struct snapraid_block* block = fs_par2block_find(disk, i); - - if (block_has_file(block)) - one_valid = 1; - if (block_has_invalid_parity(block)) - one_invalid = 1; - } - - /* if both valid and invalid, we need to update */ - if (one_invalid && one_valid) - return 1; - } - - return 0; -} - void parity_overflow(struct snapraid_state* state, data_off_t size) { tommy_node* i; diff --git a/cmdline/parity.h b/cmdline/parity.h index cfe12651..5facce87 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -84,15 +84,6 @@ block_off_t parity_allocated_size(struct snapraid_state* state); */ block_off_t parity_used_size(struct snapraid_state* state); -/** - * Check if the parity needs to be updated with a "sync". - * - * This is the same logic used in "status" to detect an incomplete "sync", - * that ignores invalid block, if they are not used by a file in any disk. - * This means that DELETED blocks won't necessarily imply an invalid parity. - */ -int parity_is_invalid(struct snapraid_state* state); - /** * Report all the files outside the specified parity size. */ diff --git a/cmdline/scan.c b/cmdline/scan.c index 977268ce..54e9cde9 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -2009,7 +2009,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) no_difference = !total.count_move && !total.count_copy && !total.count_restore && !total.count_change && !total.count_remove && !total.count_insert; - int parity_invalid = parity_is_invalid(state); + int parity_invalid = state->parity_is_invalid; if (is_diff) { if (!no_difference) { msg_status("There are differences!\n"); diff --git a/cmdline/state.c b/cmdline/state.c index d432b612..7391e292 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -181,6 +181,7 @@ void state_init(struct snapraid_state* state) state->no_conf = 0; for (i = 0; i < SMART_IGNORE_MAX; ++i) state->smartignore[i] = 0; + state->parity_is_invalid = 0; state->thermal_stop_gathering = 0; state->thermal_ambient_temperature = 0; state->thermal_highest_temperature = 0; @@ -1864,6 +1865,34 @@ static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos) return block_state_get(block) == BLOCK_STATE_DELETED; } +/** + * Check if a block parity needs sync + * + * This is the same logic used in "status" to detect an incomplete "sync", + * that ignores invalid block, if they are not used by a file in any disk. + * + * This means that DELETED blocks won't necessarily imply an invalid parity. + */ +static int fs_is_block_unsynced(struct snapraid_state* state, block_off_t pos) +{ + int one_invalid = 0; + int one_valid = 0; + for (tommy_node* i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + struct snapraid_block* block = fs_par2block_find(disk, pos); + + if (block_has_file(block)) + one_valid = 1; + if (block_has_invalid_parity(block)) + one_invalid = 1; + + if (one_invalid && one_valid) + return 1; + } + + return 0; +} + /** * Flush the file checking the final CRC. * We exploit the fact that the CRC is always stored in the last 4 bytes. @@ -1925,6 +1954,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S unsigned count_dir; unsigned count_bad; unsigned count_rehash; + unsigned count_unsynced; + unsigned count_unscrubbed; int crc_checked; char buffer[PATH_MAX]; int ret; @@ -1938,6 +1969,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S count_dir = 0; count_bad = 0; count_rehash = 0; + count_unsynced = 0; + count_unscrubbed = 0; crc_checked = 0; mapping_max = 0; tommy_array_init(&disk_mapping); @@ -2277,9 +2310,11 @@ static void state_read_content(struct snapraid_state* state, const char* path, S justsynced = (flag & 8) != 0; if (bad) - ++count_bad; + count_bad += v_count; if (rehash) - ++count_rehash; + count_rehash += v_count; + if (justsynced) + count_unscrubbed += v_count; if (rehash && state->prevhash == HASH_UNDEFINED) { /* LCOV_EXCL_START */ @@ -2312,6 +2347,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* they are discarded at the first write */ } + if (fs_is_block_unsynced(state, v_pos)) + ++count_unsynced; + /* go to next block */ ++v_pos; --v_count; @@ -3084,9 +3122,15 @@ static void state_read_content(struct snapraid_state* state, const char* path, S log_tag("content_info:hardlink:%u\n", count_hardlink); log_tag("content_info:symlink:%u\n", count_symlink); log_tag("content_info:dir_empty:%u\n", count_dir); + + log_tag("content_info:block:%u\n", blockmax); log_tag("content_info:block_bad:%u\n", count_bad); log_tag("content_info:block_rehash:%u\n", count_rehash); - log_tag("content_info:block:%u\n", blockmax); + log_tag("content_info:block_unsynced:%u\n", count_unsynced); + log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed); + + /* set the parity validity */ + state->parity_is_invalid = count_unsynced != 0; } struct state_write_thread_context { @@ -3109,6 +3153,8 @@ struct state_write_thread_context { unsigned count_dir; unsigned count_bad; unsigned count_rehash; + unsigned count_unsynced; + unsigned count_unscrubbed; }; static void* state_write_thread(void* arg) @@ -3128,6 +3174,8 @@ static void* state_write_thread(void* arg) unsigned count_dir; unsigned count_bad; unsigned count_rehash; + unsigned count_unsynced; + unsigned count_unscrubbed; tommy_node* i; block_off_t idx; block_off_t begin; @@ -3139,6 +3187,8 @@ static void* state_write_thread(void* arg) count_dir = 0; count_bad = 0; count_rehash = 0; + count_unsynced = 0; + count_unscrubbed = 0; /* force version 3 a we want to always store the parity size */ /* write header */ @@ -3497,20 +3547,28 @@ static void* state_write_thread(void* arg) while (begin < blockmax) { snapraid_info info; block_off_t end; + block_off_t count; time_t t; unsigned flag; info = info_get(&state->infoarr, begin); + /* avoid this slow operation if not needed */ + if (context->first && fs_is_block_unsynced(state, begin)) + ++count_unsynced; + /* find the end of run of blocks */ end = begin + 1; while (end < blockmax && info == info_get(&state->infoarr, end) ) { + if (context->first && fs_is_block_unsynced(state, end)) + ++count_unsynced; ++end; } - sputb32(end - begin, f); + count = end - begin; + sputb32(count, f); /* if there is info */ if (info) { @@ -3518,14 +3576,19 @@ static void* state_write_thread(void* arg) flag = 1; /* info is present */ if (info_get_bad(info)) { flag |= 2; - ++count_bad; + if (context->first) + count_bad += count; } if (info_get_rehash(info)) { flag |= 4; - ++count_rehash; + if (context->first) + count_rehash += count; } - if (info_get_justsynced(info)) + if (info_get_justsynced(info)) { flag |= 8; + if (context->first) + count_unscrubbed += count; + } sputb32(flag, f); t = info_get_time(info); @@ -3599,6 +3662,8 @@ static void* state_write_thread(void* arg) context->count_dir = count_dir; context->count_bad = count_bad; context->count_rehash = count_rehash; + context->count_unsynced = count_unsynced; + context->count_unscrubbed = count_unscrubbed; return 0; } @@ -3629,6 +3694,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) unsigned count_dir; unsigned count_bad; unsigned count_rehash; + unsigned count_unsynced; + unsigned count_unscrubbed; /* blocks of all array */ blockmax = parity_allocated_size(state); @@ -3754,6 +3821,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) count_dir = 0; count_bad = 0; count_rehash = 0; + count_unsynced = 0; + count_unscrubbed = 0; i = tommy_list_head(&state->contentlist); while (i) { struct snapraid_content* content = i->data; @@ -3803,6 +3872,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) count_dir = context->count_dir; cound_bad = context->count_bad; count_rehash = context->count_rehash; + count_unsynced = context->count_unsynced; + count_unscrubbed = context->count_unscrubbed; } else { if (crc != context->crc) { /* LCOV_EXCL_START */ @@ -3927,6 +3998,8 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) count_dir = context->count_dir; count_bad = context->count_bad; count_rehash = context->count_rehash; + count_unsynced = context->count_unsynced; + count_unscrubbed = context->count_unscrubbed; free(context); #endif @@ -3940,9 +4013,15 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) log_tag("content_info:hardlink:%u\n", count_hardlink); log_tag("content_info:symlink:%u\n", count_symlink); log_tag("content_info:dir_empty:%u\n", count_dir); + + log_tag("content_info:block:%u\n", blockmax); log_tag("content_info:block_bad:%u\n", count_bad); log_tag("content_info:block_rehash:%u\n", count_rehash); - log_tag("content_info:block:%u\n", blockmax); + log_tag("content_info:block_unsynced:%u\n", count_unsynced); + log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed); + + /* set the parity validity */ + state->parity_is_invalid = count_unsynced != 0; *out_crc = crc; } diff --git a/cmdline/state.h b/cmdline/state.h index 534dce76..a2448af3 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -142,6 +142,8 @@ struct snapraid_state { unsigned besthash; /**< Best hash suggested. */ const char* command; /**< Command running. */ int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for all devices. */ + int parity_is_invalid; /**< If the parity is invalid, and a sync is needed. */ + tommy_list contentlist; /**< List of content files. */ tommy_list disklist; /**< List of all the disks. */ tommy_list maplist; /**< List of all the disk mappings. */ diff --git a/cmdline/status.c b/cmdline/status.c index aac63ffd..bf475771 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -358,10 +358,6 @@ int state_status(struct snapraid_state* state) } } - log_tag("summary:has_unsynced:%u\n", unsynced_blocks); - log_tag("summary:has_unscrubbed:%u\n", unscrubbed_blocks); - log_tag("summary:has_rehash:%u\n", rehash); - log_tag("summary:has_bad:%u:%u:%u\n", bad, bad_first, bad_last); log_flush(); if (!count) { diff --git a/doc/snapraid_log.1 b/doc/snapraid_log.1 index 5fdd5af4..aee43c53 100644 --- a/doc/snapraid_log.1 +++ b/doc/snapraid_log.1 @@ -300,16 +300,42 @@ The size of the parity file (uint). .RE .TP .B content_info:: -The counters in the content file. +The counters in the content file. The \`kind\` is +one of the following: .RS 4 .PD 0 .HP 4 -.I -One of file, hardlink, symlink, dir_empty, -block_bad, block_rehash, block. +.I file +The count of files. .HP 4 -.I -The number of elements (uint) +.I hardlink +The counter of hardlinks. +.HP 4 +.I symlink +The counter of symlinks. +.HP 4 +.I dir_empty +The counter of empty directories. +.HP 4 +.I block_bad +The counter of bad blocks. Bad blocks indicate data +or parity corruption. +.HP 4 +.I block_rehash +The counter of blocks needing a rehash, usually +due to hash algorithm change. +.HP 4 +.I block_unscrubbed +The counter of blocks that are unscrubbed +(just synced), meaning they have never been verified +since their first sync. +.HP 4 +.I block_unsynced +The counter of blocks that are unsynced (need a sync), +meaning their parity data is out of date. +.HP 4 +.I block +The total counter of blocks. .PD .RE .SH DIAGNOSTICS TAGS @@ -825,22 +851,6 @@ capacity. .B summary:total_use_percent: Overall array use percentage. .TP -.B summary:has_unsynced: -Count of blocks that are unsynced (need a sync), meaning their -parity data is out of date. -.TP -.B summary:has_unscrubbed: -Count of blocks that are unscrubbed (just synced), meaning they -have never been verified since the last sync. -.TP -.B summary:has_rehash: -Count of blocks needing a rehash, usually due to a content file -update or hash algorithm change. -.TP -.B summary:has_bad::: -Count of bad blocks, along with the first and last bad block index. -Bad blocks indicate data or parity corruption. -.TP .B summary:scrub_oldest_days: Days ago the oldest block was last scrubbed. .TP diff --git a/doc/snapraid_log.d b/doc/snapraid_log.d index a8efc34a..e53e680f 100644 --- a/doc/snapraid_log.d +++ b/doc/snapraid_log.d @@ -192,11 +192,23 @@ Content Tags - The size of the parity file (uint). =content_info:: - The counters in the content file. + The counters in the content file. The `kind` is + one of the following: - - One of file, hardlink, symlink, dir_empty, - block_bad, block_rehash, block. - - The number of elements (uint) + file - The count of files. + hardlink - The counter of hardlinks. + symlink - The counter of symlinks. + dir_empty - The counter of empty directories. + block_bad - The counter of bad blocks. Bad blocks indicate data + or parity corruption. + block_rehash - The counter of blocks needing a rehash, usually + due to hash algorithm change. + block_unscrubbed - The counter of blocks that are unscrubbed + (just synced), meaning they have never been verified + since their first sync. + block_unsynced - The counter of blocks that are unsynced (need a sync), + meaning their parity data is out of date. + block - The total counter of blocks. Diagnostics Tags =version: @@ -536,18 +548,6 @@ Command Status Tags capacity. =summary:total_use_percent: Overall array use percentage. - =summary:has_unsynced: - Count of blocks that are unsynced (need a sync), meaning their - parity data is out of date. - =summary:has_unscrubbed: - Count of blocks that are unscrubbed (just synced), meaning they - have never been verified since the last sync. - =summary:has_rehash: - Count of blocks needing a rehash, usually due to a content file - update or hash algorithm change. - =summary:has_bad::: - Count of bad blocks, along with the first and last bad block index. - Bad blocks indicate data or parity corruption. =summary:scrub_oldest_days: Days ago the oldest block was last scrubbed. =summary:scrub_median_days: diff --git a/doc/snapraid_log.txt b/doc/snapraid_log.txt index d792f54f..e6566afd 100644 --- a/doc/snapraid_log.txt +++ b/doc/snapraid_log.txt @@ -202,11 +202,23 @@ These tags report the information stored in the content file. - The size of the parity file (uint). content_info:: - The counters in the content file. + The counters in the content file. The `kind` is + one of the following: - - One of file, hardlink, symlink, dir_empty, - block_bad, block_rehash, block. - - The number of elements (uint) + file - The count of files. + hardlink - The counter of hardlinks. + symlink - The counter of symlinks. + dir_empty - The counter of empty directories. + block_bad - The counter of bad blocks. Bad blocks indicate data + or parity corruption. + block_rehash - The counter of blocks needing a rehash, usually + due to hash algorithm change. + block_unscrubbed - The counter of blocks that are unscrubbed + (just synced), meaning they have never been verified + since their first sync. + block_unsynced - The counter of blocks that are unsynced (need a sync), + meaning their parity data is out of date. + block - The total counter of blocks. 4 DIAGNOSTICS TAGS @@ -561,18 +573,6 @@ and overall state. capacity. summary:total_use_percent: Overall array use percentage. - summary:has_unsynced: - Count of blocks that are unsynced (need a sync), meaning their - parity data is out of date. - summary:has_unscrubbed: - Count of blocks that are unscrubbed (just synced), meaning they - have never been verified since the last sync. - summary:has_rehash: - Count of blocks needing a rehash, usually due to a content file - update or hash algorithm change. - summary:has_bad::: - Count of bad blocks, along with the first and last bad block index. - Bad blocks indicate data or parity corruption. summary:scrub_oldest_days: Days ago the oldest block was last scrubbed. summary:scrub_median_days: