From cea7ce7f0a4f166dc82bbdba79c226aee48373cd Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 28 Aug 2026 09:36:12 +0200 Subject: [PATCH] Document non-obvious invariants across sync, check, scrub, parity, and handle mapping logic --- cmdline/check.c | 358 +++++++++++++++++++++++++++++++++++++++------ cmdline/handle.c | 32 ++-- cmdline/handle.h | 12 +- cmdline/parity.c | 151 ++++++++++++++++--- cmdline/parity.h | 40 +++-- cmdline/scrub.c | 346 ++++++++++++++++++++++++++++++++++++++++--- cmdline/selftest.c | 8 +- cmdline/sync.c | 301 +++++++++++++++++++++++++++++++++++-- 8 files changed, 1126 insertions(+), 122 deletions(-) diff --git a/cmdline/check.c b/cmdline/check.c index a4baa82a..aa52bf93 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -14,6 +14,176 @@ #include "raid/raid.h" #include "raid/combo.h" +/* + * SnapRAID check/fix model + * + * Check verifies the data described by the content file against the files + * currently present on the data disks. For blocks with a current hash, the + * data is verified directly by hashing it. Unless running in audit-only mode, + * parity is also read and can be used both to recover unreadable or corrupted + * data and to verify the stored parity itself. + * + * Fix uses the same verification and recovery logic as check, but writes + * validated recovered data and parity back to disk. Recovery is deliberately + * conservative: obtaining a solution from the RAID equations is not enough; + * the result must also be validated independently before it can be accepted. + * + * Block state model + * ----------------- + * + * Check/fix must interpret block states differently because the content file + * may describe an incomplete sync. In that case, the logical block state and + * the physical parity on disk do not necessarily describe the same point in + * time. + * + * State Stored hash Current file Physical parity + * ------------------------------------------------------------------------ + * EMPTY none none zero contribution + * BLK CURRENT present synchronized + * CHG OLD/ZERO/INVALID present may be OLD or CURRENT + * REP NEW present may be OLD or NEW + * DELETED OLD/INVALID none may be OLD or zero + * + * BLK has a hash for the data that should currently be present and normally + * represents a synchronized contribution. + * + * REP also has a hash for the current data, so its current contents can be + * verified. However, parity is still invalid because an interrupted sync may + * have left physical parity representing either the old or the new block. + * + * CHG is different: its stored hash describes the OLD data, not the current + * file contents. A readable CHG block therefore cannot be validated against + * block->hash. Its current contents are kept as the best available copy, but + * the old hash is useful when considering parity that may still represent the + * state before sync. + * + * DELETED has no current file data. Its stored hash, when available, describes + * the previous contribution that may still be present in physical parity. + * + * Interrupted sync recovery model + * ------------------------------- + * + * When CHG, REP or DELETED blocks are present, check/fix cannot know whether + * an interrupted sync updated the physical parity before stopping. Recovery + * therefore tries two different interpretations of the same parity. + * + * The first strategy assumes that parity represents the CURRENT state: + * + * BLK -> current data is represented + * CHG -> current data is represented + * REP -> new/current data is represented + * DELETED -> contributes zero + * + * Under this assumption, readable current data can be used directly and only + * blocks known to be bad or missing have to be reconstructed. BLK and REP + * recovery can be validated with their stored current hashes. CHG recovery is + * more subtle because only its OLD hash is available; a recovered CHG may + * therefore remain ambiguous and be marked out-of-date. + * + * If that strategy cannot produce a validated recovery, a second strategy + * assumes that parity still represents the state BEFORE the interrupted sync: + * + * BLK -> synchronized data is represented + * CHG -> OLD data is represented + * REP -> OLD data is represented + * DELETED -> OLD data is represented + * + * Under this assumption, the current contents of CHG and REP cannot be used + * as parity inputs even when they are perfectly readable. CHG, REP and + * DELETED must instead be treated as unknown old contributions, reconstructed + * only as needed to make recovery of a bad BLK possible. + * + * These reconstructed old contributions are auxiliary recovery state, not + * current user data. They are marked out-of-date so they cannot validate the + * recovery and so an old CHG/REP result is never promoted as known-good + * current data. + * + * A CHG whose OLD hash is ZERO is a useful special case: its old contribution + * is known exactly and can be supplied as an all-zero block without consuming + * a RAID recovery equation. Similarly, if an old CHG or DELETED block can be + * obtained from the import set using its stored OLD hash, that known old + * contribution reduces the number of unknowns. REP cannot use this shortcut + * because its stored hash describes the NEW data, not the old contribution. + * + * Recovery validation model + * ------------------------- + * + * Solving the RAID equations does not by itself prove that the reconstructed + * data is correct. One of the parity blocks used for reconstruction may itself + * be damaged, and after an interrupted sync different parity levels may not + * necessarily provide the expected state. + * + * For F unknown data blocks, recovery therefore requires independent evidence: + * + * - If at least one recovered block has a trustworthy current hash, F parity + * equations are sufficient. The candidate recovery is accepted only if all + * available trustworthy hashes of recovered blocks match. + * + * - If no recovered block has a trustworthy hash, F+1 parity blocks are + * required. F parity blocks are used to solve the data equations and the + * remaining parity block is reserved as an independent check of the result. + * + * When more parity levels are available, repair_step() tries their different + * combinations. Missing parity levels are skipped, and a candidate obtained + * from a bad parity combination is rejected when its hash or independent + * parity check fails. + * + * This distinction is important: a successful RAID decode is only a candidate + * recovery. It becomes an accepted recovery only after independent validation. + * + * Parity validation model + * ----------------------- + * + * Physical parity is checked or rewritten only when it is both meaningful and + * expected to be valid. + * + * used_parity means that at least one live data block uses the parity position. + * Parity for a position containing no live data is irrelevant and sync is not + * required to keep such unused parity updated. + * + * valid_parity means that no block at the position is CHG, REP or DELETED. + * When one of these states is present, parity may legitimately differ from + * parity recomputed from the current data because sync has not completed. + * Such a difference must not be reported or "fixed" by check/fix; the next + * sync is responsible for bringing that parity position to the current state. + * + * Parity is therefore compared and repaired only when: + * + * used_parity && valid_parity + * + * A parity level excluded from fix is not necessarily useless. When accessible + * it is still opened for reading because it can provide an additional recovery + * equation or independent validation, while remaining protected from writes. + * + * Fix and quarantine model + * ------------------------ + * + * failed_struct distinguishes two different properties of a recovery entry. + * + * is_bad means that the current file block is missing, unreadable or does not + * match its expected current hash and therefore needs recovery. + * + * is_outofdate means that the bytes currently associated with the entry cannot + * be proven to represent the required current data. This is used both for the + * auxiliary old CHG/REP/DELETED values of the pre-sync recovery strategy and + * for ambiguous recovered CHG blocks. + * + * An out-of-date candidate may still be the best data available. Fix may write + * such data, but the containing file remains marked damaged and is kept under + * the .unrecoverable name instead of being reported as successfully recovered. + * + * Missing files are created as .unrecoverable from the beginning, and an + * existing .unrecoverable file is reused by later fix runs. This allows a + * recovery to proceed over multiple runs without exposing a partially rebuilt + * file under its final name. + * + * Promotion to the final file name is a file-level commit operation. It happens + * only after the complete file has been processed successfully, its metadata + * has been restored, and no block remains damaged. A partial -S/-B fix operates + * only at block level and therefore never performs this file-level promotion + * or other finalization. + */ + /****************************************************************************/ /* check */ @@ -26,26 +196,45 @@ static const char* es(int err) } /** - * A block that failed the hash check, or that was deleted. + * A block participating in recovery. + * + * Entries include blocks that are actually missing or corrupted, but also + * readable CHG, REP and DELETED blocks whose old/current contribution may be + * needed to interpret parity left by an interrupted sync. */ struct failed_struct { /** - * If we know for sure that the block is garbage or missing - * and it needs to be recovered and rewritten to the disk. + * If the current file block is known to require recovery because it is + * missing, unreadable, or doesn't match its expected current hash. + * + * DELETED blocks and readable CHG/REP auxiliary entries are not bad even + * though they may participate in the recovery equations. */ int is_bad; /** - * If what we have recovered may be not updated data, - * an old version, or just garbage. + * If the data associated with this entry cannot be proven to represent the + * required CURRENT contents. * - * Essentially, it means that we are not sure what we have recovered - * is really correct. It's just our best guess. + * This does not mean that the data is known to be wrong or actually old. + * In particular, for a CHG block the stored hash describes the OLD contents, + * but the block itself may not have changed even if the file was detected as + * modified. Therefore data matching the OLD state may also be perfectly valid + * CURRENT data. * - * These "recovered" blocks are also written to the disk if the block is marked as ::is_bad. - * But these files are marked also as FILE_IS_DAMAGED, and then renamed to .unrecoverable. + * In the pre-sync parity strategy this is set on CHG, REP and DELETED entries + * because their OLD contribution is being used or reconstructed as auxiliary + * recovery state. Such data cannot be assumed to represent the required + * CURRENT contents, even though in some cases OLD and CURRENT may actually be + * identical. * - * Note that this could happen only for CHG blocks. + * It is also set on a bad CHG when the current-parity strategy recovers a + * candidate that cannot be distinguished reliably from its OLD contents. + * + * Out-of-date entries are excluded from hash validation. If an out-of-date + * block is also is_bad, fix may still preserve the recovered bytes as the best + * available copy, but the containing file remains damaged and quarantined as + * .unrecoverable. */ int is_outofdate; @@ -77,7 +266,12 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl return -1; } - /* compare to the end of the block */ + /* + * The hash covers only the logical bytes belonging to the file, while RAID + * operates on the complete zero-padded block. Verify the padding separately: + * otherwise corruption confined to the reconstructed tail could pass the hash + * check and later produce incorrect parity. + */ if (pos_size < state->block_size) { if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) { return -1; @@ -153,9 +347,15 @@ static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, un } /** - * Repair errors. - * Return <0 if failure for missing strategy, >0 if data is wrong and we cannot rebuild correctly, 0 on success. - * If success, the parity is computed in the buffer variable. + * Try to recover the selected failed inputs and independently validate the + * reconstructed result. + * + * Return: + * 0 if a recovery was successfully validated; + * >0 if one or more recovery candidates were tried but all failed validation; + * <0 if no usable recovery/validation strategy exists. + * + * On success all parity levels are recomputed in buffer. */ static int repair_step(struct snapraid_state* state, int rehash, block_off_t pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero) { @@ -189,8 +389,12 @@ static int repair_step(struct snapraid_state* state, int rehash, block_off_t pos id[i] = failed[failed_map[i]].index; /* - * Check if there is at least a failed block that can be checked for correctness using the hash - * if there isn't, we have to sacrifice a parity block to check that the result is correct + * RAID decoding alone cannot tell whether the parity equations used for the + * recovery were themselves correct. + * + * If at least one failed input has a trustworthy CURRENT hash, recovered data + * can provide the independent validation. Otherwise one additional parity + * equation must be reserved exclusively for checking the candidate result. */ has_hash = 0; for (i = 0; i < failed_count; ++i) { @@ -385,7 +589,13 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns assert(block_state != BLOCK_STATE_DELETED); /* we cannot have bad DELETED blocks */ - /* if we have the hash for it */ + /* + * Import/search can directly repair only states whose stored hash identifies + * the desired CURRENT contents. This is true for BLK and REP. + * + * A CHG hash identifies the OLD contents, so fetching data by that hash would + * not prove that we recovered the current version required by this strategy. + */ if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP) /* try to fetch the block using the known hash */ && (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0 @@ -511,12 +721,15 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns || block_state == BLOCK_STATE_REP ) { /* - * If the block is CHG, REP or DELETED, we don't have the original content of block, - * and we must try to recover it. - * This applies to CHG and REP blocks even if they are not marked bad, - * because the parity is computed with old content, and not with the new one. - * Note that this recovering is done just to make it possible to recover any other BLK one, - * we are not really interested in DELETED, CHG (old version) and REP (old version). + * Under the pre-sync hypothesis the readable buffer of a CHG or REP contains + * CURRENT data, while physical parity was generated from its OLD data. + * + * Therefore every CHG, REP and DELETED contribution must be replaced by its + * OLD value or treated as an unknown RAID input, even when the current file + * block is perfectly readable and is_bad is false. + * + * These old values are reconstructed only to make recovery of bad BLK blocks + * possible; recovering them is not itself a user-visible recovery goal. */ something_unsynced = 1; @@ -534,7 +747,14 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns * we can do this only because it's the last retry of recovering */ - /* try to fetch the old block using the old hash for CHG and DELETED blocks */ + /* + * CHG and DELETED retain an OLD hash, so a matching imported block can supply + * their exact pre-sync contribution and remove one unknown from the RAID + * equations. + * + * REP cannot use this shortcut: its stored hash describes NEW data and gives + * no direct way to identify the OLD contribution required by this strategy. + */ } else if ((block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_DELETED) && hash_is_unique(failed[j].block->hash) && state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0) { @@ -580,8 +800,14 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns } /* - * If nothing to fix, we just don't try - * if nothing unsynced we also don't retry, because it's the same try as before + * Retry only when both conditions hold. + * + * Without a bad BLK there is no useful current data to recover: reconstructing + * only OLD CHG/REP/DELETED values would have no user-visible purpose. + * + * Without any unsynced state this hypothesis is identical to the current-state + * attempt already performed above, so repeating it cannot provide a different + * solution. */ if (something_to_recover && something_unsynced) { ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero); @@ -1146,10 +1372,11 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna memset(buffer[j], 0, state->block_size); /* - * Store it in the failed set, because potentially - * the parity may be still computed with the previous content + * DELETED is not a file repair target. It is kept only because its OLD + * contribution may still be present in parity and may be required by the + * pre-sync recovery hypothesis. */ - failed[failed_count].is_bad = 0; /* note that is_bad==0 <=> file==0 */ + failed[failed_count].is_bad = 0; failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; @@ -1270,7 +1497,14 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } } - /* if it's the first open, and not excluded and larger */ + /* + * Missing data inside a short file is handled block by block by normal + * recovery writes, which extend the file as needed. + * + * Extra data past the expected end is different: it has no content block and + * therefore no later block recovery can remove it. Fix must truncate that + * untracked tail explicitly. + */ if (!file_flag_has(file, FILE_IS_OPENED) && !file_flag_has(file, FILE_IS_EXCLUDED) && !(state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED)) @@ -1406,9 +1640,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } /* - * Always insert REP blocks, the repair functions needs all of them - * because the parity may be still referring at the old state - * and the repair must be aware of it + * A readable REP has already passed its NEW/current hash check, so its current + * data is known good. It must nevertheless participate in repair because an + * interrupted sync may have left physical parity containing its OLD + * contribution. + * + * Unlike CHG, the REP stored hash cannot identify that OLD contribution + * because it describes the NEW data. */ if (block_state == BLOCK_STATE_REP) { failed[failed_count].is_bad = 0; /* it's not bad */ @@ -1445,7 +1683,15 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna if (ret == -1) { log_tag("parity_%s:%" PRIu64 ":%s: Read error. %s.\n", es(errno), i, lev_config_name(l), strerror(errno)); - buffer_recov[l] = 0; /* no parity to use */ + /* + * NULL means that this parity level cannot be trusted as an input to recovery. + * The same marker is later used for a parity block that was read successfully + * but failed comparison with recomputed parity. + * + * During fix, an accessible non-excluded parity marked this way is also the + * one that needs to be rewritten from the recomputed value. + */ + buffer_recov[l] = 0; if (is_hw(errno)) { ++io_error; @@ -1479,9 +1725,12 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } } else { /* - * Now counts partial recovers - * note that this could happen only when we have an incomplete 'sync' - * and that we have recovered is the state before the 'sync' + * repair() returning success means that a RAID solution was validated under + * one of the parity-history hypotheses. It does not necessarily mean that + * every bad unsynced block was recovered to its CURRENT version. + * + * A bad block marked is_outofdate is therefore still an unrecoverable current + * data error even though the stripe-level RAID recovery succeeded. */ int partial_recover_error = 0; @@ -1534,6 +1783,14 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna || (state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED))) continue; + /* + * Write the best recovered candidate even when it is out-of-date. Such data + * may still be useful to the user or to a later multistep recovery. + * + * An out-of-date write is not considered a successful fix: the file is marked + * damaged below and file_post() keeps or moves it to .unrecoverable instead + * of promoting it as recovered. + */ ret = handle_write(failed[j].handle, failed[j].file_pos, buffer[failed[j].index], state->block_size); if (ret == -1) { /* LCOV_EXCL_START */ @@ -1664,6 +1921,11 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } } + /* + * These objects have no parity block position, so an explicit -S/-B block + * range cannot select them. Check/fix them separately after block processing, + * including during an otherwise partial block-range operation. + */ /* for each disk, recover empty files, symlinks and empty dirs */ for (i = 0; i < diskmax; ++i) { tommy_node* node; @@ -2197,8 +2459,16 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b msg_progress("Initializing...\n"); /* - * partial follows the explicit -S/-B request, not its clipped effective - * coverage; a nonzero -B remains partial even if it reaches the parity end. + * -S/--start and -B/--count allow check/fix to operate on an arbitrary + * parity range instead of processing from the beginning. In particular, + * recovery of a later section can be retried while an earlier section is + * missing or damaged. + * + * A partial fix therefore does not establish that preceding or following + * parity is valid; it covers only the requested range. + * + * Keep partial based on the explicit -S/-B request rather than the clipped + * range: a nonzero -B remains partial even if it reaches the end of parity. */ partial = blockstart != 0 || blockcount != 0; @@ -2233,7 +2503,13 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b /* if the parity is excluded */ if (state->parity[l].is_excluded_by_filter) { - /* open for reading, and ignore error */ + /* + * Excluded means "do not modify", not "do not use". + * + * An excluded parity can still provide a recovery equation or the independent + * parity check required when recovered data has no trustworthy hash, so keep + * it available read-only whenever possible. + */ ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); if (ret == -1) { log_tag("parity_%s:%" PRIu64 ":%s: Open error. %s.\n", es(errno), blockmax, lev_config_name(l), strerror(errno)); @@ -2309,7 +2585,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b /* try to close only if opened */ for (l = 0; l < state->level; ++l) { if (parity_ptr[l]) { - /* if fixing and not excluded, truncate parity not valid */ + /* if fixing and not excluded, truncate parity to physical_reach_size */ if (fix && !state->parity[l].is_excluded_by_filter) { ret = parity_truncate(parity_ptr[l]); if (ret == -1) { diff --git a/cmdline/handle.c b/cmdline/handle.c index 06f2ebab..ff5fdfda 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -123,7 +123,7 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file, in } /* get the size of the existing data */ - handle->valid_size = handle->st.st_size; + handle->physical_reach_size = handle->st.st_size; ret = advise_open(&handle->advise, handle->f); if (ret != 0) { @@ -164,7 +164,7 @@ int handle_truncate(struct snapraid_handle* handle, struct snapraid_file* file) } /* adjust the size to the truncated size */ - handle->valid_size = file->size; + handle->physical_reach_size = file->size; return 0; } @@ -224,7 +224,7 @@ int handle_open(struct snapraid_handle* handle, struct snapraid_file* file, int } /* get the size of the existing data */ - handle->valid_size = handle->st.st_size; + handle->physical_reach_size = handle->st.st_size; ret = advise_open(&handle->advise, handle->f); if (ret != 0) { @@ -254,7 +254,7 @@ int handle_close(struct snapraid_handle* handle) /* invalidate for error */ handle->file = 0; handle->f = -1; - handle->valid_size = 0; + handle->physical_reach_size = 0; handle->is_unrecoverable = 0; return -1; /* LCOV_EXCL_STOP */ @@ -264,7 +264,7 @@ int handle_close(struct snapraid_handle* handle) /* reset the descriptor */ handle->file = 0; handle->f = -1; - handle->valid_size = 0; + handle->physical_reach_size = 0; handle->is_unrecoverable = 0; handle->readonly_errno = 0; @@ -285,9 +285,9 @@ ssize_t handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsign out_missing = log_error; /* check if we are going to read only not initialized data */ - if (offset >= handle->valid_size) { + if (offset >= handle->physical_reach_size) { /* if the file is missing, it's at 0 size, or it's rebuilt while reading */ - if (offset == handle->valid_size || handle->valid_size == 0) { + if (offset == handle->physical_reach_size || handle->physical_reach_size == 0) { errno = ENOENT; if (offset == 0) { out_missing(errno, "Missing file '%s'.\n", handle->path); @@ -389,9 +389,9 @@ int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned count += write_ret; } while (count < write_size); - /* adjust the size of the valid data */ - if (handle->valid_size < offset + write_size) { - handle->valid_size = offset + write_size; + /* adjust the size of the written/retained data */ + if (handle->physical_reach_size < offset + write_size) { + handle->physical_reach_size = offset + write_size; } ret = advise_write(&handle->advise, handle->f, offset, block_size); @@ -432,6 +432,16 @@ struct snapraid_handle* handle_mapping(struct snapraid_state* state, unsigned* h unsigned size = 0; struct snapraid_handle* handle; + /* + * The handle array is indexed by persistent parity mapping position, + * not by data disk count or enumeration order. + * + * Mapping holes must remain as empty handles: they contribute zero data + * to RAID while preserving the parity column of following disks. + * + * Do not compact this array: renumbering a disk changes its RAID column + * and invalidates existing parity against the content file mapping. + */ /* get the size of the mapping */ size = 0; for (i = state->maplist; i != 0; i = i->next) { @@ -448,7 +458,7 @@ struct snapraid_handle* handle_mapping(struct snapraid_state* state, unsigned* h handle[j].disk = 0; handle[j].file = 0; handle[j].f = -1; - handle[j].valid_size = 0; + handle[j].physical_reach_size = 0; handle[j].is_unrecoverable = 0; handle[j].readonly_errno = 0; handle[j].bw = 0; diff --git a/cmdline/handle.h b/cmdline/handle.h index 9ea4dbbc..cc35ee3b 100644 --- a/cmdline/handle.h +++ b/cmdline/handle.h @@ -18,7 +18,17 @@ struct snapraid_handle { int f; /**< Handle of the file. */ struct stat st; /**< Stat info of the opened file. */ struct advise_struct advise; /**< Advise information. */ - data_off_t valid_size; /**< Size of the valid data. */ + + /** + * High-water mark size of the written/retained data. + * + * High-water size does not indicate that data below this offset is valid; + * it only marks the physical boundary beyond which data is known to be invalid + * and can be discarded upon truncation. Data within the physical_reach_size may + * or may not be valid. Final verification is determined independently by + * block states and hashes. + */ + data_off_t physical_reach_size; int created; /**< If the file was created, otherwise it was already existing. */ int is_unrecoverable; /**< If the open descriptor refers to the .unrecoverable path. */ int readonly_errno; /**< Non-zero if opened read-only as fallback. */ diff --git a/cmdline/parity.c b/cmdline/parity.c index ac2ad08c..20fe844f 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -127,17 +127,29 @@ void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size) *out_size = size; } -void parity_valid_size(struct snapraid_parity_handle* handle, data_off_t* out_size) +void parity_physical_reach_size(struct snapraid_parity_handle* handle, data_off_t* out_size) { unsigned s; data_off_t size; - /* compute the contiguous valid prefix of the logical parity layout */ + /* + * Compute the physical reach of the logical parity layout across splits. + * + * physical_reach_size represents physical file presence, not parity validity. + * Parity is not necessarily contiguous or valid within this range: blocks + * may be invalid, stale, unwritten, or modified. Correctness is determined + * per-block by the content state and validated independently via data hashes + * and recomputation. + * + * For split parity, splits concatenate linearly. If an earlier split is + * physically shorter than its logical size, logical offsets beyond that + * missing region cannot be addressed, so subsequent splits are not counted. + */ size = 0; for (s = 0; s < handle->split_mac; ++s) { struct snapraid_split_handle* split = &handle->split_map[s]; - data_off_t run = split->valid_size; + data_off_t run = split->physical_reach_size; /* don't count physical data outside the logical split */ if (run > split->size) @@ -145,7 +157,7 @@ void parity_valid_size(struct snapraid_parity_handle* handle, data_off_t* out_si size += run; - /* later splits cannot compensate for a hole in this one */ + /* later splits cannot extend the logical extent past this boundary */ if (run < split->size) break; } @@ -196,8 +208,25 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p /* LCOV_EXCL_STOP */ } - /* the initial valid size is the size on disk */ - split->valid_size = split->st.st_size; + /* + * Initialize the physical read/truncation boundary from the current EOF. + * + * physical_reach_size does NOT imply that parity below this offset is valid. + * Parity may be invalid, stale, unwritten, or sparse in any region below + * this boundary. It only records the physical EOF boundary for I/O and + * truncation. Parity validity is tracked independently per block through + * the content state, data hashes, and recomputation. + * + * While a handle is open, physical_reach_size can intentionally differ from EOF. + * In particular, preallocation may extend EOF without extending physical_reach_size. + * Writable fix paths normally truncate the file back to physical_reach_size before + * closing. + * + * Across close/reopen there is no separate persistent physical_reach value, so EOF + * is used to reconstruct this physical boundary. This reconstruction must not + * be interpreted as validation of the parity contained below EOF. + */ + split->physical_reach_size = split->st.st_size; /** * If the parity size is not yet set, set it now. @@ -515,15 +544,42 @@ static int parity_handle_chsize(struct snapraid_split_handle* split, data_off_t /* LCOV_EXCL_STOP */ } - /* if we shrink, update the valid size, but don't update when growing */ - if (split->valid_size > split->st.st_size) - split->valid_size = split->st.st_size; + /* + * Keep the physical read/truncation boundary consistent with resizing. + * + * Shrinking reduces physical_reach_size because bytes beyond the new EOF no longer + * exist physically and therefore cannot be read. + * + * Growing leaves physical_reach_size unchanged. Allocation or preallocation alone + * does not make the newly created physical extent readable as parity; the + * boundary is advanced only after a parity write has completed. + * + * This must not be confused with parity validity. Advancing physical_reach_size + * after a completed write only records the physical offset reached by writes. + * It does not certify preceding bytes as correct parity; parity may remain + * invalid or unwritten in any region below this boundary. + */ + if (split->physical_reach_size > split->st.st_size) + split->physical_reach_size = split->st.st_size; return 0; } static int parity_split_is_fixed(struct snapraid_parity_handle* handle, unsigned s) { + /* + * Parity splits concatenate into a single logical file. + * + * Once a following split contains data, this split's boundary becomes + * fixed: changing it would shift logical offsets in all subsequent + * splits, invalidating their parity positions. + * + * Only the last active split may grow. If total parity shrinks into a + * previously fixed split, that split becomes the new tail and may shrink. + * + * An elastic split may also grow partially; remaining space is allocated + * in the next split. + */ /* next one */ ++s; @@ -697,8 +753,25 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par /* LCOV_EXCL_STOP */ } - /* the initial valid size is the size on disk */ - split->valid_size = split->st.st_size; + /* + * Initialize the physical read/truncation boundary from the current EOF. + * + * physical_reach_size does NOT imply that parity below this offset is valid. + * Parity may be invalid, stale, unwritten, or sparse in any region below + * this boundary. It only records the physical EOF boundary for I/O and + * truncation. Parity validity is tracked independently per block through + * the content state, data hashes, and recomputation. + * + * While a handle is open, physical_reach_size can intentionally differ from EOF. + * In particular, preallocation may extend EOF without extending physical_reach_size. + * Writable fix paths normally truncate the file back to physical_reach_size before + * closing. + * + * Across close/reopen there is no separate persistent physical_reach value, so EOF + * is used to reconstruct this physical boundary. This reconstruction must not + * be interpreted as validation of the parity contained below EOF. + */ + split->physical_reach_size = split->st.st_size; /** * If the parity size is not yet set, set it now. @@ -766,6 +839,16 @@ int parity_sync(struct snapraid_parity_handle* handle) return 0; } +/* + * Truncate each split to its current physical_reach_size boundary. + * + * This removes preallocated or otherwise disposable data beyond physical_reach_size. + * It does not imply that parity blocks retained below physical_reach_size are + * valid; parity validity is tracked independently per block by the content state. + * + * Persisting this boundary in physical EOF allows the next open to restore + * physical_reach_size from st_size. + */ int parity_truncate(struct snapraid_parity_handle* handle) { unsigned s; @@ -775,11 +858,18 @@ int parity_truncate(struct snapraid_parity_handle* handle) struct snapraid_split_handle* split = &handle->split_map[s]; int ret; - /* truncate any data that we know it's not valid */ - ret = ftruncate(split->f, split->valid_size); + /* + * Discard physical space beyond the extent reached by parity writes + * during this operation. + * + * The retained region below physical_reach_size is not thereby declared valid + * parity. physical_reach_size is only a physical truncation boundary; parity + * validity is tracked independently per block in the content state. + */ + ret = ftruncate(split->f, split->physical_reach_size); if (ret != 0) { /* LCOV_EXCL_START */ - log_fatal(errno, "Error truncating the parity file '%s' to size %" PRIu64 ". %s.\n", split->path, split->valid_size, strerror(errno)); + log_fatal(errno, "Error truncating the parity file '%s' to size %" PRIu64 ". %s.\n", split->path, split->physical_reach_size, strerror(errno)); f_ret = -1; /* LCOV_EXCL_STOP */ @@ -861,6 +951,11 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne /* LCOV_EXCL_STOP */ } + /* + * Parity blocks cannot cross split boundaries: offsets and split sizes + * are block-aligned, so the entire block belongs to this split. + */ + bw_limit(handle->bw, block_size); count = 0; @@ -890,9 +985,17 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne count += write_ret; } while (count < block_size); - /* update the valid range */ - if (split->valid_size < offset + block_size) - split->valid_size = offset + block_size; + /* + * Extend the physical read/truncation limit only after the complete block + * has been written. + * + * This is not a validity marker. In particular, writes may occur non-contiguously, + * so bytes between the old boundary and this block are not implied to contain + * valid parity. Parity correctness is tracked independently per block in the + * content state. + */ + if (split->physical_reach_size < offset + block_size) + split->physical_reach_size = offset + block_size; ret = advise_write(&split->advise, split->f, offset, block_size); if (ret != 0) { @@ -924,8 +1027,18 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned /* LCOV_EXCL_STOP */ } - /* if read is completely out of the valid range */ - if (offset >= split->valid_size) { + /* + * Parity blocks cannot cross split boundaries: offsets and split sizes + * are block-aligned, so the entire block belongs to this split. + */ + + /* + * physical_reach_size only limits the physical offset that this handle considers + * readable. Passing this check does not mean that the parity bytes are valid. + * A readable region may contain stale, sparse, unwritten, or corrupted parity, + * which higher-level check/recovery logic must validate independently. + */ + if (offset >= split->physical_reach_size) { /* LCOV_EXCL_START */ errno = ENXIO; log_error(errno, "Reading over the end from parity file '%s' at offset %" PRIu64 " for size %u.\n", split->path, offset, block_size); diff --git a/cmdline/parity.h b/cmdline/parity.h index 77feb161..439ab9e5 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -26,14 +26,24 @@ struct snapraid_split_handle { data_off_t size; /** - * Valid size of the parity split. - * This is the size effectively written, and not the result of a chsize operation. - * It's used to make read operations failing if read over that size. + * High-water mark size of the parity split. * - * Parity is also truncated to that size when fixing it, in case of a Break (Ctrl+C) - * of the program. + * This is the physical truncation and read limit of the split. + * + * physical_reach_size does NOT indicate that data below this offset is valid parity. + * Parity may be invalid, stale, unwritten, or sparse at any point below this + * boundary. Correctness is tracked independently per block in the content state + * and verified through data hashes and recomputation. + * + * Data beyond physical_reach_size is disposable/preallocated and discarded by + * parity_truncate(). A completed write advances physical_reach_size to the end of + * the written block so it is not truncated away. + * + * Before closing writable parity handles during fix operations, parity_truncate() + * persists this boundary into the physical EOF. On the next open, physical_reach_size can + * therefore be initialized from st_size. */ - data_off_t valid_size; + data_off_t physical_reach_size; /** * Artificial size limit for testing. @@ -95,13 +105,19 @@ int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size); /** - * Get the size of the contiguous valid prefix of the parity. + * Get the physical reach of the logical parity file across splits. * - * This returns the logical prefix covered by valid data in all consecutive splits. - * For example, with two 100 GiB splits, if the first is truncated to 90 GiB - * and the second is fully valid, the valid prefix is 90 GiB, not 190 GiB. + * This composes the physical_reach_size boundaries of consecutive splits. + * If the physical_reach_size of an earlier split is smaller than its logical size, + * later splits cannot extend the logical offset past that missing region. + * + * Parity within this range is not necessarily valid or synchronized; + * per-block validity is tracked independently in the content state. + * + * For example, with two 100 GiB splits, if the first has physical_reach_size 90 GiB + * and the second has physical_reach_size 100 GiB, the result is 90 GiB, not 190 GiB. */ -void parity_valid_size(struct snapraid_parity_handle* handle, data_off_t* out_size); +void parity_physical_reach_size(struct snapraid_parity_handle* handle, data_off_t* out_size); /** * Open an already existing parity file. @@ -114,7 +130,7 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par int parity_sync(struct snapraid_parity_handle* handle); /** - * Truncate the parity file to the valid size. + * Truncate each split to its physical physical_reach_size boundary. */ int parity_truncate(struct snapraid_parity_handle* handle); diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 0e54b7c2..10fe1bc1 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -11,6 +11,198 @@ #include "io.h" #include "raid/raid.h" +/* + * SnapRAID scrub model + * + * Scrub periodically verifies data and parity without modifying either of + * them. It reads selected parity positions, checks the data blocks against + * their stored hashes when a current hash is available, recomputes all parity + * levels from the data read, and compares the result with physical parity. + * + * The unit of scrub processing is a parity position, not an individual file + * block. Selecting one position means considering all mapped data blocks and + * all parity levels belonging to the same stripe. + * + * Scrub does modify the content state. For each processed position it records + * when the position was last verified, maintains the persistent bad flag, and + * can progressively migrate block hashes while a rehash is in progress. + * + * Verification model + * ------------------ + * + * The block states have different meanings for scrub: + * + * State File Stored hash Data verification Parity + * ------------------------------------------------------------------------ + * EMPTY no none zero input expected + * BLK yes CURRENT hash checked expected + * CHG yes OLD not hash checked unsynced + * REP yes NEW hash checked unsynced + * DELETED no OLD/INVALID zero input unsynced + * + * BLK and REP have a hash describing their current file data and can therefore + * be checked directly. + * + * CHG is deliberately different. Its stored hash describes the OLD data, so + * comparing the current file contents against block->hash would not validate + * the current block. Scrub still reads the current data because it is needed + * to recompute parity, but does not use the CHG hash as a current-data check. + * + * EMPTY and DELETED have no current file contribution and are supplied to the + * RAID computation as zero. DELETED still makes the position unsynced because + * physical parity may contain its previous contribution. + * + * A position is also considered unsynced when a file's current size or + * modification time differs from the metadata recorded in the content state. + * This does not by itself abort scrub. It changes how a later data or parity + * mismatch is classified. + * + * Parity is recomputed only when every required data input was read without an + * error and every applicable current hash matched. Without a complete and + * trustworthy set of data inputs, a parity comparison would not identify + * whether an error belongs to data or parity. + * + * Error classification model + * -------------------------- + * + * A hash or parity mismatch is a silent data error only when the corresponding + * data and parity position are expected to be synchronized. + * + * If the file metadata differs from the content state, or any block at the + * position is CHG, REP or DELETED, a mismatch can be the normal consequence + * of an unsynchronized array. It is therefore reported as a soft error rather + * than as silent corruption. + * + * The unsynced classification is conservative and affects only the meaning of + * a mismatch. It does not make the position fail automatically. If the data + * that can be checked is correct and physical parity already agrees with the + * current logical inputs, the scrub of that position can still complete. + * + * I/O errors are tracked separately from logical or hash mismatches. A + * recoverable per-position I/O failure marks the position bad; errors severe + * enough to abort processing may terminate the command before normal + * per-position bookkeeping is completed. + * + * Scrub plan model + * ---------------- + * + * Scrub selection is based on the per-position snapraid_info stored in the + * content file. An info entry records: + * + * - the last time the position was known to be correct, + * - whether the position is marked bad, + * - whether its hashes still use the previous hash algorithm, + * - whether it was just synchronized and has never been scrubbed. + * + * Positions without an info entry are not scrub candidates. + * + * Bad positions override every scrub plan and are always selected. This makes + * the persistent bad flag self-checking: a later scrub rechecks the complete + * position regardless of its age or the requested normal plan. + * + * The explicit plans otherwise select positions as follows: + * + * full all tracked positions + * even even parity positions + * new positions never scrubbed since synchronization + * bad only positions marked bad + * + * The automatic plan selects the oldest positions first. The bucket list + * groups positions by their last-known-good timestamp and is sorted from the + * oldest timestamp to the newest. + * + * Selection is limited by both the requested amount and the optional age + * limit. When the requested count ends inside a timestamp bucket, timelimit + * identifies that bucket and lastlimit specifies how many positions from that + * exact timestamp may be selected. block_is_enabled() is called in increasing + * parity-position order, providing a deterministic tie break inside the + * bucket. + * + * Bad positions do not consume this boundary quota: they are selected before + * normal plan filtering and may therefore make the actual scrub count larger + * than the requested automatic amount. + * + * The complete block selection is materialized before I/O starts. Autosaves + * performed during the run may rebuild the scrub-time bucket list in the + * content state, but they do not change the set of positions selected for the + * current scrub. + * + * Bad state and persistence model + * ------------------------------- + * + * The bad flag belongs to the whole parity position. It does not identify a + * specific data disk or parity level. A silent data/parity error or a + * continuable I/O error at any contribution makes the complete position bad, + * because scrub cannot safely reduce that observation to a single trusted + * component. + * + * When a position fails with a silent or I/O error, info_set_bad() preserves + * its previous timestamp and all other flags. In particular, a failed scrub + * does not make the position appear recently verified and does not discard + * pending rehash or just-synchronized state. + * + * A generic soft error also leaves the previous info unchanged. Such a + * position has not completed a successful scrub and therefore retains its + * previous scrub age. + * + * Only a completely successful position is replaced with: + * + * info_make(now, 0, 0, 0) + * + * This records the new last-known-good time and clears bad, rehash and + * just-synchronized state. Consequently, "scrub -p bad" clears a bad position + * only after the whole position can again be verified successfully. + * + * Scrub writes only content metadata and hashes; it never writes user data or + * parity. Autosave can therefore persist completed scrub positions + * independently. A crash before the next content save may lose some scrub + * progress, causing those positions to be checked again, but cannot leave a + * partially repaired data/parity state because scrub performs no such repair. + * + * Rehash model + * ------------ + * + * Rehash state is also tracked per parity position. While rehash is pending, + * hashes that can be validated are checked with the previous hash algorithm, + * while hashes using the new algorithm are computed into temporary storage. + * + * The new hashes are not committed independently for each disk. They are + * staged until the complete parity position has passed data reads, applicable + * hash checks, and parity verification. Only then are the staged hashes copied + * into the block state and the per-position rehash flag cleared. + * + * This keeps the hash generation and its info flag consistent across content + * saves: a persisted rehash flag means the previous algorithm must still be + * used for that position, while clearing the flag publishes the newly computed + * hashes at the same successful scrub checkpoint. + * + * A CHG block has no current hash to validate. If such a position passes scrub, + * physical parity has nevertheless been verified against the bytes just read. + * The newly computed hash can then describe that verified parity baseline + * using the new algorithm, while the block itself remains CHG and is still + * conservatively treated as unsynchronized. + * + * Live filesystem model + * --------------------- + * + * Scrub does not freeze the data filesystem. Files can be modified while the + * command is running. + * + * A size or modification-time difference observed when opening a file marks + * that file and parity position as unsynced for error classification. Scrub + * continues reading so that it can still report the state actually observed. + * + * If the changed file later produces a hash or parity mismatch, the mismatch + * is treated as a soft synchronization error rather than silent corruption. + * If its bytes still agree with the stored hash and parity, scrub may complete + * successfully even though the metadata difference remains for a later sync + * to process. + * + * As with other metadata-based operations, a concurrent modification that + * preserves all metadata used for change detection cannot be distinguished + * reliably from silent corruption by metadata checks alone. + */ + /****************************************************************************/ /* scrub */ @@ -23,7 +215,11 @@ static const char* es(int err) } /** - * Buffer for storing the new hashes. + * Staged hashes for a rehash operation. + * + * New hashes are kept here until the complete parity position has passed + * verification. They are copied into the block state only together with + * clearing the per-position rehash flag. */ struct snapraid_rehash { unsigned char hash[HASH_MAX]; @@ -32,12 +228,16 @@ struct snapraid_rehash { /** * Scrub plan to use. + * + * AUTO selects complete buckets older than timelimit and, when the requested + * count cuts through the boundary bucket, at most lastlimit positions whose + * timestamp is exactly timelimit. */ struct snapraid_plan { struct snapraid_state* state; int plan; /**< One of the SCRUB_*. */ time_t timelimit; /**< Time limit. Valid only with SCRUB_AUTO. */ - block_off_t lastlimit; /**< Number of blocks allowed with time exactly at ::timelimit. Valid only with SCRUB_AUTO. */ + block_off_t lastlimit; /**< Maximum number of normal AUTO selections whose timestamp is exactly timelimit. */ }; /** @@ -53,7 +253,13 @@ static int block_is_enabled(struct snapraid_plan* plan, block_off_t* countlast, if (info == 0) return 0; - /* bad blocks are always scrubbed in all plans */ + /* + * Bad is an override, not a normal plan criterion. + * + * Always recheck bad positions so a successful scrub can clear the persistent + * bad flag even if the position would normally be excluded by age, parity, + * NEW, or any other plan. + */ if (info_get_bad(info)) return 1; @@ -65,7 +271,7 @@ static int block_is_enabled(struct snapraid_plan* plan, block_off_t* countlast, /* in 'even' plan, scrub only even blocks */ return i % 2 == 0; case SCRUB_NEW : - /* in 'sync' plan, only blocks never scrubbed */ + /* in 'new' plan, only positions never scrubbed after sync */ return info_get_justsynced(info); case SCRUB_BAD : /* in 'bad' plan, only bad blocks (already reported) */ @@ -80,8 +286,12 @@ static int block_is_enabled(struct snapraid_plan* plan, block_off_t* countlast, } /* - * If the time is less than the limit, always include - * otherwise, check if we reached the last limit count + * AUTO buckets have timestamp granularity, so the count limit may end inside + * a bucket shared by many positions. + * + * All positions older than timelimit are selected. At timelimit select only + * lastlimit normal positions; block_is_enabled() is called in ascending parity + * order, which provides the deterministic tie break. */ if (blocktime == plan->timelimit) { /* if we reached the count limit */ @@ -281,8 +491,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par silent_error = 0; io_error = 0; - msg_progress("Selecting...\n"); - + /* + * Materialize the complete selection before starting I/O. + * + * Autosave may later rebuild state->bucketlist as scrub timestamps change, but + * the current run must continue using the plan computed from the state that + * existed at startup. + */ /* first count the number of blocks to process */ countmax = 0; countlast = 0; @@ -295,9 +510,11 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } /* - * Compute the autosave size for all disk, even if not read - * this makes sense because the speed should be almost the same - * if the disks are read in parallel + * Convert the byte autosave interval to parity positions. + * + * A selected position normally reads all data columns in parallel, so use the + * aggregate data width rather than the actual serialized number of bytes read. + * Autosave only persists scrub metadata/hashes; data and parity are read-only. */ autosavelimit = state->autosave / (diskmax * state->block_size); autosavemissing = countmax; /* blocks to do */ @@ -403,7 +620,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par state_usage_file(state, disk, file); - /* if the block is unsynced, errors are expected */ + /* + * Do this before skipping blocks without a current file. + * + * DELETED has no file contribution but still has invalid parity: physical + * parity may contain its OLD data. It must therefore make the whole position + * unsynced so a later parity mismatch is not reported as silent corruption. + */ if (block_has_invalid_parity(block)) { /* report that the block and the file are not synced */ block_is_unsynced = 1; @@ -415,7 +638,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par if (!block_has_file(block)) continue; - /* if the block is unsynced, errors are expected */ + /* + * Metadata mismatch does not itself make this scrub position bad. + * + * It only tells us that the current file is not the version described by the + * content metadata. Any later data/parity mismatch is therefore an expected + * synchronization error, not evidence of silent corruption. + */ if (task->is_timestamp_different) { /* report that the block and the file are not synced */ block_is_unsynced = 1; @@ -464,7 +693,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par countsize += read_size; - /* now compute the hash */ + /* + * During rehash, validate the existing content state with the previous hash + * algorithm and stage the digest for the new algorithm separately. + * + * Do not publish the new digest yet: another data block or parity level at the + * same position may still fail verification. + */ if (rehash) { memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size); @@ -478,12 +713,26 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par /* until now is hash */ state_usage_hash(state); + /* + * Only BLK and REP contain a hash for the current file data. + * + * CHG deliberately skips this comparison because block->hash describes its + * OLD parity contribution. A CHG can still participate in the parity check + * using the current bytes read from disk. + */ if (block_has_updated_hash(block)) { /* compare the hash */ if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) { unsigned diff = memdiff(hash, block->hash, BLOCK_HASH_SIZE); - /* it's a silent error only if we are dealing with synced files */ + /* + * A mismatch is evidence of silent corruption only when this file is expected + * to match the synchronized content state. + * + * For an already unsynced file the difference may simply be the legitimate + * file modification that requires the next sync, so report it as soft and do + * not mark the parity position bad. + */ if (file_is_unsynced) { log_tag("error:%" PRIu64 ":%s:%s: Data error at position %" PRIu64 ", diff hash bits %u/%zu\n", blockcur, disk->name, esc_tag(file->sub), file_pos, diff, BLOCK_HASH_SIZE * 8); ++soft_error; @@ -564,7 +813,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } } - /* if we have read all the data required and it's correct, proceed with the parity check */ + /* + * Recomputed parity is meaningful only with a complete trusted set of current + * data inputs. + * + * If any data read or applicable hash check failed, skip parity comparison: + * a mismatch could be caused entirely by the missing/bad data input and would + * not identify a parity error. + */ if (!error_on_this_block && !silent_error_on_this_block && !io_error_on_this_block) { /* compute the parity */ @@ -575,7 +831,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par if (buffer_recov[l] && memcmp(buffer[diskmax + l], buffer_recov[l], state->block_size) != 0) { unsigned diff = memdiff(buffer[diskmax + l], buffer_recov[l], state->block_size); - /* it's a silent error only if we are dealing with synced blocks */ + /* + * Invalid block states or changed file metadata mean that physical parity is + * not required to match parity recomputed from the current filesystem. + * + * Report such a mismatch as a synchronization error. Only a mismatch on a + * position expected to be fully synchronized is a silent parity error. + */ if (block_is_unsynced) { log_tag("parity_error:%" PRIu64 ":%s: Data error, diff parity bits %u/%u\n", blockcur, lev_config_name(l), diff, state->block_size * 8); ++soft_error; @@ -594,15 +856,33 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } if (silent_error_on_this_block || io_error_on_this_block) { - /* set the error status keeping other info */ + /* + * Mark the complete parity position bad, preserving its previous time, + * rehash state and just-synced state. + * + * A failed scrub must not refresh the last-known-good time or make pending + * metadata work appear completed. + */ info_set(&state->infoarr, blockcur, info_set_bad(info)); } else if (error_on_this_block) { /* - * Do nothing, as this is a generic error - * likely caused by a not synced array + * A soft error does not establish either success or corruption. + * + * Keep the previous info unchanged: do not mark the position bad, but also do + * not refresh its scrub time, clear justsynced, or complete a pending rehash. */ } else { - /* if rehash is needed */ + /* + * Publish staged hashes before clearing the rehash flag. Both changes are then + * persisted by the same content write, so a saved position is interpreted + * consistently with either the previous or the current hash algorithm. + * + * For a CHG block there was no current hash to validate directly. Reaching + * this point nevertheless means parity agrees with the current bytes just + * read, so these bytes form the verified parity baseline represented by the + * newly stored digest. The block remains CHG and therefore still conservatively + * requires sync. + */ if (rehash) { /* store all the new hash already computed */ for (j = 0; j < diskmax; ++j) { @@ -612,8 +892,12 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } /* - * Update the time info of the block - * and clear any other flag + * A completely clean scrub establishes a new last-known-good checkpoint for + * the whole parity position. + * + * Refresh the time and clear bad, rehash and justsynced together. In + * particular this is the only normal scrub path that clears a previous bad + * mark. */ info_set(&state->infoarr, blockcur, info_make(now, 0, 0, 0)); } @@ -874,6 +1158,14 @@ int state_scrub(struct snapraid_state* state, int plan100, int olderthan) tommy_node* j = tommy_list_head(&state->bucketlist); block_off_t processed_count = 0; + + /* + * Buckets are sorted oldest first. Walk them until either the age cutoff or + * requested count is reached. + * + * If the count ends inside a bucket, timelimit identifies that bucket and + * lastlimit records how many positions from it may be selected. + */ while (j) { struct snapraid_bucket* bucket = j->data; block_off_t bucket_count = bucket->count_justsynced + bucket->count_scrubbed; @@ -895,7 +1187,13 @@ int state_scrub(struct snapraid_state* state, int plan100, int olderthan) j = j->next; } - /* if nothing to scrub, disable also other limits */ + /* + * Disable selection completely when no normal AUTO position is eligible. + * Leaving timelimit initialized to 'now' would otherwise make the subsequent + * per-position filter appear permissive. + * + * Bad positions are unaffected because they bypass AUTO limits. + */ if (processed_count == 0) { ps.timelimit = 0; ps.lastlimit = 0; diff --git a/cmdline/selftest.c b/cmdline/selftest.c index d8722005..1f148d37 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -303,7 +303,7 @@ static void test_parity(void) static const struct { unsigned split_mac; data_off_t size[2]; - data_off_t valid_size[2]; + data_off_t physical_reach_size[2]; data_off_t expected; } test[] = { { 1, { 100, 0 }, { 100, 0 }, 100 }, @@ -326,13 +326,13 @@ static void test_parity(void) handle.split_mac = test[i].split_mac; for (s = 0; s < handle.split_mac; ++s) { handle.split_map[s].size = test[i].size[s]; - handle.split_map[s].valid_size = test[i].valid_size[s]; + handle.split_map[s].physical_reach_size = test[i].physical_reach_size[s]; } - parity_valid_size(&handle, &size); + parity_physical_reach_size(&handle, &size); if (size != test[i].expected) { /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "Failed parity valid size test\n"); + log_fatal(EINTERNAL, "Failed parity physical reach size test\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/sync.c b/cmdline/sync.c index ab72df45..e5d54383 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -11,6 +11,240 @@ #include "io.h" #include "raid/raid.h" +/* + * SnapRAID sync model + * + * Sync brings the stored parity and content metadata in line with the data + * files observed by SnapRAID. + * + * On the first sync there is no previously synchronized state. SnapRAID scans + * the filesystem, allocates parity blocks, reads the data, computes hashes, + * and generates parity. The resulting content file records the synchronized + * file metadata, block allocation and hashes. + * + * On subsequent syncs the filesystem is scanned again and compared with the + * previous content state. Added, removed, moved, restored or changed files + * invalidate only the parity positions affected by those changes. + * + * Content and parity are related but distinct persistent states. The content + * file records SnapRAID's logical knowledge of the array; the parity files + * contain the physically generated redundancy. + * + * They do not advance atomically. If a sync is interrupted, some parity + * positions may already have been updated while the content file still + * describes an intermediate state. CHG, REP, BLK and DELETED preserve enough + * information to handle these conditions safely on a later run. + * + * Consequently, content state must not be interpreted as an unconditional + * description of the bytes currently present in parity. In particular, + * CHG/REP/DELETED deliberately represent states for which physical parity + * cannot be trusted to correspond to the current logical data. + * + * Block state model + * ----------------- + * + * A block state describes the relationship between: + * + * - the data currently associated with the parity position, + * - the hash stored in the content state, + * - the parity physically stored on disk. + * + * OLD and NEW below refer to generations of data, not necessarily to the hash + * algorithm. During rehashing, per-position information determines whether a + * hash belongs to the previous or current hash algorithm. + * + * The stable states are: + * + * State File Stored hash Parity meaning + * ------------------------------------------------------------------------ + * EMPTY no none logical input is zero + * CHG yes OLD / ZERO / INVALID invalid for current data + * REP yes NEW invalid for current data + * BLK yes CURRENT synchronized contribution + * DELETED no OLD / INVALID invalid for logical zero + * + * "Invalid" is conservative. It does not necessarily mean that the bytes in + * parity are wrong. After an interrupted sync parity may still represent OLD + * data, may already represent NEW data, or may have been only partly updated. + * CHG, REP and DELETED mean that no particular physical parity state may be + * assumed. + * + * EMPTY contributes zero and does not by itself require an update. Therefore + * a parity position containing only EMPTY and DELETED blocks may be skipped: + * no live file depends on it. A DELETED block alone does not necessarily mean + * that the array is considered unsynchronized. + * + * BLK is a per-data-block property. It means that this block does not itself + * require a parity update. It does not imply that the whole parity stripe is + * valid if another disk at the same position contains CHG, REP or DELETED. + * + * The main state transitions are: + * + * Transition Hash after Physical parity + * ------------------------------------------------------------------------ + * EMPTY -> CHG ZERO represents the previous zero input + * DELETED -> CHG OLD/INVALID unknown and not trusted + * EMPTY -> REP NEW represents the previous zero input + * DELETED -> REP NEW unknown and not trusted + * + * BLK -> DELETED OLD initially still represents OLD data + * CHG -> DELETED unchanged already unknown + * REP -> DELETED INVALID already unknown + * + * REP -> CHG INVALID unchanged and not trusted + * CHG -> REP NEW unchanged and not trusted + * + * CHG -> BLK CURRENT valid when sync progress is committed + * REP -> BLK CURRENT valid when sync progress is committed + * DELETED -> EMPTY none valid when sync progress is committed + * + * EMPTY/DELETED -> CHG/REP and BLK/CHG/REP -> DELETED are normally produced + * by the filesystem scan before sync starts modifying parity. REP -> CHG is + * also used when a previously stored copy-derived hash is invalidated. + * + * CHG -> REP is performed by the optional prehash pass. It means that NEW data + * has been read and hashed, while parity remains explicitly invalid. + * + * CHG can become BLK in two ways. Normally parity is regenerated. However, if + * the previous sync completed normally, the OLD hash is unambiguous and the + * current data has the same hash, the existing parity contribution is already + * correct and no parity write is needed. This optimization is disabled after + * an interrupted sync because the stored OLD hash can no longer be reliably + * associated with the physical parity state. + * + * The table describes stable state semantics. During processing, a CHG hash + * may temporarily be replaced in memory by the hash just computed for current + * data before the block is promoted to BLK. Such temporary values are + * implementation state and do not redefine the persistent meaning of CHG. + * + * Crash consistency model + * ----------------------- + * + * State changes made by state_sync_process() are not immediate durability + * guarantees. CHG/REP may become BLK, and DELETED may become EMPTY, in memory + * before the corresponding asynchronous parity write has completed. + * + * This is safe because completed state is not published to the content file + * until parity has crossed a durability barrier. The ordering is: + * + * 1. Scan creates CHG, REP and DELETED pending states. + * + * 2. The pending content state is durably written before parity is changed. + * A crash after this point therefore leaves enough information to + * identify every parity position that may have been modified. + * + * 3. Sync reads current data and computes new parity. Blocks may already be + * changed to BLK/EMPTY in memory while parity writes are still pending. + * + * 4. Before an autosave publishes these completed states, state_barrier() + * drains pending parity writes and synchronizes the parity files. + * + * 5. Only after that barrier may BLK/EMPTY progress be written to content. + * Final content persistence follows the same ordering. + * + * The persistent crash invariant is therefore: + * + * CHG / REP / DELETED in content + * => physical parity must be treated as unknown. + * + * BLK / EMPTY progress published by sync + * => the parity required by that progress was made durable before the + * content state was written. + * + * If a crash happens after parity was written but before completed content + * state was saved, content simply retains CHG/REP/DELETED. A later sync may + * repeat work, but it does not incorrectly trust the modified parity. + * + * DELETED has one additional persistence case. Trailing positions containing + * no live data can disappear when parity is shortened, because + * parity_allocated_size() excludes trailing positions without files. The + * subsequent content state therefore no longer needs to serialize those + * trailing DELETED entries. + * + * Parity stripe model + * ------------------- + * + * Sync operates on parity positions. At each position, the blocks from all + * data disks form the data inputs of the same parity stripe and must therefore + * be considered together. + * + * A stripe requires sync processing only when it contains both an invalid + * parity contribution and at least one live data block. If no contribution is + * invalid, parity already represents all current live inputs. If no live block + * exists, the parity at that position is irrelevant to any file and may be + * left untouched even when DELETED blocks remain. + * + * In simplified form, the decision is: + * + * no invalid block + * => no parity update is required + * + * invalid block, but no live block + * => the stripe may be skipped + * + * invalid block and at least one live block + * => the stripe must be processed + * + * This is why a DELETED block alone does not necessarily make the array + * unsynchronized. Its old contribution only matters when the same parity + * position is still used by live data on another disk. + * + * Invalid blocks are also handled conservatively during silent-error + * recovery. CHG, REP and DELETED cannot in general be used as known inputs to + * the recovery equations because their current logical data may differ from + * the data represented by physical parity, especially after an interrupted + * sync. + * + * They are therefore treated as unknown inputs while reconstructing a + * suspected BLK block. A CHG whose stored OLD value is known to be zero is a + * special case: the old zero contribution can be supplied directly and does + * not consume a recovery equation. + * + * Any reconstructed values for CHG, REP or DELETED are only intermediate + * values needed to recover synchronized data and regenerate parity. Their + * current data buffers remain the authoritative inputs for completing sync. + * + * Live filesystem model + * --------------------- + * + * SnapRAID is not a real-time RAID system. A successful sync establishes a + * synchronized point in filesystem history, but data files remain writable + * before, during and after the operation. + * + * Sync does not require every file to remain unchanged for its whole duration + * and does not provide a transactional or atomic filesystem view. + * + * A file may therefore change while it is being processed. Such a change has + * essentially the same effect as changing the file immediately after it was + * synchronized: the parity generated by the current sync may describe an + * older or intermediate version while the live filesystem already contains a + * newer one. + * + * This is expected. Parity that no longer matches a file modified during or + * after sync is not by itself evidence of parity corruption. A later sync + * brings the parity back in line with the then-current filesystem state. + * + * A concurrent modification does not therefore have to be detected by that + * same sync to preserve the normal SnapRAID consistency model. If the change + * affects metadata used by normal change detection, such as size or mtime, a + * later scan will detect it and process the file again. + * + * The same applies to races around file access, including writes after open, + * between block reads, or immediately before or after the final read. These + * are equivalent, for consistency purposes, to modifying a file immediately + * after synchronization. + * + * Metadata checks performed during sync are therefore opportunistic checks, + * not a mechanism for freezing the filesystem or detecting every concurrent + * modification. + * + * This model relies on later changes being visible to normal change detection. + * An in-place modification that preserves all relevant metadata, for example + * both file size and modification time, may not be detected by a later scan. + * This is a general limitation of metadata-based change detection, not a + * special property of modifications occurring during sync. + */ + /****************************************************************************/ /* hash */ @@ -275,6 +509,14 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta /* the only other case is BLOCK_STATE_CHG */ assert(block_state == BLOCK_STATE_CHG); + /* + * Prehash is a persistent state transition: CHG -> REP records that + * new data has been read and hashed while parity remains invalid. + * + * Persisting this state before parity writes preserves verified hashes + * across interruptions without claiming synchronization. + */ + /* copy the hash in the block */ memcpy(block->hash, hash, BLOCK_HASH_SIZE); @@ -463,7 +705,18 @@ static int block_is_enabled(struct snapraid_plan* plan, block_off_t i) one_invalid = 1; } - /* if none valid or none invalid, we don't need to update */ + /* + * A parity position needs processing only if it contains both: + * - at least one live data block, and + * - at least one block with invalid parity. + * + * Positions with only EMPTY/DELETED blocks are skipped even if parity + * contains old non-zero data: no live file depends on it. Reused positions + * become CHG/REP, forcing parity regeneration before becoming BLK. + * + * Requiring one_valid also avoids rewriting unused parity that will be + * removed by parity truncation. + */ if (!one_invalid || !one_valid) return 0; @@ -1064,9 +1317,21 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } /* - * If we have only silent errors we can try to fix them on-the-fly - * note the fix is not written to disk, but used only to - * compute the new parity + * On silent errors, attempt on-the-fly recovery to avoid contaminating + * new parity with bad data. Repaired data is kept in memory only. + * + * Recovery reconstructs the state represented by on-disk parity. For + * CHG/REP/DELETED blocks, current buffers may differ from on-disk parity, + * especially after an interrupted sync. Treating them as erasures prevents + * raid_rec() from trusting their current contents to recover a bad BLK. + * + * CHG blocks with known ZERO old values supply zero directly without + * consuming a parity equation. If an interrupted sync had already moved + * parity to the new CHG value, hash validation fails and recovery is + * discarded. + * + * After recovery, current buffers are restored for CHG/REP/DELETED: + * reconstructed old values must not roll back pending filesystem changes. */ if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) { unsigned failed_mac; @@ -1157,10 +1422,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* if no error in parity read */ if (!io_error_on_this_block) { /* - * Try to fix the data - * note that this is a simple fix algorithm, that doesn't take into - * account the case of a wrong parity - * only 'fix' supports the most advanced fixing + * Reconstruct data using on-disk parity. Unlike 'fix', this simple + * recovery does not handle corrupt parity. + * + * Recovered BLK data is repaired in memory only to compute clean + * parity without writing back to disk. The block remains bad on + * disk for check/fix to repair later. */ raid_rec(failed_mac, failed_map, diskmax, state->level, state->block_size, buffer); @@ -1542,6 +1809,16 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t msg_progress("Initializing...\n"); + /* + * Track two distinct parity extents: + * + * - allocated_size: extent required by the new layout, including live + * CHG/REP blocks whose parity is not yet generated. + * - used_size: extent claimed valid by current content state (last BLK). + * + * Physical parity may be shorter than allocated_size (sync will extend it), + * but must not be shorter than used_size (which indicates missing parity). + */ blockmax = parity_allocated_size(state); size = blockmax * (data_off_t)state->block_size; @@ -1578,7 +1855,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t } /* number of block in the parity file */ - parity_valid_size(&parity_handle[l], &out_size); + parity_physical_reach_size(&parity_handle[l], &out_size); parityblocks = out_size / state->block_size; /* if the file is too small */ @@ -1586,7 +1863,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t log_fatal(ESOFT, "WARNING! The %s parity has only %" PRIu64 " blocks instead of %" PRIu64 ".\n", lev_name(l), parityblocks, used_paritymax); } - /* keep the smallest valid parity size */ + /* keep the smallest physical parity extent */ if (l == 0 || file_parity_size > out_size) file_parity_size = out_size; } @@ -1722,6 +1999,10 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t } } + /* + * On close errors, abort without committing or persisting state, + * keeping the previous content file intact for recovery on the next run. + */ if (process_error != 0) return -1;