// SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni #include "os/portable.h" #include "support.h" #include "util.h" #include "elem.h" #include "import.h" #include "search.h" #include "state.h" #include "parity.h" #include "handle.h" #include "raid/raid.h" #include "raid/combo.h" /** * SnapRAID check/fix model * * Check verifies the data described by the content file against the files * currently present on the data disks. For blocks with a current hash, the * data is verified directly by hashing it. Unless running in audit-only mode, * parity is also read and can be used both to recover unreadable or corrupted * data and to verify the stored parity itself. * * Fix uses the same verification and recovery logic as check, but writes * validated recovered data and parity back to disk. Recovery is deliberately * conservative: obtaining a solution from the RAID equations is not enough; * the result must also be validated independently before it can be accepted. * * Block state model * ----------------- * * Check/fix must interpret block states differently because the content file * may describe an incomplete sync. In that case, the logical block state and * the physical parity on disk do not necessarily describe the same point in * time. * * State Stored hash Current file Physical parity * ------------------------------------------------------------------------ * EMPTY none none zero contribution * BLK CURRENT present synchronized * CHG OLD/ZERO/INVALID present may be OLD or CURRENT * REP NEW present may be OLD or NEW * DELETED OLD/INVALID none may be OLD or zero * REBUILD CURRENT present parity untrusted / rebuild pending * * BLK has a hash for the data that should currently be present and normally * represents a synchronized contribution. * * REP also has a hash for the current data, so its current contents can be * verified. However, parity is still invalid because an interrupted sync may * have left physical parity representing either the old or the new block. * * CHG is different: its stored hash describes the OLD data, not the current * file contents. A readable CHG block therefore cannot be validated against * block->hash. Its current contents are kept as the best available copy, but * the old hash is useful when considering parity that may still represent the * state before sync. * * DELETED has no current file data. Its stored hash, when available, describes * the previous contribution that may still be present in physical parity. * * Interrupted sync recovery model * ------------------------------- * * When CHG, REP or DELETED blocks are present, check/fix cannot know whether * an interrupted sync updated the physical parity before stopping. Recovery * therefore tries two different interpretations of the same parity. * * The first strategy assumes that parity represents the CURRENT state: * * BLK -> current data is represented * CHG -> current data is represented * REP -> new/current data is represented * DELETED -> contributes zero * REBUILD -> data and hash are CURRENT, but physical parity remains untrusted * * Under this assumption, readable current data can be used directly and only * blocks known to be bad or missing have to be reconstructed. BLK and REP * recovery can be validated with their stored current hashes. CHG recovery is * more subtle because only its OLD hash is available; a recovered CHG may * therefore remain ambiguous and be marked out-of-date. * * If that strategy cannot produce a validated recovery, a second strategy * assumes that parity still represents the state BEFORE the interrupted sync: * * BLK -> synchronized data is represented * CHG -> OLD data is represented * REP -> OLD data is represented * DELETED -> OLD data is represented * REBUILD -> data and hash remain CURRENT; only physical parity is untrusted * * * REBUILD is orthogonal to the OLD/CURRENT sync-history distinction. Unlike * CHG, REP and DELETED, it does not represent a different data generation. * Its data and stored hash are CURRENT in both strategies. Only its physical * parity is untrusted because an in-place --force-full rebuild may have been * interrupted. * * Under this assumption, the current contents of CHG and REP cannot be used * as parity inputs even when they are perfectly readable. CHG, REP and * DELETED must instead be treated as unknown old contributions, reconstructed * only as needed to make recovery of a bad BLK possible. * * These reconstructed old contributions are auxiliary recovery state, not * current user data. They are marked out-of-date so they cannot validate the * recovery and so an old CHG/REP result is never promoted as known-good * current data. * * A CHG whose OLD hash is ZERO is a useful special case: its old contribution * is known exactly and can be supplied as an all-zero block without consuming * a RAID recovery equation. Similarly, if an old CHG or DELETED block can be * obtained from the import set using its stored OLD hash, that known old * contribution reduces the number of unknowns. REP cannot use this shortcut * because its stored hash describes the NEW data, not the old contribution. * * Recovery validation model * ------------------------- * * Solving the RAID equations does not by itself prove that the reconstructed * data is correct. One of the parity blocks used for reconstruction may itself * be damaged, and after an interrupted sync different parity levels may not * necessarily provide the expected state. * * For F unknown data blocks, recovery therefore requires independent evidence: * * - If at least one recovered block has a trustworthy current hash, F parity * equations are sufficient. The candidate recovery is accepted only if all * available trustworthy hashes of recovered blocks match. * * - If no recovered block has a trustworthy hash, F+1 parity blocks are * required. F parity blocks are used to solve the data equations and the * remaining parity block is reserved as an independent check of the result. * * When more parity levels are available, repair_step() tries their different * combinations. Missing parity levels are skipped, and a candidate obtained * from a bad parity combination is rejected when its hash or independent * parity check fails. * * This distinction is important: a successful RAID decode is only a candidate * recovery. It becomes an accepted recovery only after independent validation. * * Parity validation model * ----------------------- * * Physical parity is checked or rewritten only when it is both meaningful and * expected to be valid. * * used_parity means that at least one live data block uses the parity position. * Parity for a position containing no live data is irrelevant and sync is not * required to keep such unused parity updated. * * valid_parity means that no block at the position is CHG, REP, DELETED or * REBUILD. * * For CHG, REP and DELETED, parity is invalid because its data-generation * relationship is uncertain after an interrupted sync. For REBUILD the reason * is different: the data and hash are CURRENT, but physical parity itself is * deliberately untrusted until the in-place rebuild completes. * When one of these states is present, parity may legitimately differ from * parity recomputed from the current data because sync has not completed. * Such a difference must not be reported or "fixed" by check/fix; the next * sync is responsible for bringing that parity position to the current state. * * Parity is therefore compared and repaired only when: * * used_parity && valid_parity * * A parity level excluded from fix is not necessarily useless. When accessible * it is still opened for reading because it can provide an additional recovery * equation or independent validation, while remaining protected from writes. * * Fix and quarantine model * ------------------------ * * failed_struct distinguishes two properties of a recovery entry. * * is_bad means that the current file block is missing, unreadable or does not * match its expected current hash and therefore needs recovery. * * recovery tracks the data state of the block in buffer[index]: * * RECOVERY_NONE not recovered (no valid replacement data available) * RECOVERY_CURRENT certified CURRENT data (via import/search or RAID) * RECOVERY_OUTOFDATE recovered candidate that cannot be certified CURRENT * * Outside repair(), file_recover() writes RECOVERY_CURRENT data to the original * file and reports it as fixed. An out-of-date candidate (RECOVERY_OUTOFDATE) * is written as best-effort data, but the containing file remains marked damaged * and is quarantined under the .unrecoverable name. * * Missing files are created under their normal names. They are quarantined only * if recovery produces data that cannot be certified as CURRENT. * * A later fix starts a new recovery attempt under the normal filename even if a * previous .unrecoverable file exists. A previous quarantine cannot safely be * reused as recovery input because it may contain an ambiguous recovered CHG. * Such a CHG may have caused the file to be quarantined precisely because its * CURRENT generation could not be proven. That provenance is runtime state and * is not persisted in the .unrecoverable file. Reopening it in a later fix would * make the CHG merely appear readable and could incorrectly accept it as CURRENT. * * The previous quarantine is therefore kept separate while the new recovery is * in progress. If the new attempt succeeds completely, the obsolete quarantine * is removed. If the new attempt also becomes unrecoverable, it replaces the * previous quarantine. * * A quarantined file may be reopened later during the same fix run. In that case * FILE_IS_QUARANTINED identifies it as belonging to the current failed recovery and * handle_open_create() continues operating on its .unrecoverable file. * * A partial -S/-B fix operates only at block level and therefore never performs * file-level finalization. */ /****************************************************************************/ /* check */ static const char* es(int err) { if (is_hw(err)) return "error_io"; else return "error"; } #define RECOVERY_NONE 0 /**< Not recovered (no valid candidate available) */ #define RECOVERY_CURRENT 1 /**< Recovered and certified CURRENT (via import/search or RAID) */ #define RECOVERY_OUTOFDATE 2 /**< Recovered candidate that cannot be certified CURRENT (best-effort / OLD / ambiguous) */ /** * A block participating in recovery. * * Entries include blocks that are actually missing or corrupted, but also * readable CHG, REP and DELETED blocks whose old/current contribution may be * needed to interpret parity left by an interrupted sync. */ struct failed_struct { /** * If the current file block is known to require recovery because it is * missing, unreadable, or doesn't match its expected current hash. * * DELETED blocks and readable CHG/REP auxiliary entries are not bad even * though they may participate in the recovery equations. * * is_bad describes the state of the original file block. It may remain set * even when buffer[index] already contains valid replacement data. */ int is_bad; /** * Data state of the buffer associated with this entry: * * RECOVERY_NONE: * The block is not recovered or does not contain valid replacement data. * * RECOVERY_CURRENT: * buffer[index] contains certified CURRENT data. * During repair() strategy construction, this marks blocks obtained * and validated independently through import/search, making them reusable * as known inputs across strategies whose data generation is unchanged * (such as BLK and REBUILD). * Upon successful recovery, repair() marks all recovered non-outofdate blocks * with RECOVERY_CURRENT. * Outside repair(), file_recover() writes this data to the original file * and marks it FIXED. * * RECOVERY_OUTOFDATE: * buffer[index] contains a recovered candidate that cannot be proven to * represent the required CURRENT contents (e.g. reconstructed under the * pre-sync parity strategy, or ambiguous CHG). * If an out-of-date block is also is_bad, fix may still preserve the recovered * bytes as the best available copy, but the containing file remains damaged * and quarantined as .unrecoverable. */ int recovery; unsigned index; /**< Index of the failed block. */ struct snapraid_block* block; /**< The failed block */ struct snapraid_disk* disk; /**< The failed disk. */ struct snapraid_file* file; /**< The failed file. 0 for DELETED block. */ block_off_t file_pos; /**< Offset inside the file */ struct snapraid_handle* handle; /**< The handle containing the failed block, or 0 for a DELETED block */ }; /** * Check if a block hash matches the specified buffer. * * This function validates hash identity only and deliberately omits the * zero-padding check beyond the logical file size. It is used when testing * candidate identity against a stored hash whose original logical size and * padding provenance may differ from the CURRENT file metadata (such as * during CHG generation inference). * * Return ==0 if equal, -1 if mismatch. */ static int block_hash_cmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, size_t pos_size, unsigned char* buffer) { unsigned char hash[HASH_MAX]; /* compute the hash according to the block-hash semantics */ if (rehash) { memhash_block(state->prevhash, state->prevhashseed, hash, buffer, pos_size, state->block_size); } else { memhash_block(state->hash, state->hashseed, hash, buffer, pos_size, state->block_size); } /* compare the hash */ if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) { return -1; } return 0; } /** * Check if block data matches the specified block and is canonical for the current logical size. * * Unlike block_hash_cmp(), this function validates both hash identity and the * canonical zero-padding requirement beyond the CURRENT logical file size. * For legacy hashes (Murmur3/Spooky2), the hash covers only logical bytes, so * trailing garbage or corrupted parity could otherwise pass validation and later * corrupt parity recalculations. For MuseAir, this enforces the canonical RAID * block invariant. * * Use this for all normal CURRENT data validation (e.g. is_hash_matching()). * * Return ==0 if equal and valid, -1 if mismatch. */ static int block_data_cmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, size_t pos_size, unsigned char* buffer) { if (block_hash_cmp(state, rehash, block, pos_size, buffer) != 0) { return -1; } /* * Legacy hashes cover only the logical bytes belonging to the file, while RAID * operates on the complete zero-padded block. Verify the padding separately: * otherwise corruption confined to the reconstructed tail could pass a legacy * hash check and later produce incorrect parity. This is redundant for MuseAir, * but also makes the canonical-block invariant explicit. */ if (pos_size < state->block_size) { if (membcmp(buffer + pos_size, 0, state->block_size - pos_size) != 0) { return -1; } } return 0; } /** * Check if the specified block has a trustworthy CURRENT hash under the given strategy. * * In the OLD recovery strategy, physical parity reflects the pre-sync state, * so RAID reconstructs the OLD content of this block. However, for a REP * block, block->hash describes the NEW (post-replace) data. Comparing the * reconstructed OLD bytes against the NEW hash would falsely fail validation. */ static int block_has_recovery_hash(const struct snapraid_block* block, int is_old_strategy) { unsigned state = block_state_get(block); if (!block_has_updated_hash(block)) return 0; if (is_old_strategy && state == BLOCK_STATE_REP) return 0; return 1; } /** * Check if the hash of all the failed blocks we are expecting to recover are now matching. */ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, int is_old_strategy) { unsigned j; int hash_checked; hash_checked = 0; /* keep track if we check at least one block */ /* check if the recovered blocks are OK */ for (j = 0; j < failed_count; ++j) { /* if the block has a hash to check in this strategy */ if (block_has_recovery_hash(failed[failed_map[j]].block, is_old_strategy)) { /* if a hash doesn't match, fail the check */ size_t pos_size = file_block_size(failed[failed_map[j]].file, failed[failed_map[j]].file_pos, state->block_size); if (block_data_cmp(state, rehash, failed[failed_map[j]].block, pos_size, buffer[failed[failed_map[j]].index]) != 0) { log_tag("repair_hash_error:%u: Hash mismatch\n", failed_map[j]); return 0; } hash_checked = 1; } } /* * If nothing checked, we reject it * note that we are excluding this case at upper level * but checking again doesn't hurt */ if (!hash_checked) { /* LCOV_EXCL_START */ return 0; /* LCOV_EXCL_STOP */ } /* * If we checked something, and no block failed the check * recompute all the redundancy information */ raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 1; } /** * Check if specified parity is now matching with a recomputed one. */ static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, unsigned i, void** buffer, void** buffer_recov) { /* recompute parity, note that we don't need parity over i */ raid_gen(diskmax, i + 1, state->block_size, buffer, 0); /* if the recovered parity block matches */ if (memcmp(buffer[diskmax + i], buffer_recov[i], state->block_size) == 0) { /* recompute all the redundancy information */ raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 1; } return 0; } /** * Try to recover the selected failed inputs and validate the reconstructed * result using either a CURRENT hash or an additional RAID equation not used * for reconstruction. * * Return: * 0 if a recovery was successfully validated; * >0 if one or more recovery candidates were tried but all failed validation; * <0 if no usable recovery/validation strategy exists. * * On success all parity levels are recomputed in buffer. */ static int repair_step(struct snapraid_state* state, int rehash, block_off_t pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, int is_old_strategy) { unsigned i, n; int error; int has_hash; int id[LEV_MAX]; int ip[LEV_MAX]; /* no fix required, already checked at higher level, but just to be sure */ if (failed_count == 0) { /* LCOV_EXCL_START */ /* recompute only the parity */ raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 0; /* LCOV_EXCL_STOP */ } n = state->level; error = 0; /* if failures exceed parity level, recovery is impossible */ if (failed_count > n) { log_tag("recover_strategy_error:%" PRIu64 ": Impossible to recover from %u failures with %u parity\n", pos, failed_count, n); return -1; } /* setup vector of failed disk indexes */ for (i = 0; i < failed_count; ++i) id[i] = failed[failed_map[i]].index; /* * RAID decoding alone cannot tell whether the parity equations used for the * recovery were themselves correct. * * If at least one failed input has a trustworthy CURRENT hash, recovered data * can provide independent validation. Otherwise one additional parity equation * is reserved exclusively for checking the candidate result. * * Which block states provide a trustworthy hash depends on the strategy: * * - CURRENT strategy (is_old_strategy == 0): BLK, REP, and REBUILD have * trustworthy CURRENT hashes. CHG blocks do not (their stored hash describes * pre-sync OLD data). * * - OLD strategy (is_old_strategy == 1): only BLK and REBUILD provide trustworthy * hashes (BLK represents synchronized data covered by pre-sync parity; REBUILD * data is always CURRENT). REP is excluded because its stored hash describes * NEW data while RAID reconstructs its pre-sync OLD contribution. CHG and DELETED * have no updated hash. * * The additional parity is an independent RAID equation, not an independent * hash of the recovered data. The candidate is accepted only if the complete * reconstructed block set satisfies this extra equation for the whole block. * Therefore an accidental false match requires the data differences to satisfy * the corresponding RAID relation at every byte position. Such correlated * differences are possible in principle because RAID equations are linear, but * are extremely unlikely for unrelated data corruption. */ has_hash = 0; for (i = 0; i < failed_count; ++i) { if (block_has_recovery_hash(failed[failed_map[i]].block, is_old_strategy)) has_hash = 1; } /* * If we don't have a hash, but we have an extra parity * (strictly-less failures than number of parities) */ if (!has_hash && failed_count < n) { /* number of parity to use, one more to check the recovering */ unsigned r = failed_count + 1; /* all combinations (r of n) parities */ combination_first(r, n, ip); do { /* if a parity is missing, do nothing */ for (i = 0; i < r; ++i) { if (buffer_recov[ip[i]] == 0) break; } if (i != r) continue; /* copy the parities to use, one less because the last is used for checking */ for (i = 0; i < r - 1; ++i) memcpy(buffer[diskmax + ip[i]], buffer_recov[ip[i]], state->block_size); /* recover using one less parity, the ip[r-1] one */ raid_data(r - 1, id, ip, diskmax, state->block_size, buffer); /* use the remaining ip[r-1] parity to check the result */ if (is_parity_matching(state, diskmax, ip[r - 1], buffer, buffer_recov)) return 0; /* log */ log_tag("recover_parity_error:%" PRIu64 ":", pos); for (i = 0; i < r; ++i) { if (i != 0) log_tag(","); log_tag("%s", lev_config_name(ip[i])); } log_tag(": Parity mismatch\n"); ++error; } while (combination_next(r, n, ip)); } /* * If we have a hash, and enough parities * (less-or-equal failures than number of parities) */ if (has_hash && failed_count <= n) { /* number of parities to use equal to the number of failures */ unsigned r = failed_count; /* all combinations (r of n) parities */ combination_first(r, n, ip); do { /* if a parity is missing, do nothing */ for (i = 0; i < r; ++i) { if (buffer_recov[ip[i]] == 0) break; } if (i != r) continue; /* copy the parities to use */ for (i = 0; i < r; ++i) memcpy(buffer[diskmax + ip[i]], buffer_recov[ip[i]], state->block_size); /* recover */ raid_data(r, id, ip, diskmax, state->block_size, buffer); /* use the hash to check the result */ if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, is_old_strategy)) return 0; /* log */ log_tag("recover_hash_error:%" PRIu64 ":", pos); for (i = 0; i < r; ++i) { if (i != 0) log_tag("/"); log_tag("%s", lev_config_name(ip[i])); } log_tag(": Hash mismatch\n"); ++error; } while (combination_next(r, n, ip)); } /* return the number of failed attempts, or -1 if no strategy */ if (error) return error; log_tag("recover_strategy_error:%" PRIu64 ": No strategy to recover from %u failures with %u parity %s hash\n", pos, failed_count, n, has_hash ? "with" : "without"); return -1; } /** * Try to recover all the bad blocks of a parity stripe. * * Recovery after an interrupted sync is ambiguous because physical parity may * represent either the CURRENT filesystem state or the OLD state that existed * before the interrupted sync. For this reason two recovery strategies are * attempted. * * This recovery logic relies on the following invariant: * * After an interrupted sync, the data-disk filesystem must not be changed * before running recovery or before a subsequent sync completes successfully. * * Re-running sync without changing the filesystem is allowed, even if that sync * is interrupted again. Under this invariant an unsynchronized CHG has only two * relevant generations: * * OLD identified by the stored CHG hash; * CURRENT represented by the unchanged live filesystem contents. * * Physical parity may contain either generation because the interrupted sync * may or may not have updated it, but there cannot be a third intermediate data * generation created by a later filesystem change. * * This invariant is essential for CHG generation inference. If the filesystem * is changed after an interrupted sync, for example by adding, deleting, * replacing or modifying files, a stored CHG/DELETED OLD hash may no longer * identify the generation actually represented by physical parity. In that * situation candidate != stored OLD would not be sufficient to prove CURRENT. * * 1. CURRENT strategy. * * Assume that parity was already updated by the interrupted sync. * Readable blocks are therefore used as their CURRENT values and only bad * blocks are reconstructed. * * BLK, REP and REBUILD have hashes identifying their required CURRENT * contents and can therefore be independently validated. * * CHG is different: its stored hash identifies the OLD contents, not the * CURRENT ones. After repair_step() has independently validated the RAID * solution, the recovered CHGs are examined as a group. * * Under the recovery invariant above, a recovered CHG candidate different * from its stored OLD contents cannot belong to the OLD generation and * therefore proves that the validated RAID solution is CURRENT. * * All CHGs reconstructed by the same repair_step() belong to that same * validated RAID solution. Consequently, if at least one recovered CHG * proves that the solution is CURRENT, all CHGs reconstructed by that * solution are CURRENT, including CHGs whose OLD hash is INVALID or whose * OLD and CURRENT contents happen to be identical. * * If no recovered CHG distinguishes the solution from OLD, all recovered * CHGs remain RECOVERY_OUTOFDATE. This does not mean that their contents are * necessarily wrong or actually old, only that CURRENT cannot be proven. * * 2. OLD strategy. * * If the CURRENT strategy fails, assume that physical parity still * represents the state before the interrupted sync. * * CHG, REP and DELETED entries are therefore converted to their OLD * contribution when possible, or reconstructed as auxiliary RAID unknowns. * This strategy is useful only when there is bad BLK/REBUILD data to * recover. Any bad CHG/REP reconstructed under this strategy remains * RECOVERY_OUTOFDATE because it cannot be certified as the required CURRENT data. * * REBUILD is orthogonal to the OLD/CURRENT sync-history distinction. Its data * and stored hash are always CURRENT, while its physical parity is untrusted * because an in-place --force-full rebuild may have been interrupted. REBUILD * therefore never requires conversion to an OLD data contribution and does not * participate in the CHG generation inference. A matching REBUILD hash proves * the recovered REBUILD data, but does not prove whether CHG/REP/DELETED parity * belongs to the OLD or CURRENT sync generation. * * repair_step() is responsible only for solving and independently validating * the RAID equations. This function is responsible for interpreting the * resulting solution according to the OLD/CURRENT history of unsynchronized * blocks and the recovery invariant described above. * * Parameters: * * state SnapRAID state and configuration. * * rehash If non-zero, hashes must be computed using the previous hash * algorithm/seed while processing a rehash operation. * * pos Parity block position being recovered. Used for logging. * * diskmax Number of data disks participating in the RAID stripe. * * failed Array describing all blocks relevant to recovery. Entries may * represent actually bad blocks, but also readable CHG/REP or * DELETED blocks needed by the OLD recovery strategy. * * failed_map Temporary array used to select which failed[] entries are RAID * unknowns for each call to repair_step(). * * failed_count Number of valid entries in failed[]. * * buffer Data and generated parity buffers. Data buffers are indexed by * disk, followed by generated parity buffers. * * buffer_recov Physical parity blocks read from disk. A NULL entry means that * the corresponding parity level is unavailable or cannot be * trusted for recovery. * * buffer_scratch Scratch buffers used to hold OLD RAID unknowns without * destroying certified CURRENT data. * * Return value: * * 0 A RAID solution was successfully validated under one of the two history * hypotheses. Some originally bad CHG/REP entries may still be marked * RECOVERY_OUTOFDATE, meaning that their required CURRENT contents could not * be proven. The caller must inspect the recovery status before considering such * blocks successfully recovered. * * >0 One or more recovery candidates were attempted but failed independent * validation. The value is the accumulated number of failed attempts. * * <0 No usable recovery strategy could be attempted successfully, for example * because there were too many failures or insufficient usable parity. * * On successful return buffer[] contains the recovered data and parity * recomputed from that data. */ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void** buffer_scratch) { int ret; int error; unsigned j, k; unsigned n; int something_to_recover; int something_unsynced; int too_many_unknowns; struct { unsigned j; void* buffer; } rep_saved[LEV_MAX]; unsigned scratch_count; error = 0; for (j = 0; j < failed_count; ++j) failed[j].recovery = RECOVERY_NONE; /* * Nothing is missing or damaged. * * repair() is also responsible for leaving buffer[] with parity * recomputed from the data buffers, so do that even if there is no * actual recovery to perform. */ if (failed_count == 0) { raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 0; } /* * Log all entries participating in recovery. * * failed[] contains more than just bad blocks. Readable CHG/REP and * DELETED entries may also be present because their OLD contribution can * be needed by the pre-sync recovery strategy below. */ for (j = 0; j < failed_count; ++j) { const char* desc; const char* hash; const char* data; struct snapraid_block* block = failed[j].block; unsigned block_state = block_state_get(block); switch (block_state) { case BLOCK_STATE_DELETED : desc = "delete"; break; case BLOCK_STATE_CHG : desc = "change"; break; case BLOCK_STATE_REP : desc = "replace"; break; case BLOCK_STATE_BLK : desc = "block"; break; case BLOCK_STATE_REBUILD : desc = "rebuild"; break; /* LCOV_EXCL_START */ default : desc = "unknown"; break; /* LCOV_EXCL_STOP */ } if (block == BLOCK_NULL) { hash = "none"; } else if (hash_is_invalid(block->hash)) { hash = "lost"; } else if (hash_is_zero(block->hash)) { hash = "zero"; } else { hash = "known"; } if (failed[j].is_bad) data = "bad"; else data = "good"; if (failed[j].file) { struct snapraid_disk* disk = failed[j].disk; struct snapraid_file* file = failed[j].file; block_off_t file_pos = failed[j].file_pos; log_tag("repair_entry:%u:%s:%s:%s:%s:%s:%" PRIu64 ":\n", j, desc, hash, data, disk->name, esc_tag(file->sub), file_pos); } else { log_tag("repair_entry:%u:%s:%s:%s:\n", j, desc, hash, data); } } /* * Under the recovery invariant documented above, an interrupted sync leaves * exactly two possible data generations for unsynchronized blocks: * * OLD the state identified by the pre-sync metadata; * CURRENT the unchanged live filesystem state. * * The content file tells us which blocks were not fully synchronized, but it * cannot tell us whether physical parity had already advanced from OLD to * CURRENT before the interruption. * * We therefore try the only two possible history hypotheses: * * 1. parity represents CURRENT data; * 2. parity still represents OLD data. * * Repeated interrupted sync attempts do not add another generation as long as * the filesystem remains unchanged between them. * * REBUILD does not introduce another data-generation hypothesis. Its data is * CURRENT in both cases; only the physical parity associated with it is * independently untrusted. * * If sync completed normally there are no CHG/REP/DELETED states and the * distinction disappears. */ /* * First strategy: parity represents CURRENT data. * * Under the CURRENT hypothesis: * * BLK hash CURRENT, parity normally synchronized * REP hash CURRENT, parity may represent OLD or CURRENT * REBUILD hash CURRENT, data CURRENT, physical parity untrusted * CHG hash OLD, parity may represent OLD or CURRENT * DELETED no CURRENT data, parity may represent OLD or zero * * Readable blocks already contain their CURRENT filesystem contents, so * only bad blocks have to be reconstructed. * * BLK/REP/REBUILD can first be fetched from import/search because their * stored hash identifies exactly the contents required by this strategy. * * CHG cannot use this shortcut because fetching its stored hash would * retrieve OLD contents. */ n = 0; something_to_recover = 0; for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad) { unsigned block_state = block_state_get(failed[j].block); assert(block_state != BLOCK_STATE_DELETED); if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP || block_state == BLOCK_STATE_REBUILD) && (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0 || state_search_fetch(state, rehash, failed[j].file, failed[j].file_pos, failed[j].block, buffer[failed[j].index]) == 0) ) { /* * The required CURRENT contents were obtained directly * using their CURRENT hash, so this block no longer * consumes a RAID recovery equation. * * Keep is_bad set because the file block still needs to be * written back. RECOVERY_CURRENT records that buffer[index] already * contains certified CURRENT data. */ failed[j].recovery = RECOVERY_CURRENT; log_tag("repair_hash_import:%u: Fixed by import\n", j); } else { /* * Otherwise reconstruct the block from parity. */ failed_map[n] = j; ++n; something_to_recover = 1; } } } /* * All originally bad blocks were recovered through import/search. */ if (!something_to_recover) { log_tag("recover_sync:%" PRIu64 ":%u: Skipped for already recovered\n", pos, n); raid_gen(diskmax, state->level, state->block_size, buffer, 0); return 0; } /* * Recover the selected bad blocks and independently validate the RAID * solution. * * repair_step() validates the reconstructed vector either using one or * more trustworthy CURRENT hashes, or, when no such hash exists, using * one additional independent parity equation. * * BLK, REP and REBUILD have CURRENT hashes and can therefore validate * their own recovered data directly. In particular, a REBUILD hash is * trustworthy even though the physical parity used to obtain the candidate * is not. * * Recovered CHGs require an additional history interpretation because * they have no stored CURRENT hash. Under the recovery invariant there are * only OLD and CURRENT generations, allowing their generation to be inferred * after the RAID solution itself has been validated. */ ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, 0); if (ret == 0) { int has_recovered_chg; int current_generation_proven; has_recovered_chg = 0; current_generation_proven = 0; /* * Determine the generation of all CHGs reconstructed by this RAID * solution as a group. * * This inference relies on the recovery invariant documented by repair(): * the filesystem has not changed since the interrupted sync. Therefore the * only possible CHG data generations are OLD and CURRENT, and the stored CHG * hash identifies OLD. * * Under this invariant: * * candidate != OLD * * proves that this particular candidate cannot belong to the OLD generation. * Since no third generation is possible, it therefore proves CURRENT. * * This would NOT be valid if the filesystem had been changed after the * interrupted sync. For example, deleting and later reallocating a parity * position may preserve a historical CHG/DELETED hash while physical parity * represents another generation. In such a history candidate != stored hash * would only prove that the candidate differs from that historical hash, not * that it is CURRENT. * * All CHGs reconstructed by this repair_step() belong to the same validated * RAID solution. Consequently, once one recovered CHG proves that the * solution is CURRENT, that generation classification applies to every CHG * reconstructed by the same solution. * * This is intentionally a solution-level decision rather than a block-level * one. * * For example: * * CHG A has an INVALID OLD hash; * CHG B has candidate != OLD. * * CHG B identifies the validated RAID solution as CURRENT, so CHG A is also * CURRENT despite being unable to make that determination by itself. * * Similarly: * * CHG A candidate == OLD; * CHG B candidate != OLD. * * CHG A may simply have identical OLD and CURRENT bytes. Matching OLD is * therefore not evidence that CHG A is out-of-date once another CHG has * identified the whole solution as CURRENT. * * REBUILD does not participate in this generation inference. Its recovered * contents may be independently proven CURRENT by its hash, but this only * validates REBUILD itself and says nothing about whether CHG parity belongs * to the OLD or CURRENT sync generation. * * If no recovered CHG distinguishes the solution from OLD, the RAID solution * remains valid but its generation is ambiguous. In that case all recovered * CHGs are marked RECOVERY_OUTOFDATE together. */ for (j = 0; j < failed_count; ++j) { unsigned block_state; if (!failed[j].is_bad) continue; block_state = block_state_get(failed[j].block); if (block_state != BLOCK_STATE_CHG) { /* * BLK, REP and REBUILD have CURRENT hashes, so their own * recovered contents can be validated directly and don't * need this OLD/CURRENT generation inference. * * REBUILD is deliberately not a generation discriminator: * its data is CURRENT independently from the state of its * physical parity. */ assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP || block_state == BLOCK_STATE_REBUILD); continue; } has_recovered_chg = 1; /* * With an INVALID OLD hash this CHG cannot tell us whether * the candidate differs from OLD. It neither proves CURRENT * nor disproves it. */ if (hash_is_invalid(failed[j].block->hash)) continue; if (hash_is_zero(failed[j].block->hash)) { /* * ZERO exactly identifies the OLD contribution under the * recovery invariant. * * A non-zero candidate therefore proves candidate != OLD and, * since no third generation is possible, identifies the solution * as CURRENT. * * A zero candidate is ambiguous because OLD and CURRENT * may legitimately contain the same zero block. */ if (membcmp(buffer[failed[j].index], 0, state->block_size) != 0) current_generation_proven = 1; } else { unsigned hash_kind; hash_kind = rehash ? state->prevhash : state->hash; /* * The stored CHG hash identifies OLD contents under the recovery invariant. * Generation detection must compare hash identity only and must not apply the * CURRENT logical-size padding requirement. * * When the hash covers the complete canonical RAID block (memhash_is_block()), * candidate identity is independent of logical size: a mismatch proves * candidate != OLD and therefore identifies this validated RAID solution as CURRENT. * * For hashes that depend on the logical size (Murmur3/Spooky2), the OLD * logical size is not available. Because a mismatch against CURRENT pos_size * cannot distinguish candidate != OLD from a change in file size, the * conservative approach keeps the generation ambiguous and does not treat * the mismatch as proof of CURRENT. */ if (memhash_is_block(hash_kind)) { size_t pos_size; pos_size = file_block_size(failed[j].file, failed[j].file_pos, state->block_size); if (block_hash_cmp(state, rehash, failed[j].block, pos_size, buffer[failed[j].index]) != 0) current_generation_proven = 1; } } } if (has_recovered_chg) { if (current_generation_proven) log_tag("repair_generation_current:%" PRIu64 ": Recovered CHG proves CURRENT generation\n", pos); else log_tag("repair_generation_unknown:%" PRIu64 ": Recovered CHG generation is ambiguous\n", pos); } /* mark all recovered blocks according to their proven generation */ for (j = 0; j < failed_count; ++j) { unsigned block_state; if (!failed[j].is_bad) continue; block_state = block_state_get(failed[j].block); if (block_state == BLOCK_STATE_CHG) { if (current_generation_proven) { /* * At least one recovered CHG distinguishes this validated * RAID solution from OLD. * * Under the recovery invariant this proves that the solution * is CURRENT. The CURRENT classification therefore applies to * every CHG reconstructed by the same solution, including CHGs * with an INVALID OLD hash or with OLD == CURRENT contents. */ failed[j].recovery = RECOVERY_CURRENT; } else { /* * The RAID solution itself was successfully validated, * but none of its recovered CHGs distinguishes CURRENT * from OLD. * * This does not prove that the bytes are actually old or * wrong. It only means that CURRENT cannot be proven. * * Preserve all recovered CHGs as best-effort data but * mark them out-of-date so the caller does not report * them as successfully recovered CURRENT contents. */ failed[j].recovery = RECOVERY_OUTOFDATE; } } else { /* BLK, REP and REBUILD have trustworthy CURRENT hashes */ failed[j].recovery = RECOVERY_CURRENT; } } return 0; } if (ret > 0) error += ret; if (ret < 0) log_tag("recover_sync:%" PRIu64 ":%u: Failed with no attempts\n", pos, n); else log_tag("recover_sync:%" PRIu64 ":%u: Failed with %d attempts\n", pos, n, ret); /* * Second strategy: parity still represents OLD data. * * The CURRENT attempt failed, so retry under the opposite history * hypothesis: physical parity still represents the state before the * interrupted sync. * * This interpretation also relies on the recovery invariant. Because no * filesystem changes occurred after the interruption, CHG/DELETED OLD * hashes still refer to the only pre-CURRENT generation that physical * parity may represent. * * Under this hypothesis: * * BLK OLD parity contains the synchronized data we want to * recover; * * REBUILD data and hash are CURRENT and unchanged by this history * choice. Physical parity remains independently untrusted; * * CHG parity contains OLD while the readable file contains * CURRENT; * * REP parity contains OLD while its stored hash describes * CURRENT; * * DELETED parity contains OLD contents that are no longer present * in the current filesystem. * * Every CHG/REP/DELETED contribution must therefore be converted to OLD * when possible, or reconstructed as an auxiliary RAID unknown. * * REBUILD does not require such conversion because it has no alternate OLD * data contribution associated with this state. If REBUILD itself is bad, * it is recovered as required CURRENT data and independently validated * through its CURRENT hash. * * The reconstructed OLD CHG/REP/DELETED values exist only to make recovery * of bad BLK/REBUILD data possible. They are not themselves considered a * successful recovery of CURRENT unsynchronized data. */ n = 0; something_to_recover = 0; something_unsynced = 0; scratch_count = 0; too_many_unknowns = 0; for (j = 0; j < failed_count; ++j) { unsigned block_state = block_state_get(failed[j].block); /* * Only CHG/REP/DELETED make the OLD strategy different from the * CURRENT one because they have different OLD and CURRENT data * contributions. * * REBUILD is deliberately excluded. Its data contribution is CURRENT * in both history hypotheses; only its physical parity is untrusted, * and repair_step() deals with that through independent validation. */ if (block_state == BLOCK_STATE_DELETED || block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_REP) { /* * There is at least one entry whose OLD and CURRENT parity * contributions may differ, making this history hypothesis * distinct from the CURRENT attempt. */ something_unsynced = 1; if (block_state == BLOCK_STATE_CHG && hash_is_zero(failed[j].block->hash) ) { /* * ZERO exactly identifies the OLD CHG contribution under * the recovery invariant. * * Supply it directly without consuming a RAID equation. * * For a readable CHG, is_bad is false, so this temporary * OLD buffer will not later be written back to the file. */ memset(buffer[failed[j].index], 0, state->block_size); } else if ((block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_DELETED) && hash_is_unique(failed[j].block->hash) && state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0) { /* * CHG and DELETED retain a hash identifying OLD contents * under the recovery invariant, so import can provide their * exact pre-sync contribution. * * REP cannot use this shortcut because its stored hash * identifies CURRENT/NEW contents instead. * * Again, readable auxiliary entries have is_bad == 0 and * this OLD value will not be written to the user file. */ } else { /* * Adding this OLD unknown would exceed the number of available parity * equations. No OLD recovery strategy can possibly use it, so stop before * consuming a scratch buffer or changing independently validated CURRENT data. */ if (n >= state->level) { too_many_unknowns = 1; break; } /* * If buffer[index] already contains CURRENT data validated by import/search, * swap it with a preallocated scratch buffer so that the OLD reconstruction * is solved into scratch without overwriting the certified CURRENT bytes. */ if (block_state == BLOCK_STATE_REP && failed[j].is_bad && failed[j].recovery == RECOVERY_CURRENT) { assert(scratch_count < state->level); rep_saved[scratch_count].j = j; rep_saved[scratch_count].buffer = buffer[failed[j].index]; buffer[failed[j].index] = buffer_scratch[scratch_count]; ++scratch_count; } /* * The required OLD contribution is unavailable directly, * so recover it as another RAID unknown. */ failed_map[n] = j; ++n; /* * Do not set something_to_recover. * * Reconstructing only OLD CHG/REP/DELETED values has no * user-visible purpose. They are useful only as auxiliary * state for recovering a BLK/REBUILD block. */ } } else if (failed[j].is_bad) { /* * All generation-dependent unsynchronized states were handled * above. * * A bad BLK is synchronized data represented by the pre-sync * parity. * * A bad REBUILD also belongs here because its required data and * hash are CURRENT and do not have a separate OLD version. Its * physical parity is untrusted, but repair_step() accepts the * candidate only after independent validation. * * If CURRENT recovery already obtained this BLK/REBUILD through * import/search, buffer[index] contains certified CURRENT data. * BLK and REBUILD have the same required data contribution under * both history hypotheses, so that buffer remains a known RAID * input and must not consume another recovery equation. * * Keep is_bad unchanged because the original file block still * needs to be written back after the overall recovery succeeds. */ assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REBUILD); if (failed[j].recovery != RECOVERY_CURRENT) { if (n >= state->level) { too_many_unknowns = 1; break; } something_to_recover = 1; failed_map[n] = j; ++n; } } } /* * Retry only when: * * - there is BLK/REBUILD data that we actually need to recover; and * * - there is some generation-dependent CHG/REP/DELETED state making * this strategy different from the CURRENT attempt. * * REBUILD alone does not justify this retry because changing from the * CURRENT to the OLD history hypothesis does not change its data input. * * Otherwise this strategy either has no useful CURRENT data to recover * or would just repeat the same recovery equations already attempted. */ if (!too_many_unknowns && something_to_recover && something_unsynced) { ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, 1); if (ret == 0) { /* * The RAID solution is valid under the OLD-history hypothesis. * * Recovered BLK data is useful CURRENT data because its * synchronized contents are represented by OLD parity. * * Recovered REBUILD data is also required CURRENT data, but for * a different reason: REBUILD has no alternate OLD data * generation and its candidate is independently validated by * its CURRENT hash despite its untrusted physical parity. */ /* * Restore the independently proven CURRENT bytes for REP blocks that * used a scratch buffer during the OLD strategy. */ for (k = 0; k < scratch_count; ++k) { unsigned idx = rep_saved[k].j; buffer[failed[idx].index] = rep_saved[k].buffer; failed[idx].recovery = RECOVERY_CURRENT; log_tag("repair_hash_import:%u: Restored CURRENT data after OLD retry\n", idx); } scratch_count = 0; /* * Mark all recovered blocks according to their proven generation. * Any bad CHG/REP reconstructed here without validated CURRENT data is * OLD-compatible auxiliary data and cannot be certified as CURRENT. * Bad BLK and REBUILD blocks are certified CURRENT by their hash. */ for (j = 0; j < failed_count; ++j) { unsigned block_state; /* * Skip blocks that do not need recovery, or blocks already validated as * CURRENT via import/search (including imported REPs restored above from * scratch buffers) to prevent demoting them to RECOVERY_OUTOFDATE. */ if (!failed[j].is_bad || failed[j].recovery == RECOVERY_CURRENT) continue; block_state = block_state_get(failed[j].block); if (block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_REP) { /* out-of-date entries cannot be certified CURRENT */ failed[j].recovery = RECOVERY_OUTOFDATE; log_tag("repair_hash_unknown:%u: Surely old data\n", j); } else { /* BLK and REBUILD have trustworthy CURRENT hashes */ failed[j].recovery = RECOVERY_CURRENT; } } return 0; } if (ret > 0) error += ret; if (ret < 0) log_tag("recover_unsync:%" PRIu64 ":%u: Failed with no attempts\n", pos, n); else log_tag("recover_unsync:%" PRIu64 ":%u: Failed with %d attempts\n", pos, n, ret); } else { if (too_many_unknowns) log_tag("recover_unsync:%" PRIu64 ":%u: Skipped for too many failures\n", pos, n + 1); else log_tag("recover_unsync:%" PRIu64 ":%u: Skipped for%s%s\n", pos, n, !something_to_recover ? " nothing to recover" : "", !something_unsynced ? " nothing unsynced" : ""); } /* restore original buffer pointers and state if OLD strategy failed or was skipped */ for (k = 0; k < scratch_count; ++k) { unsigned idx = rep_saved[k].j; buffer[failed[idx].index] = rep_saved[k].buffer; failed[idx].recovery = RECOVERY_CURRENT; } /* * Neither history hypothesis produced a usable RAID solution. * * Return the accumulated number of failed validation attempts when at * least one candidate was actually tried. Otherwise return -1 to indicate * that no usable recovery strategy was available. */ if (error) return error; else return -1; } /** * Post process all the files at the specified block index ::i. * For each file, if we are at the last block, close it, * adjust the timestamp, and print the result. * * This works only if the whole file is processed, including its last block. * This doesn't always happen, like with an explicit end block. * * In such case, the check/fix command won't report any information of the * files partially checked. */ static int file_post(struct snapraid_state* state, int fix, int partial, block_off_t i, struct snapraid_handle* handle, unsigned diskmax) { unsigned j; int ret; /* * For all the files print the final status, and do the final time fix * we also ensure to close files after processing the last block */ for (j = 0; j < diskmax; ++j) { struct snapraid_block* block; struct snapraid_disk* disk; struct snapraid_file* collide_file; struct snapraid_file* file; block_off_t file_pos; disk = handle[j].disk; if (!disk) { /* if no disk, nothing to do */ continue; } block = fs_par2block_find(disk, i); if (!block_has_file(block)) { /* if no file, nothing to do */ continue; } file = fs_par2file_get(disk, i, &file_pos); /* if it isn't the last block in the file */ if (!file_block_is_last(file, file_pos)) { /* nothing to do */ continue; } /* if the file is excluded, we have nothing to adjust as the file is never written */ if (file_flag_has(file, FILE_IS_EXCLUDED) || (state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED))) { /* nothing to do, but close the file */ goto close_and_continue; } /* finish the fix process if it's the last block of the files */ if (fix) { if (handle[j].file != file) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency in file handle '%s' instead of '%s'\n", handle[j].file != 0 ? handle[j].file->sub : "", file->sub); os_abort(); /* LCOV_EXCL_STOP */ } /* * A partial fix operates only at block level. Reaching the last block of a * file doesn't imply that the whole file was processed, so don't perform any * file-level finalization, including marking it finished, reporting its final * status, or restoring its timestamp. * This is intentional also when an explicit block range happens to cover all * the parity blocks, to keep -S/-B behavior independent of the parity size. */ if (partial) goto close_and_continue; /* * Mark that we finished with this file * to identify later any NOT finished ones */ file_flag_set(file, FILE_IS_FINISHED); /* if the file is damaged, meaning that a fix failed */ if (file_flag_has(file, FILE_IS_DAMAGED)) { log_tag("status:unrecoverable:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("unrecoverable %s\n", fmt_term(disk, file->sub)); /* and do not set the time if damaged */ goto close_and_continue; } /* if the file is not fixed, meaning that it is untouched */ if (!file_flag_has(file, FILE_IS_FIXED)) { /* nothing to do, but close the file */ goto close_and_continue; } /* * If the file is closed or different than the one expected, reopen it * a different open file could happen when filtering for bad blocks */ if (handle[j].file != file) { /* keep a pointer at the file we are going to close for error reporting */ struct snapraid_file* report = handle[j].file; ret = handle_close(&handle[j]); if (ret != 0) { /* LCOV_EXCL_START */ log_tag("%s::%s:%s: Close error. %s.\n", es(errno), disk->name, esc_tag(report->sub), strerror(errno)); log_fatal_errno(errno, disk->name); return -1; /* LCOV_EXCL_STOP */ } /* reopen the file for writing, as required to set the mtime on Windows */ ret = handle_open_create(&handle[j], file, state->file_mode, 0); if (ret != 0) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Open error. %s.\n", es(errno), i, disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); return -1; /* LCOV_EXCL_STOP */ } } /* search for the corresponding inode */ uint64_t inode = handle[j].st.st_ino; /* don't know the exact type of st_ino and we cannot pass it by pointer in the search */ if (inode != INODE_INVALID) collide_file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); else collide_file = 0; /* * If the inode is already in the database and it refers to a different file name, * we can fix the file time ONLY if the time and size allow to differentiate * between the two files * * For example, suppose we delete a bunch of files with all the same size and time, * when recreating them the inodes may be reused in a different order, * and at the next sync some files may have matching inode/size/time even if different name * not allowing sync to detect that the file is changed and not renamed */ if (!collide_file /* if not in the database, there is no collision */ || strcmp(collide_file->sub, file->sub) == 0 /* if the name is the same, it's the right collision */ || collide_file->size != file->size /* if the size is different, the collision is identified */ || collide_file->mtime_sec != file->mtime_sec /* if the mtime is different, the collision is identified */ || collide_file->mtime_nsec != file->mtime_nsec /* same for mtime_nsec */ ) { /* set the original modification time */ ret = handle_utime(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Time error. %s.\n", es(errno), i, disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); /* mark the file as damaged */ file_flag_set(file, FILE_IS_DAMAGED); return -1; /* LCOV_EXCL_STOP */ } } else { log_tag("collision:%s:%s:%s: Not setting modification time to avoid inode collision\n", disk->name, esc_tag(file->sub), esc_tag(collide_file->sub)); } /* * Close the normal file before removing the previous quarantine. * A delayed close error means that recovery is not yet successful, * and the previous .unrecoverable file must remain available. */ ret = handle_close(&handle[j]); if (ret != 0) { /* LCOV_EXCL_START */ log_tag("%s::%s:%s: Close error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); return -1; /* LCOV_EXCL_STOP */ } /* * The normal file has now been completely recovered, finalized, and closed. * Remove any quarantine left by an earlier failed recovery attempt. */ ret = handle_unrecoverable_remove(&handle[j], file); if (ret != 0) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Remove unrecoverable error. %s.\n", es(errno), i, disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); return -1; /* LCOV_EXCL_STOP */ } log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("recovered %s\n", fmt_term(disk, file->sub)); } else { /* * We are not fixing, but only checking * print just the final status */ if (file_flag_has(file, FILE_IS_DAMAGED)) { log_tag("status:unrecoverable:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("unrecoverable %s\n", fmt_term(disk, file->sub)); } else if (file_flag_has(file, FILE_IS_FIXED)) { log_tag("status:recoverable:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("recoverable %s\n", fmt_term(disk, file->sub)); } else { /* we don't use msg_verbose() because it also goes into the log */ if (msg_level >= MSG_VERBOSE) { log_tag("status:correct:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("correct %s\n", fmt_term(disk, file->sub)); } } } close_and_continue: /* * If the opened file is the correct one, close it * in case of excluded and fragmented files it's possible * that the opened file is not the current one */ if (handle[j].file == file) { /* * Ensure to close the file just after finishing with it * to avoid keeping it open without any possible use */ ret = handle_close(&handle[j]); if (ret != 0) { /* LCOV_EXCL_START */ log_tag("%s::%s:%s: Close error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); return -1; /* LCOV_EXCL_STOP */ } } } return 0; } /** * Check if we have to process the specified block index ::i. */ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct snapraid_handle* handle, unsigned diskmax) { unsigned j; unsigned l; /* filter for bad blocks */ if (state->opt.badblockonly) { snapraid_info info; /* get block specific info */ info = info_get(&state->infoarr, i); /* * Filter specifically only for bad blocks */ return info_get_bad(info); } /* filter for the parity */ if (state->opt.badfileonly) { snapraid_info info; /* get block specific info */ info = info_get(&state->infoarr, i); /* * If the block is bad, it has to be processed * * This is not necessary in normal cases because if a block is bad, * it necessary needs to have a file related to it, and files with * bad blocks are fully included. * * But some files may be excluded by additional filter options, * so it's not always true, and this ensures to always check all * the bad blocks. */ if (info_get_bad(info)) return 1; } else { /* if a parity is not excluded, include all blocks, even unused ones */ for (l = 0; l < state->level; ++l) { if (!state->parity[l].is_excluded_by_filter) { return 1; } } } /* filter for the files */ for (j = 0; j < diskmax; ++j) { struct snapraid_block* block; /* if no disk, nothing to check */ if (!handle[j].disk) continue; block = fs_par2block_find(handle[j].disk, i); /* * Try to recover all files, even the ones without hash * because in some cases we can recover also them */ if (block_has_file(block)) { struct snapraid_file* file = fs_par2file_get(handle[j].disk, i, 0); if (!file_flag_has(file, FILE_IS_EXCLUDED)) { /* only if the file is not filtered out */ return 1; } } } return 0; } /** * Write recovered data to files and update file status flags. */ static int file_recover(struct snapraid_state* state, int fix, int partial, block_off_t block_pos, struct failed_struct* failed, unsigned failed_count, void** buffer, unsigned* recovered_error) { unsigned j; for (j = 0; j < failed_count; ++j) { int ret; /* nothing to do if it doesn't need recovering */ if (!failed[j].is_bad) continue; /* nothing to do if it could not be recovered */ if (failed[j].recovery == RECOVERY_NONE) continue; if (!fix) { /* * In check mode, report as recoverable only blocks whose * recovered contents can be certified as the required CURRENT data. * * repair() may return success after validating the RAID solution while * leaving an unsynchronized bad block marked RECOVERY_OUTOFDATE. Such a block * is still unrecoverable as CURRENT data and must make the containing * file DAMAGED rather than FIXED. */ if (failed[j].recovery == RECOVERY_OUTOFDATE) file_flag_set(failed[j].file, FILE_IS_DAMAGED); else file_flag_set(failed[j].file, FILE_IS_FIXED); continue; } /* do not fix if the file is excluded */ if (file_flag_has(failed[j].file, FILE_IS_EXCLUDED) || (state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED))) continue; /* * Write the best recovered candidate even when it is out-of-date. Such data * may still be useful to the user or to a later multistep recovery. * * An out-of-date candidate is useful best-effort data, * but it cannot be certified as the required CURRENT * contents. Persist this fact before modifying the file * so that a SIGKILL cannot leave unproven recovered data * under the normal filename. */ if (failed[j].recovery == RECOVERY_OUTOFDATE) { file_flag_set(failed[j].file, FILE_IS_DAMAGED); /* * Quarantine is a file-level operation because it renames the whole file. * A partial -S/-B fix may still write the selected best-effort block, but it * must not rename or quarantine the complete file outside the requested range. */ if (!partial) { ret = handle_quarantine(failed[j].handle, failed[j].file, state->file_mode); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Quarantine error. %s.\n", es(errno), block_pos, failed[j].disk->name, esc_tag(failed[j].file->sub), strerror(errno)); log_fatal_errno(errno, failed[j].disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", block_pos); return -1; /* LCOV_EXCL_STOP */ } } } ret = handle_write(failed[j].handle, failed[j].file_pos, buffer[failed[j].index], state->block_size); if (ret == -1) { /* LCOV_EXCL_START */ int saved_errno = errno; /* mark the file as damaged */ file_flag_set(failed[j].file, FILE_IS_DAMAGED); /* * Quarantine is a file-level operation because it renames the whole file. * A partial -S/-B fix must leave the complete file under its normal name even * when a selected block write fails; FILE_IS_DAMAGED still records the failure. */ if (!partial) { ret = handle_quarantine(failed[j].handle, failed[j].file, state->file_mode); if (ret == -1) { log_tag("%s:%" PRIu64 ":%s:%s: Quarantine error. %s.\n", es(errno), block_pos, failed[j].disk->name, esc_tag(failed[j].file->sub), strerror(errno)); log_fatal_errno(errno, failed[j].disk->name); } } errno = saved_errno; log_tag("%s:%" PRIu64 ":%s:%s: Write error. %s.\n", es(errno), block_pos, failed[j].disk->name, esc_tag(failed[j].file->sub), strerror(errno)); log_fatal_errno(errno, failed[j].disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", block_pos); return -1; /* LCOV_EXCL_STOP */ } /* * A recovery write changes the file modification time. With syncedonly, * later recovery writes are allowed only while the file still matches * the metadata stored in the content file. Restore the original mtime * after every successful write so an interruption between blocks does * not make a partially recovered file appear unsynced on the next run. */ if (state->opt.syncedonly) { ret = handle_utime(failed[j].handle); if (ret == -1) { /* LCOV_EXCL_START */ file_flag_set(failed[j].file, FILE_IS_DAMAGED); log_tag("%s:%" PRIu64 ":%s:%s: Time error. %s.\n", es(errno), block_pos, failed[j].disk->name, esc_tag(failed[j].file->sub), strerror(errno)); log_fatal_errno(errno, failed[j].disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", block_pos); return -1; /* LCOV_EXCL_STOP */ } } /* if we are not sure that the recovered content is uptodate */ if (failed[j].recovery == RECOVERY_OUTOFDATE) continue; /* * Mark the file as containing some fixes * note that it could be also marked as damaged in other iterations */ file_flag_set(failed[j].file, FILE_IS_FIXED); log_tag("fixed:%" PRIu64 ":%s:%s: Fixed data error at position %" PRIu64 "\n", block_pos, failed[j].disk->name, esc_tag(failed[j].file->sub), failed[j].file_pos); ++*recovered_error; } return 0; } static int state_check_process(struct snapraid_state* state, int fix, struct snapraid_parity_handle** parity, block_off_t blockstart, block_off_t blockmax, int partial) { struct snapraid_handle* handle; unsigned diskmax; block_off_t i; unsigned j; void* buffer_alloc; void** buffer; void** buffer_scratch; unsigned buffermax; ssize_t ret; data_off_t countsize; block_off_t countpos; block_off_t countmax; unsigned soft_error; unsigned io_error; unsigned silent_error; unsigned unrecoverable_error; unsigned recovered_error; struct failed_struct* failed; unsigned* failed_map; unsigned l; bit_vect_t* block_enabled; struct snapraid_bw bw; handle = handle_mapping(state, &diskmax); /* initialize the bandwidth context */ bw_init(&bw, state->opt.bwlimit); /* share the bandwidth context with all handles */ for (j = 0; j < diskmax; ++j) handle[j].bw = &bw; for (j = 0; j < state->level; ++j) if (parity[j]) parity[j]->bw = &bw; /* we need 1 * data + 2 * parity + 1 * scratch */ buffermax = diskmax + 3 * state->level; buffer = malloc_nofail_vector_align(buffermax, state->block_size, &buffer_alloc); if (!state->opt.skip_self) mtest_vector(buffermax, state->block_size, buffer); buffer_scratch = buffer + diskmax + 2 * state->level; failed = nalloc_nofail(diskmax, sizeof(struct failed_struct)); failed_map = nalloc_nofail(diskmax, sizeof(unsigned)); soft_error = 0; io_error = 0; silent_error = 0; unrecoverable_error = 0; recovered_error = 0; msg_progress("Selecting...\n"); /* first count the number of blocks to process */ countmax = 0; block_enabled = calloc_nofail(1, bit_vect_size(blockmax)); /* preinitialize to 0 */ for (i = blockstart; i < blockmax; ++i) { if (!block_is_enabled(state, i, handle, diskmax)) continue; bit_vect_set(block_enabled, i); ++countmax; } if (fix) msg_progress("Fixing...\n"); else if (!state->opt.auditonly) msg_progress("Checking...\n"); else msg_progress("Hashing...\n"); /* check all the blocks in files */ countsize = 0; countpos = 0; int alert = state_progress_begin(state, blockstart, blockmax, countmax); if (alert > 0) goto end; if (alert < 0) goto bail; for (i = blockstart; i < blockmax; ++i) { unsigned failed_count; int valid_parity; int used_parity; snapraid_info info; int rehash; if (!bit_vect_test(block_enabled, i)) { /* continue with the next block */ continue; } /* * If we have valid parity, and it makes sense to check its content. * If we already know that the parity is invalid, we just read the file * but we don't report parity errors * Note that with auditonly, we anyway skip the full parity check, * because we also don't read it at all */ valid_parity = 1; /* if the parity is used by at least one file */ used_parity = 0; /* keep track of the number of failed blocks */ failed_count = 0; /* get block specific info */ info = info_get(&state->infoarr, i); /* if we have to use the old hash */ rehash = info_get_rehash(info); /* for each disk, process the block */ for (j = 0; j < diskmax; ++j) { ssize_t read_size; unsigned char hash[HASH_MAX]; struct snapraid_disk* disk; struct snapraid_block* block; struct snapraid_file* file; block_off_t file_pos; unsigned block_state; /* if the disk position is not used */ disk = handle[j].disk; if (!disk) { /* use an empty block */ memset(buffer[j], 0, state->block_size); continue; } /* if the disk block is not used */ block = fs_par2block_find(disk, i); if (block == BLOCK_NULL) { /* use an empty block */ memset(buffer[j], 0, state->block_size); continue; } /* get the state of the block */ block_state = block_state_get(block); /* if the parity is not valid */ if (block_has_invalid_parity(block)) { /* * Mark the parity as invalid, and don't try to check/fix it * because it will be recomputed at the next sync */ valid_parity = 0; /* follow */ } /* if the block is DELETED */ if (block_state == BLOCK_STATE_DELETED) { /* use an empty block */ memset(buffer[j], 0, state->block_size); /* * DELETED is not a file repair target. It is kept only because its OLD * contribution may still be present in parity and may be required by the * pre-sync recovery hypothesis. */ failed[failed_count].is_bad = 0; failed[failed_count].recovery = RECOVERY_NONE; failed[failed_count].index = j; failed[failed_count].block = block; failed[failed_count].disk = disk; failed[failed_count].file = 0; failed[failed_count].file_pos = 0; failed[failed_count].handle = 0; ++failed_count; continue; } /* here we are sure that the parity is used by a file */ used_parity = 1; /* get the file of this block */ file = fs_par2file_get(disk, i, &file_pos); /* if we are only hashing, we can skip excluded files and don't even read them */ if (state->opt.auditonly && file_flag_has(file, FILE_IS_EXCLUDED)) { /* * Use an empty block * in true, this is unnecessary, because we are not checking any parity * but we keep it for completeness */ memset(buffer[j], 0, state->block_size); continue; } /* if the file is closed or different than the current one */ if (handle[j].file == 0 || handle[j].file != file) { /* keep a pointer at the file we are going to close for error reporting */ struct snapraid_file* report = handle[j].file; ret = handle_close(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s::%s:%s: Close error. %s.\n", es(errno), disk->name, esc_tag(report->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* if fixing, and the file is not excluded, we must open for writing */ if (fix && !file_flag_has(file, FILE_IS_EXCLUDED)) { if (!state->opt.syncedonly) { /* ordinary fix: create the file, open for writing and resize if required */ ret = handle_open_create(&handle[j], file, state->file_mode, 1); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Create error. %s.\n", es(errno), i, disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* check if the file was just created */ if (handle[j].created != 0) { /* * If fragmented, it may be reopened, so remember that the file * was originally missing */ file_flag_set(file, FILE_IS_CREATED); } } else { /* in syncedonly mode, open for writing but do not create missing files */ if (!file_flag_has(file, FILE_IS_MISSING)) { ret = handle_open_create(&handle[j], file, state->file_mode, 0); } else { errno = ENOENT; ret = -1; } /* if missing, mark as unsynced and let common error handler record the missing block */ if (ret == -1 && errno == ENOENT) { file_flag_set(file, FILE_IS_UNSYNCED); } else if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Open error. %s.\n", es(errno), i, disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } } } else { /* open the file only for reading */ if (!file_flag_has(file, FILE_IS_MISSING)) { ret = handle_open(&handle[j], file, state->file_mode, state->opt.expected_missing ? log_expected : 0); } else { errno = ENOENT; ret = -1; /* if the file is missing, we cannot open it */ } } if (ret == -1) { /* save the failed block for the check/fix */ failed[failed_count].is_bad = 1; failed[failed_count].recovery = RECOVERY_NONE; failed[failed_count].index = j; failed[failed_count].block = block; failed[failed_count].disk = disk; failed[failed_count].file = file; failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; log_tag("%s:%" PRIu64 ":%s:%s: Open error at position %" PRIu64 ". %s.\n", es(errno), i, disk->name, esc_tag(file->sub), file_pos, strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } /* * Mark the file as missing, to avoid to retry to open it again * note that this can be done only if we are not fixing it * or if syncedonly is active so it won't be recreated */ file_flag_set(file, FILE_IS_MISSING); continue; } /* if it's the first open, and not excluded */ if (!file_flag_has(file, FILE_IS_OPENED) && !file_flag_has(file, FILE_IS_EXCLUDED)) { /* check if the file is changed */ if (handle[j].st.st_size != file->size || handle[j].st.st_mtime != file->mtime_sec || STAT_NSEC(&handle[j].st) != file->mtime_nsec /* don't check the inode to support file-system without persistent inodes */ ) { /* report that the file is not synced */ file_flag_set(file, FILE_IS_UNSYNCED); } } /* * Missing data inside a short file is handled block by block by normal * recovery writes, which extend the file as needed. * * Extra data past the expected end has no content block and therefore requires * a file-level truncate during a full fix. * * A partial -S/-B fix operates only at block level, so it must leave any * untracked tail untouched, even if the file is larger than expected. */ if (!file_flag_has(file, FILE_IS_OPENED) && !file_flag_has(file, FILE_IS_EXCLUDED) && !partial && !(state->opt.syncedonly && file_flag_has(file, FILE_IS_UNSYNCED)) && handle[j].st.st_size > file->size ) { log_error(ESOFT, "File '%s' is larger than expected.\n", handle[j].path); log_tag("error:%" PRIu64 ":%s:%s: Size error\n", i, disk->name, esc_tag(file->sub)); ++soft_error; if (fix) { ret = handle_truncate(&handle[j], file); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Truncate error. %s.\n", es(errno), i, disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } file_flag_set(file, FILE_IS_FIXED); log_tag("fixed:%" PRIu64 ":%s:%s: Fixed size\n", i, disk->name, esc_tag(file->sub)); ++recovered_error; } } /* * Mark the file as opened at least one time * this is used to avoid to check the unsynced and size * more than one time, in case the file is reopened later */ file_flag_set(file, FILE_IS_OPENED); } /* read from the file */ if (file_flag_has(file, FILE_IS_MISSING)) { /* if the file is reported missing, don't even try to read it */ errno = ENOENT; read_size = -1; } else { read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, state->opt.expected_missing ? log_expected : 0); } if (read_size == -1) { /* save the failed block for the check/fix */ failed[failed_count].is_bad = 1; /* it's bad because we cannot read it */ failed[failed_count].recovery = RECOVERY_NONE; failed[failed_count].index = j; failed[failed_count].block = block; failed[failed_count].disk = disk; failed[failed_count].file = file; failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; log_tag("%s:%" PRIu64 ":%s:%s: Read error at position %" PRIu64 ". %s.\n", es(errno), i, disk->name, esc_tag(file->sub), file_pos, strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } /* if we are reading at the end, mark the file as missing to avoid to try to read it again at the next block */ if (errno == ENOENT) { file_flag_set(file, FILE_IS_MISSING); } continue; } countsize += read_size; /* * Always insert CHG blocks, the repair functions needs all of them * because the parity may be still referring at the old state * and the repair must be aware of it */ if (block_state == BLOCK_STATE_CHG) { /* * A CHG block has no hash for the current data. The stored hash * refers to the previous content, and after an interrupted sync * parity may refer either to the old or to the new content. * * If the block is readable, preserve it and assume it is the best * available copy of the current data. We must not try to validate it * against block->hash, as a match would only identify the old data * and a mismatch would not prove that the current data is correct. * * This assumption applies only to data belonging to the current recovery * attempt. A CHG from a previous .unrecoverable file cannot safely be reused: * it may itself be an ambiguous recovery result whose CURRENT generation was * not proven. Once persisted, that provenance cannot be inferred from the * bytes alone. */ failed[failed_count].is_bad = 0; /* we assume the CHG block correct */ failed[failed_count].recovery = RECOVERY_NONE; failed[failed_count].index = j; failed[failed_count].block = block; failed[failed_count].disk = disk; failed[failed_count].file = file; failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; continue; } assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP || block_state == BLOCK_STATE_REBUILD); /* compute the hash of the block just read */ if (rehash) { memhash_block(state->prevhash, state->prevhashseed, hash, buffer[j], read_size, state->block_size); } else { memhash_block(state->hash, state->hashseed, hash, buffer[j], read_size, state->block_size); } /* compare the hash */ if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) { unsigned diff = memdiff(hash, block->hash, BLOCK_HASH_SIZE); /* save the failed block for the check/fix */ failed[failed_count].is_bad = 1; /* it's bad because the hash doesn't match */ failed[failed_count].recovery = RECOVERY_NONE; failed[failed_count].index = j; failed[failed_count].block = block; failed[failed_count].disk = disk; failed[failed_count].file = file; failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; log_tag("error_data:%" PRIu64 ":%s:%s: Data error at position %" PRIu64 ", diff hash bits %u/%zu\n", i, disk->name, esc_tag(file->sub), file_pos, diff, BLOCK_HASH_SIZE * 8); ++silent_error; continue; } /* * A readable REP has already passed its NEW/current hash check, so its current * data is known good. It must nevertheless participate in repair because an * interrupted sync may have left physical parity containing its OLD * contribution. * * Unlike CHG, the REP stored hash cannot identify that OLD contribution * because it describes the NEW data. */ if (block_state == BLOCK_STATE_REP) { failed[failed_count].is_bad = 0; /* it's not bad */ failed[failed_count].recovery = RECOVERY_NONE; failed[failed_count].index = j; failed[failed_count].block = block; failed[failed_count].disk = disk; failed[failed_count].file = file; failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; continue; } } /* now read and check the parity if requested */ if (!state->opt.auditonly) { void* buffer_recov[LEV_MAX]; /* buffers for parity read and not computed */ for (l = 0; l < state->level; ++l) buffer_recov[l] = buffer[diskmax + state->level + l]; for (; l < LEV_MAX; ++l) buffer_recov[l] = 0; /* read the parity */ for (l = 0; l < state->level; ++l) { if (parity[l]) { ret = parity_read(parity[l], i, buffer_recov[l], state->block_size); if (ret == -1) { log_tag("parity_%s:%" PRIu64 ":%s: Read error. %s.\n", es(errno), i, lev_config_name(l), strerror(errno)); /* * NULL means that this parity level cannot be trusted as an input to recovery. * The same marker is later used for a parity block that was read successfully * but failed comparison with recomputed parity. * * During fix, an accessible non-excluded parity marked this way is also the * one that needs to be rewritten from the recomputed value. */ buffer_recov[l] = 0; if (is_hw(errno)) { ++io_error; } else { ++soft_error; } } } else { buffer_recov[l] = 0; } } /* try all the recovering strategies */ ret = repair(state, rehash, i, diskmax, failed, failed_map, failed_count, buffer, buffer_recov, buffer_scratch); if (ret != 0) { /* increment the number of errors */ if (ret > 0) silent_error += ret; ++unrecoverable_error; /* print a list of all unrecoverable errors in files */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad && failed[j].recovery != RECOVERY_CURRENT) log_tag("unrecoverable:%" PRIu64 ":%s:%s: Unrecoverable error at position %" PRIu64 "\n", i, failed[j].disk->name, esc_tag(failed[j].file->sub), failed[j].file_pos); } /* update files that have blocks fixed by import/search */ if (file_recover(state, fix, partial, i, failed, failed_count, buffer, &recovered_error) != 0) { ++unrecoverable_error; goto bail; } /* keep track of damaged files for blocks that could not be recovered */ for (j = 0; j < failed_count; ++j) { if (!failed[j].is_bad || failed[j].recovery == RECOVERY_CURRENT) continue; file_flag_set(failed[j].file, FILE_IS_DAMAGED); /* * Persist the unrecoverable state immediately for a full fix. * Quarantine is a file-level operation because it renames the whole file. * A partial -S/-B fix operates only at block level, so keep FILE_IS_DAMAGED * but leave the file under its normal name when partial is set. */ if (fix && !file_flag_has(failed[j].file, FILE_IS_EXCLUDED) && !(state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED)) && !partial ) { ret = handle_quarantine(failed[j].handle, failed[j].file, state->file_mode); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("%s:%" PRIu64 ":%s:%s: Quarantine error. %s.\n", es(errno), i, failed[j].disk->name, esc_tag(failed[j].file->sub), strerror(errno)); log_fatal_errno(errno, failed[j].disk->name); log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } } } } else { /* * repair() returning success means that a RAID solution was validated under * one of the parity-history hypotheses. It does not necessarily mean that * every bad unsynced block was recovered to its CURRENT version. * * A bad block marked RECOVERY_OUTOFDATE is therefore still an unrecoverable current * data error even though the stripe-level RAID recovery succeeded. */ int partial_recover_error = 0; /* print a list of all the errors in files */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad && failed[j].recovery == RECOVERY_OUTOFDATE) { ++partial_recover_error; log_tag("unrecoverable:%" PRIu64 ":%s:%s: Unrecoverable unsynced error at position %" PRIu64 "\n", i, failed[j].disk->name, esc_tag(failed[j].file->sub), failed[j].file_pos); } } if (partial_recover_error != 0) { silent_error += partial_recover_error; ++unrecoverable_error; } /* * Check parities, but only if all the blocks have it computed and it's used. * * If you check/fix after a partial sync, it's OK to have parity errors * on the blocks with invalid parity and doesn't make sense to try to fix it. * * It's also OK to have data errors on unused parity, because sync doesn't * update it. */ if (used_parity && valid_parity) { /* check the parity */ for (l = 0; l < state->level; ++l) { if (buffer_recov[l] != 0 && memcmp(buffer_recov[l], buffer[diskmax + l], state->block_size) != 0) { unsigned diff = memdiff(buffer_recov[l], buffer[diskmax + l], state->block_size); /* mark that the read parity is wrong, setting ptr to 0 */ buffer_recov[l] = 0; log_tag("parity_error_data:%" PRIu64 ":%s: Data error, diff parity bits %u/%u\n", i, lev_config_name(l), diff, state->block_size * 8); ++silent_error; } } } /* update the fixed files */ if (file_recover(state, fix, partial, i, failed, failed_count, buffer, &recovered_error) != 0) { ++unrecoverable_error; goto bail; } if (fix) { /* * Update parity only if all the blocks have it computed and it's used. * * If you check/fix after a partial sync, you do not want to fix parity * for blocks that are going to have it computed in the sync completion. * * For unused parity there is no need to write or allocate it because no live * file depends on that position and its physical contents do not matter. */ if (used_parity && valid_parity) { /* update the parity */ for (l = 0; l < state->level; ++l) { /* if the parity on disk is wrong */ if (buffer_recov[l] == 0 /* and we have access at the parity */ && parity[l] != 0 /* and the parity is not excluded */ && !state->parity[l].is_excluded_by_filter ) { ret = parity_write(parity[l], i, buffer[diskmax + l], state->block_size, state->opt.skip_fallocate); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("parity_%s:%" PRIu64 ":%s: Write error. %s.\n", es(errno), i, lev_config_name(l), strerror(errno)); log_fatal_errno(errno, lev_config_name(l)); log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } log_tag("parity_fixed:%" PRIu64 ":%s: Fixed data error\n", i, lev_config_name(l)); ++recovered_error; } } } } } } else { /* * If we are not checking, we just set the DAMAGED flag * to report that the file is damaged, and we don't know if we can fix it */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad) { file_flag_set(failed[j].file, FILE_IS_DAMAGED); } } } /* post process the files */ ret = file_post(state, fix, partial, i, handle, diskmax); if (ret == -1) { /* LCOV_EXCL_START */ log_fatal(errno, "Stopping at block %" PRIu64 "\n", i); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* count the number of processed block */ ++countpos; /* progress */ if (state_progress(state, 0, i, countpos, countmax, countsize)) { /* LCOV_EXCL_START */ goto end; /* LCOV_EXCL_STOP */ } /* thermal control */ if (state_thermal_alarm(state)) { /* until now is misc */ state_usage_misc(state); state_progress_stop(state); state_thermal_cooldown(state); state_progress_restart(state); /* drop until now */ state_usage_waste(state); if (os_signal_interrupt()) goto end; } } /* * These objects have no parity block position, so an explicit -S/-B block * range cannot select them. Check/fix them separately after block processing, * including during an otherwise partial block-range operation. */ /* for each disk, recover empty files, symlinks and empty dirs */ for (i = 0; i < diskmax; ++i) { tommy_node* node; struct snapraid_disk* disk; if (!handle[i].disk) continue; /* for each empty file in the disk */ disk = handle[i].disk; node = disk->filelist; while (node) { char path[PATH_MAX]; struct stat st; struct snapraid_file* file; int unsuccessful = 0; int unrecoverable = 0; file = node->data; node = node->next; /* next node */ /* if not empty, it's already checked and continue to the next one */ if (file->size != 0) { continue; } /* if excluded continue to the next one */ if (file_flag_has(file, FILE_IS_EXCLUDED)) { continue; } /* lstat the file */ pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub); ret = lstat(path, &st); if (ret == -1) { unsuccessful = 1; if (errno != ENOENT) unrecoverable = 1; log_error(errno, "Error stating empty file '%s'. %s.\n", path, strerror(errno)); log_tag("empty_%s:%s:%s: Empty file stat error\n", es(errno), disk->name, esc_tag(file->sub)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } } else if (!S_ISREG(st.st_mode)) { unsuccessful = 1; unrecoverable = 1; log_error(ESOFT, "Error stating empty file '%s' for not regular file.\n", path); log_tag("empty_error:%s:%s: Empty file error for not regular file\n", disk->name, esc_tag(file->sub)); ++soft_error; #if HAVE_LSTAT_SYNC /* NTFS hard-link directory entries may report a stale size. */ } else if (lstat_sync(path, &st, 0) != 0) { unsuccessful = 1; unrecoverable = 1; log_error(errno, "Error stating empty file '%s'. %s.\n", path, strerror(errno)); log_tag("empty_%s:%s:%s: Empty file stat error\n", es(errno), disk->name, esc_tag(file->sub)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } #endif } else if (st.st_size != 0) { unsuccessful = 1; log_error(ESOFT, "Error stating empty file '%s' for not empty file.\n", path); log_tag("empty_error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc_tag(file->sub), (uint64_t)st.st_size); ++soft_error; } /* * In check mode a type mismatch is still potentially recoverable if the * conflicting object is removed manually. In fix mode, however, we don't * remove objects of a different type automatically. */ if (fix && unrecoverable) ++unrecoverable_error; if (fix && unsuccessful && !unrecoverable) { int f; /* create the ancestor directories */ ret = mkancestor(path); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error creating ancestor '%s%s'. %s.\n", disk->dir, file->sub, strerror(errno)); log_tag("empty_%s:%s:%s: Create ancestor error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* * Create it * O_NOFOLLOW: do not follow links to ensure to open the real file */ f = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_NOFOLLOW, 0600); if (f == -1) { /* LCOV_EXCL_START */ log_fatal(errno, "Error creating '%s%s'. %s.\n", disk->dir, file->sub, strerror(errno)); log_tag("empty_%s:%s:%s: Create error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* set the original modification time */ ret = fmtime(f, file->mtime_sec, file->mtime_nsec); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error timing '%s%s'. %s.\n", disk->dir, file->sub, strerror(errno)); log_tag("empty_%s:%s:%s: Time error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); close(f); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* close it */ ret = close(f); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error closing '%s%s'. %s.\n", disk->dir, file->sub, strerror(errno)); log_tag("empty_%s:%s:%s: Close error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } log_tag("empty_fixed:%s:%s: Fixed empty file\n", disk->name, esc_tag(file->sub)); ++recovered_error; log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(file->sub)); msg_info("recovered %s\n", fmt_term(disk, file->sub)); } } /* for each link in the disk */ disk = handle[i].disk; node = disk->linklist; while (node) { char path[PATH_MAX]; char pathto[PATH_MAX]; char linkto[PATH_MAX]; struct stat st; struct stat stto; struct snapraid_link* slink; int unsuccessful = 0; int unrecoverable = 0; slink = node->data; node = node->next; /* next node */ /* if excluded continue to the next one */ if (link_flag_has(slink, FILE_IS_EXCLUDED)) { continue; } if (link_flag_has(slink, FILE_IS_HARDLINK)) { /* stat the hardlink itself without following symbolic links */ pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub); ret = lstat(path, &st); if (ret == -1) { unsuccessful = 1; log_error(errno, "Error stating hardlink '%s'. %s.\n", path, strerror(errno)); log_tag("hardlink_%s:%s:%s:%s: Hardlink stat error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), esc_tag(slink->linkto), strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } } else if (!S_ISREG(st.st_mode)) { unsuccessful = 1; log_error(ESOFT, "Error stating hardlink '%s' for not regular file.\n", path); log_tag("hardlink_error:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc_tag(slink->sub), esc_tag(slink->linkto)); ++soft_error; #if HAVE_LSTAT_SYNC /* Windows lstat doesn't report the file identity required to validate hardlinks. */ } else if (lstat_sync(path, &st, 0) != 0) { unsuccessful = 1; log_error(errno, "Error stating hardlink '%s'. %s.\n", path, strerror(errno)); log_tag("hardlink_%s:%s:%s:%s: Hardlink stat error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), esc_tag(slink->linkto), strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } #endif } /* stat the target itself without following symbolic links */ pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, slink->linkto); ret = lstat(pathto, &stto); if (ret == -1) { unsuccessful = 1; if (errno == ENOENT) { unrecoverable = 1; if (fix) { /* * If the target doesn't exist, it's unrecoverable * because we cannot create an hardlink of a file that * doesn't exists * but in check, we can assume that fixing will recover * such missing file, so we assume a less drastic error */ ++unrecoverable_error; } } log_error(errno, "Error stating hardlink-to '%s'. %s.\n", pathto, strerror(errno)); log_tag("hardlink_%s:%s:%s:%s: Hardlink to stat error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), esc_tag(slink->linkto), strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } } else if (!S_ISREG(stto.st_mode)) { unsuccessful = 1; unrecoverable = 1; log_error(ESOFT, "Error stating hardlink-to '%s' for not regular file.\n", pathto); log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub), esc_tag(slink->linkto)); ++soft_error; if (fix) ++unrecoverable_error; #if HAVE_LSTAT_SYNC /* Windows lstat doesn't report the file identity required to validate hardlinks. */ } else if (lstat_sync(pathto, &stto, 0) != 0) { unsuccessful = 1; log_error(errno, "Error stating hardlink-to '%s'. %s.\n", pathto, strerror(errno)); log_tag("hardlink_%s:%s:%s:%s: Hardlink to stat error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), esc_tag(slink->linkto), strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } #endif } else if (!unsuccessful && st.st_ino != INODE_INVALID && stto.st_ino != INODE_INVALID && (st.st_dev != stto.st_dev || st.st_ino != stto.st_ino)) { unsuccessful = 1; log_error(ESOFT, "Mismatch hardlink '%s' and '%s'. Different file identity.\n", path, pathto); log_tag("hardlink_error:%s:%s:%s: Hardlink mismatch for different file identity\n", disk->name, esc_tag(slink->sub), esc_tag(slink->linkto)); ++soft_error; } } else { /* read the symlink */ pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub); ret = readlink(path, linkto, sizeof(linkto)); if (ret < 0) { unsuccessful = 1; log_error(errno, "Error reading symlink '%s'. %s.\n", path, strerror(errno)); log_tag("symlink_%s:%s:%s: Symlink read error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } } else if (ret >= PATH_MAX) { unsuccessful = 1; log_error(ESOFT, "Error reading symlink '%s'. Symlink too long.\n", path); log_tag("symlink_error:%s:%s: Symlink too long\n", disk->name, esc_tag(slink->sub)); ++soft_error; } else { linkto[ret] = 0; if (strcmp(linkto, slink->linkto) != 0) { unsuccessful = 1; log_tag("symlink_error:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc_tag(slink->sub), linkto, slink->linkto); ++soft_error; } } } if (fix && unsuccessful && !unrecoverable) { const char* link = link_flag_has(slink, FILE_IS_HARDLINK) ? "hard" : "sym"; /* create the ancestor directories */ ret = mkancestor(path); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error creating ancestor '%s%s'. %s.\n", disk->dir, slink->sub, strerror(errno)); log_tag("%slink_%s:%s:%s: Create ancestor error. %s.\n", link, es(errno), disk->name, esc_tag(slink->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* if it exists, it must be deleted before recreating */ ret = remove(path); if (ret != 0 && errno != ENOENT) { /* LCOV_EXCL_START */ log_fatal(errno, "Error removing '%s%s'. %s.\n", disk->dir, slink->sub, strerror(errno)); log_tag("%slink_%s:%s:%s: Remove error. %s.\n", link, es(errno), disk->name, esc_tag(slink->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* create it */ if (link_flag_has(slink, FILE_IS_HARDLINK)) { ret = hardlink(pathto, path); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error writing hardlink '%s' to '%s'. %s.\n", path, pathto, strerror(errno)); log_tag("hardlink_%s:%s:%s: Hardlink error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } log_tag("hardlink_fixed:%s:%s: Fixed hardlink error\n", disk->name, esc_tag(slink->sub)); ++recovered_error; } else { ret = symlink(slink->linkto, path); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error writing symlink '%s' to '%s'. %s.\n", path, slink->linkto, strerror(errno)); log_tag("symlink_%s:%s:%s: Hardlink error. %s.\n", es(errno), disk->name, esc_tag(slink->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } log_tag("symlink_fixed:%s:%s: Fixed symlink error\n", disk->name, esc_tag(slink->sub)); ++recovered_error; } log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(slink->sub)); msg_info("recovered %s\n", fmt_term(disk, slink->sub)); } } /* for each dir in the disk */ disk = handle[i].disk; node = disk->dirlist; while (node) { char path[PATH_MAX]; struct stat st; struct snapraid_dir* dir; int unsuccessful = 0; int unrecoverable = 0; dir = node->data; node = node->next; /* next node */ /* if excluded continue to the next one */ if (dir_flag_has(dir, FILE_IS_EXCLUDED)) { continue; } /* lstat the dir */ pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub); ret = lstat(path, &st); if (ret == -1) { unsuccessful = 1; if (errno != ENOENT) unrecoverable = 1; log_error(errno, "Error stating dir '%s'. %s.\n", path, strerror(errno)); log_tag("dir_%s:%s:%s: Dir stat error. %s.\n", es(errno), disk->name, esc_tag(dir->sub), strerror(errno)); if (is_hw(errno)) { ++io_error; } else { ++soft_error; } } else if (!S_ISDIR(st.st_mode)) { unsuccessful = 1; unrecoverable = 1; log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc_tag(dir->sub)); ++soft_error; } /* * In check mode a type mismatch is still potentially recoverable if the * conflicting object is removed manually. In fix mode, however, we don't * remove objects of a different type automatically. */ if (fix && unrecoverable) ++unrecoverable_error; if (fix && unsuccessful && !unrecoverable) { /* create the ancestor directories */ ret = mkancestor(path); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal(errno, "Error creating ancestor '%s%s'. %s.\n", disk->dir, dir->sub, strerror(errno)); log_tag("dir_%s:%s:%s: Create ancestor error. %s.\n", es(errno), disk->name, esc_tag(dir->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } /* create it */ ret = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); if (ret != 0 && errno != EEXIST) { /* LCOV_EXCL_START */ log_fatal(errno, "Error creating directory '%s%s'. %s.\n", disk->dir, dir->sub, strerror(errno)); log_tag("dir_%s:%s:%s: Create directory error. %s.\n", es(errno), disk->name, esc_tag(dir->sub), strerror(errno)); log_fatal_errno(errno, disk->name); log_fatal(errno, "Stopping\n"); ++unrecoverable_error; goto bail; /* LCOV_EXCL_STOP */ } log_tag("dir_fixed:%s:%s: Fixed dir error\n", disk->name, esc_tag(dir->sub)); ++recovered_error; log_tag("status:recovered:%s:%s\n", disk->name, esc_tag(dir->sub)); msg_info("recovered %s\n", fmt_term(disk, dir->sub)); } } } end: state_progress_end(state, countpos, countmax, countsize, "Nothing to check.\n"); bail: /* close all the files left open */ for (j = 0; j < diskmax; ++j) { struct snapraid_file* file = handle[j].file; struct snapraid_disk* disk = handle[j].disk; ret = handle_close(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ /* * If handle_close fails, the handle was open (f != -1), which * guarantees that both file and disk pointers are valid. */ log_tag("%s::%s:%s: Close error. %s.\n", es(errno), disk->name, esc_tag(file->sub), strerror(errno)); log_fatal_errno(errno, disk->name); ++unrecoverable_error; /* continue, as we are already exiting */ /* LCOV_EXCL_STOP */ } } if (soft_error || io_error || silent_error || recovered_error || unrecoverable_error) { msg_status("\n"); msg_status("%8u soft errors\n", soft_error); msg_status("%8u io errors\n", io_error); msg_status("%8u data errors\n", silent_error); if (fix) { msg_status("%8u recovered errors\n", recovered_error); } if (unrecoverable_error) { msg_status("%8u UNRECOVERABLE errors\n", unrecoverable_error); } else { /* without checking, we don't know if they are really recoverable or not */ if (!state->opt.auditonly) msg_status("%8u unrecoverable errors\n", unrecoverable_error); } } if ((soft_error || io_error || silent_error) && !fix) { if (soft_error) log_fatal(ESOFT, "WARNING! There are soft errors!\n"); if (io_error) log_fatal(EIO, "DANGER! Unexpected input/output errors!\n"); if (silent_error) log_fatal(EDATA, "DANGER! Unexpected silent data errors!\n"); } if (unrecoverable_error) log_fatal(ESOFT, "DANGER! Unrecoverable errors detected!\n"); log_tag("summary:error_soft:%u\n", soft_error); log_tag("summary:error_io:%u\n", io_error); log_tag("summary:error_data:%u\n", silent_error); if (fix) log_tag("summary:error_recovered:%u\n", recovered_error); if (!state->opt.auditonly) log_tag("summary:error_unrecoverable:%u\n", unrecoverable_error); if (fix) { if (soft_error + io_error + silent_error + recovered_error + unrecoverable_error == 0) log_tag("summary:exit:ok\n"); else if (unrecoverable_error == 0) log_tag("summary:exit:recovered\n"); else log_tag("summary:exit:unrecoverable\n"); } else if (!state->opt.auditonly) { if (soft_error + io_error + silent_error + unrecoverable_error == 0) log_tag("summary:exit:ok\n"); else if (unrecoverable_error == 0) log_tag("summary:exit:recoverable\n"); else log_tag("summary:exit:unrecoverable\n"); } else { /* audit only */ if (soft_error + silent_error + io_error == 0) log_tag("summary:exit:ok\n"); else if (silent_error + io_error == 0) log_tag("summary:exit:warning\n"); else log_tag("summary:exit:error\n"); } log_flush(); free(failed); free(failed_map); free(block_enabled); free(handle); free(buffer_alloc); free(buffer); bw_done(&bw); /* fail if some error are present after the run */ if (fix) { if (state->opt.expect_unrecoverable) { if (unrecoverable_error == 0) return -1; } else { if (unrecoverable_error != 0) return -1; } } else { if (state->opt.expect_unrecoverable) { if (unrecoverable_error == 0) return -1; } else if (state->opt.expect_recoverable) { if (unrecoverable_error != 0) return -1; if (soft_error + silent_error + io_error == 0) return -1; } else { if (unrecoverable_error != 0) return -1; if (soft_error + silent_error + io_error != 0) return -1; } } if (alert < 0) return -1; return 0; } int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount) { block_off_t blockmax; data_off_t size; int ret; int partial; struct snapraid_parity_handle parity[LEV_MAX]; struct snapraid_parity_handle* parity_ptr[LEV_MAX]; unsigned process_error; unsigned l; msg_progress("Initializing...\n"); /* * -S/--start and -B/--count allow check/fix to operate on an arbitrary * parity range instead of processing from the beginning. In particular, * recovery of a later section can be retried while an earlier section is * missing or damaged. * * A partial fix therefore does not establish that preceding or following * parity is valid; it covers only the requested range. * * Keep partial based on the explicit -S/-B request rather than the clipped * range: a nonzero -B remains partial even if it reaches the end of parity. */ partial = blockstart != 0 || blockcount != 0; /* * Fix is best-effort on each stripe. If a stripe cannot be recovered, it * is left unfixed and processing continues with the following stripes. * Consequently, repaired parity is not guaranteed to be contiguous: an * unrecoverable stripe may remain as a hole between successfully repaired * stripes. This is intentional, as stopping would prevent recovery of all * subsequent recoverable stripes. */ blockmax = parity_allocated_size(state); size = blockmax * state->block_size; if (blockstart > blockmax) { /* LCOV_EXCL_START */ log_fatal(EUSER, "Error in the specified starting block %" PRIu64 ". It's larger than the parity size %" PRIu64 ".\n", blockstart, blockmax); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } /* adjust the number of block to process */ if (blockcount != 0 && blockcount < blockmax - blockstart) { blockmax = blockstart + blockcount; } if (fix) { /* * If fixing, create the file and open for writing * if it fails, we cannot continue */ for (l = 0; l < state->level; ++l) { /* skip parity disks that are not accessible */ if (state->parity[l].skip_access) { parity_ptr[l] = 0; continue; } parity_ptr[l] = &parity[l]; /* if the parity is excluded */ if (state->parity[l].is_excluded_by_filter) { /* * Excluded means "do not modify", not "do not use". * * An excluded parity can still provide a recovery equation or the independent * parity check required when recovered data has no trustworthy hash, so keep * it available read-only whenever possible. */ ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); if (ret == -1) { log_tag("parity_%s:%u:%s: Open error. %s.\n", es(errno), 0, lev_config_name(l), strerror(errno)); if (is_hw(errno)) { log_fatal_errno(errno, lev_config_name(l)); exit(EXIT_FAILURE); } /* continue anyway */ parity_ptr[l] = 0; } } else { /* open for writing */ ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("parity_%s:%u:%s: Create error. %s.\n", es(errno), 0, lev_config_name(l), strerror(errno)); log_fatal_errno(errno, lev_config_name(l)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } /* * Fix restores the physical state described by the content file. * * For an existing parity level, preserve every persisted split boundary * while limiting the physical layout to the parity extent actually required * by the content. Preceding splits needed to reach an existing later split * are restored immediately; the remaining tail grows lazily as repaired * parity blocks are written. * * A new parity level has no persisted layout yet (PARITY_SIZE_INVALID), so * its split layout must instead be allocated from scratch. */ if (state->parity[l].split_map[0].size == PARITY_SIZE_INVALID) ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder); else ret = parity_restore(parity_ptr[l], size, state->block_size, state->opt.skip_fallocate); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("parity_%s:%u:%s: Create error. %s.\n", es(errno), 0, lev_config_name(l), strerror(errno)); log_fatal_errno(errno, lev_config_name(l)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } } } } else if (!state->opt.auditonly) { /* * If checking, open the file for reading * it may fail if the file doesn't exist, in this case we continue to check the files */ for (l = 0; l < state->level; ++l) { parity_ptr[l] = &parity[l]; ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); if (ret == -1) { log_tag("parity_%s:%u:%s: Open error. %s.\n", es(errno), 0, lev_config_name(l), strerror(errno)); if (is_hw(errno)) { log_fatal_errno(errno, lev_config_name(l)); exit(EXIT_FAILURE); } msg_status("No accessible %s file, only files will be checked.\n", lev_name(l)); /* continue anyway */ parity_ptr[l] = 0; } } } else { /* otherwise don't use any parity */ for (l = 0; l < state->level; ++l) parity_ptr[l] = 0; } process_error = 0; /* skip degenerated cases of empty parity, or skipping all */ if (blockstart < blockmax) { ret = state_check_process(state, fix, parity_ptr, blockstart, blockmax, partial); if (ret == -1) { /* LCOV_EXCL_START */ ++process_error; /* continue, as we are already exiting */ /* LCOV_EXCL_STOP */ } } /* try to close only if opened */ for (l = 0; l < state->level; ++l) { if (parity_ptr[l]) { ret = parity_close(parity_ptr[l]); if (ret == -1) { /* LCOV_EXCL_START */ log_tag("parity_%s::%s: Close error. %s.\n", es(errno), lev_config_name(l), strerror(errno)); log_fatal_errno(errno, lev_config_name(l)); ++process_error; /* continue, as we are already exiting */ /* LCOV_EXCL_STOP */ } } } if (process_error != 0) return -1; if (log_hardware_errors() != 0) { msg_status("Everything done, but hardware errors were encountered in the process\n"); return -1; } msg_status("Everything OK\n"); return 0; }