From b023939c6b16fd97d1d23651e9ad63ec4f5acffd Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 25 Sep 2026 23:40:58 +0200 Subject: [PATCH] Track current and previous hash generations for deallocated blocks --- cmdline/elem.c | 38 ++++++++++++++++++---------- cmdline/elem.h | 13 ++++++++-- cmdline/import.c | 33 ++++++++---------------- cmdline/rehash.c | 12 ++++++++- cmdline/scan.c | 2 +- cmdline/state.c | 66 +++++++++++++++++++++++++++++++++--------------- 6 files changed, 103 insertions(+), 61 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 0765dbf2..db7ed250 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -495,7 +495,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ file->mtime_nsec = mtime_nsec; file->inode = inode; file->flag = 0; - file->blockvec = nalloc_nofail((size_t)file->blockmax, block_sizeof()); + file->blockvec = nalloc_nofail(file->blockmax, block_sizeof()); return file; } @@ -522,7 +522,7 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) file->mtime_nsec = copy->mtime_nsec; file->inode = copy->inode; file->flag = copy->flag; - file->blockvec = nalloc_nofail((size_t)file->blockmax, block_sizeof()); + file->blockvec = nalloc_nofail(file->blockmax, block_sizeof()); for (i = 0; i < file->blockmax; ++i) { struct snapraid_block* block = file_block(file, i); @@ -873,39 +873,52 @@ struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, dat exit(EXIT_FAILURE); } - dealloc = malloc_nofail(sizeof(struct snapraid_dealloc)); + dealloc = malloc_nofail(sizeof(struct snapraid_dealloc) + blockmax * sizeof(struct snapraid_dealloc_block)); dealloc->sub = strdup_nofail(sub); dealloc->size = size; dealloc->mtime_sec = mtime_sec; dealloc->mtime_nsec = mtime_nsec; dealloc->blockmax = blockmax; - dealloc->blockhash = nalloc_nofail((size_t)dealloc->blockmax, BLOCK_HASH_SIZE); return dealloc; } -void dealloc_import(struct snapraid_dealloc* dealloc, struct snapraid_file* file) +void dealloc_import(tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_dealloc* dealloc, struct snapraid_file* file) { assert(dealloc->blockmax == file->blockmax); /* for all the blocks of the file */ for (block_off_t i = 0; i < dealloc->blockmax; ++i) { struct snapraid_block* block = fs_file2block_get(file, i); - unsigned char* hash = dealloc->blockhash + i * BLOCK_HASH_SIZE; + struct snapraid_dealloc_block* dealloc_block = &dealloc->block[i]; + unsigned block_state = block_state_get(block); - unsigned state = block_state_get(block); - switch (state) { + switch (block_state) { case BLOCK_STATE_BLK : case BLOCK_STATE_REP : - case BLOCK_STATE_REBUILD : - hash_copy(hash, block->hash); + case BLOCK_STATE_REBUILD : { + block_off_t parity_pos = fs_file2par_find(disk, file, i); + snapraid_info info; + + if (parity_pos == POS_NULL) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Internal inconsistency: Missing parity position for dealloc block %" PRIu64 "\n", i); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + info = info_get(infoarr, parity_pos); + dealloc_block->prev = info_get_rehash(info); + hash_copy(dealloc_block->hash, block->hash); break; + } case BLOCK_STATE_CHG : - hash_invalid_set(hash); + dealloc_block->prev = 0; + hash_invalid_set(dealloc_block->hash); break; default : /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "Internal inconsistency: State for dealloc block %" PRIu64 " state %u\n", i, state); + log_fatal(EINTERNAL, "Internal inconsistency: State for dealloc block %" PRIu64 " state %u\n", i, block_state); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -917,7 +930,6 @@ void dealloc_free(void* void_dealloc) struct snapraid_dealloc* dealloc = void_dealloc; free(dealloc->sub); - free(dealloc->blockhash); free(dealloc); } diff --git a/cmdline/elem.h b/cmdline/elem.h index f633c36d..0d9ba53c 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -368,6 +368,14 @@ struct snapraid_dir { tommy_hashdyn_node nodeset; }; +/** + * Deallocated block. + */ +struct snapraid_dealloc_block { + unsigned char prev; /**< If the hash uses the previous hash generation. */ + unsigned char hash[HASH_MAX]; /**< Hash of the block. Only BLOCK_HASH_SIZE bytes are significant. */ +}; + /** * Deallocated file. */ @@ -377,10 +385,11 @@ struct snapraid_dealloc { int64_t mtime_sec; /**< Modification time. */ int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or STAT_NSEC_INVALID if not present. */ block_off_t blockmax; /**< Number of blocks. */ - unsigned char* blockhash; /**< Hash of all blocks. */ /* nodes for data structures */ tommy_node nodelist; + + struct snapraid_dealloc_block block[]; /**< Blocks of the file. */ }; /** @@ -1126,7 +1135,7 @@ void dealloc_free(void* void_dealloc); /** * Import file hash in the dealloc. */ -void dealloc_import(struct snapraid_dealloc* dealloc, struct snapraid_file* file); +void dealloc_import(tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_dealloc* dealloc, struct snapraid_file* file); /** * Allocate a disk. diff --git a/cmdline/import.c b/cmdline/import.c index e7c8a9e4..02c1e41a 100644 --- a/cmdline/import.c +++ b/cmdline/import.c @@ -224,33 +224,20 @@ static void import_dealloc(struct snapraid_state* state, const char* dir, struct block->offset = offset; block->size = read_size; - hash_copy(block->hash, dealloc->blockhash + i * BLOCK_HASH_SIZE); + if (dealloc->block[i].prev) { + hash_invalid_set(block->hash); + hash_copy(block->prevhash, dealloc->block[i].hash); - /* do not insert invalid hashes */ - if (!hash_is_invalid(block->hash)) { - tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash)); - - /* if we are in a rehash state */ - if (state->prevhash != HASH_UNDEFINED) { - /* - * The deallocation record stores only one digest per block without - * recording which hash algorithm generated it. To avoid disk I/O at - * startup, index the stored digest under both algorithms. - * - * This is safe against false positives because state_import_fetch() - * validates candidate bytes with the requested algorithm upon read. - * - * Note that cross-generation matches (an old-hash deallocation supplying - * a new-hash block, or vice-versa) will miss in hash lookup and fall - * back to parity reconstruction. Only same-generation matches will succeed. - */ - hash_copy(block->prevhash, block->hash); + /* do not insert invalid hashes */ + if (!hash_is_invalid(block->prevhash)) tommy_hashdyn_insert(&state->previmportset, &block->prevnodeset, block, import_block_hash(block->prevhash)); - } else { - hash_invalid_set(block->prevhash); - } } else { + hash_copy(block->hash, dealloc->block[i].hash); hash_invalid_set(block->prevhash); + + /* do not insert invalid hashes */ + if (!hash_is_invalid(block->hash)) + tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash)); } offset += read_size; diff --git a/cmdline/rehash.c b/cmdline/rehash.c index 57ae1061..0c994b7c 100644 --- a/cmdline/rehash.c +++ b/cmdline/rehash.c @@ -18,6 +18,7 @@ void state_rehash(struct snapraid_state* state) { block_off_t blockmax; block_off_t i; + int has_pending_dealloc = 0; blockmax = parity_allocated_size(state); @@ -38,8 +39,17 @@ void state_rehash(struct snapraid_state* state) /* LCOV_EXCL_STOP */ } + /* deleted-only positions are not counted in unsynced_blocks, but their hashes still need the current generation */ + for (tommy_node* node = state->disklist; node != 0; node = node->next) { + struct snapraid_disk* disk = node->data; + if (!tommy_list_empty(&disk->dealloclist)) { + has_pending_dealloc = 1; + break; + } + } + /* a rehash can start only from a fully synchronized state */ - if (state->unsynced_blocks != 0) { + if (state->unsynced_blocks != 0 || has_pending_dealloc) { /* LCOV_EXCL_START */ log_tag("summary:exit:sync_required\n"); log_fatal(EUSER, "You cannot start a rehash with an incomplete sync.\n"); diff --git a/cmdline/scan.c b/cmdline/scan.c index 7f72aee6..8a54633b 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -616,7 +616,7 @@ static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* f if (keep_track) { struct snapraid_dealloc* dealloc = dealloc_alloc(scan->state->block_size, file->sub, file->size, file->mtime_sec, file->mtime_nsec); - dealloc_import(dealloc, file); + dealloc_import(&scan->state->infoarr, disk, dealloc, file); /* insert the dealloc in the dealloc containers */ tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc); diff --git a/cmdline/state.c b/cmdline/state.c index 1bece746..41ba951a 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -3075,11 +3075,31 @@ static void state_read_content(struct snapraid_state* state, const char* path, S log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", disk->name, esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec); - /* read all hashes */ + /* read all hashes with their generation */ for (block_off_t k = 0; k < dealloc->blockmax; ++k) { - unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE; + struct snapraid_dealloc_block* block = &dealloc->block[k]; - ret = sread(f, hash, BLOCK_HASH_SIZE); + c = sgetc(f); + if (c == 'c') { + block->prev = 0; + } else if (c == 'p') { + if (state->prevhash == HASH_UNDEFINED) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Previous dealloc hash without previous checksum!\n"); + os_abort(); + /* LCOV_EXCL_STOP */ + } + block->prev = 1; + } else { + /* LCOV_EXCL_START */ + decoding_error(path, f); + log_fatal(ECONTENT, "Invalid dealloc hash type!\n"); + os_abort(); + /* LCOV_EXCL_STOP */ + } + + ret = sread(f, block->hash, BLOCK_HASH_SIZE); if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); @@ -3889,7 +3909,7 @@ struct state_write_context { block_off_t blockmax; time_t info_oldest; time_t info_now; - int info_has_rehash; + int has_prevhash_data; uint64_t count_unsynced; STREAM* f; int first; @@ -3963,7 +3983,7 @@ static void* state_write_thread(void* arg) block_off_t blockmax = context->blockmax; time_t info_oldest = context->info_oldest; time_t info_now = context->info_now; - int info_has_rehash = context->info_has_rehash; + int has_prevhash_data = context->has_prevhash_data; STREAM* f = context->f; uint32_t crc; uint64_t t64; @@ -4035,10 +4055,9 @@ static void* state_write_thread(void* arg) /* LCOV_EXCL_STOP */ } - /* previous hash only present */ + /* previous hash is also needed by deallocated blocks after active rehash positions disappear */ if (state->prevhash != HASH_UNDEFINED) { - /* if at least one rehash tag found, we have to save the previous hash */ - if (info_has_rehash) { + if (has_prevhash_data) { sputc('C', f); if (state->prevhash == HASH_MURMUR3) { sputc('u', f); @@ -4393,8 +4412,13 @@ static void* state_write_thread(void* arg) if (context->first) log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", disk->name, esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec); - /* deallocated hashes are already contiguous in memory */ - swrite(dealloc->blockhash, (size_t)dealloc->blockmax * BLOCK_HASH_SIZE, f); + /* write all hashes with their generation */ + for (block_off_t k = 0; k < dealloc->blockmax; ++k) { + struct snapraid_dealloc_block* block = &dealloc->block[k]; + + sputc(block->prev ? 'p' : 'c', f); + swrite(block->hash, BLOCK_HASH_SIZE, f); + } } if (serror(f)) { @@ -4567,6 +4591,7 @@ static void state_write_content(struct snapraid_state* state, struct state_write time_t content_mtime; int info_has_rehash; int state_has_rebuild; + int has_dealloc; int mapping_idx; block_off_t idx; uint32_t crc; @@ -4669,22 +4694,21 @@ static void state_write_content(struct snapraid_state* state, struct state_write } int content_version = 3; + has_dealloc = 0; /* * Force at least version 3 as we want to always store the parity size. * If there is a REBUILD block or a dealloc list, force version 4. */ - if (state_has_rebuild) { - content_version = 4; - } else { - for (i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - if (!tommy_list_empty(&disk->dealloclist)) { - content_version = 4; - break; - } + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + if (!tommy_list_empty(&disk->dealloclist)) { + has_dealloc = 1; + break; } } + if (state_has_rebuild || has_dealloc) + content_version = 4; fs_single_thread(state, 0); @@ -4735,7 +4759,7 @@ static void state_write_content(struct snapraid_state* state, struct state_write context->blockmax = blockmax; context->info_oldest = info_oldest; context->info_now = info_now; - context->info_has_rehash = info_has_rehash; + context->has_prevhash_data = info_has_rehash || has_dealloc; context->count_unsynced = count_unsynced; context->f = f; context->first = first; @@ -4914,7 +4938,7 @@ static void state_write_content(struct snapraid_state* state, struct state_write context->blockmax = blockmax; context->info_oldest = info_oldest; context->info_now = info_now; - context->info_has_rehash = info_has_rehash; + context->has_prevhash_data = info_has_rehash || has_dealloc; context->count_unsynced = count_unsynced; context->f = f; context->first = 1;