rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Track deallocated files in the content file for parity recovery

Browse source 
Add a new 'dealloc' list per disk to record files that were removed
or modified during a scan. This list is saved in the content file
(new format version SNAPCNT4) so that past parity contributions of
those files can be imported from previous snapshots.
This commit is contained in:
Andrea Mazzoleni 2026-04-27 14:39:38 +02:00
parent e9d4b0152a
commit f413e96ec4
9 changed files with 417 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

51
cmdline/elem.c
View file

@ -748,6 +748,55 @@ int dir_name_compare(const void* void_arg, const void* void_data)
return strcmp(arg, dir->sub);
}
struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec)
{
struct snapraid_dealloc* dealloc;
dealloc = malloc_nofail(sizeof(struct snapraid_dealloc));
dealloc->sub = strdup_nofail(sub);
dealloc->size = size;
dealloc->mtime_sec = mtime_sec;
dealloc->mtime_nsec = mtime_nsec;
dealloc->blockmax = (size + block_size - 1) / block_size;
dealloc->blockhash = malloc_nofail(dealloc->blockmax * BLOCK_HASH_SIZE);
return dealloc;
}
void dealloc_import(struct snapraid_dealloc* dealloc, struct snapraid_file* file)
{
assert(dealloc->blockmax == file->blockmax);
/* for all the blocks of the file */
for (block_off_t i = 0; i < dealloc->blockmax; ++i) {
struct snapraid_block* block = fs_file2block_get(file, i);
unsigned char* hash = dealloc->blockhash + i * BLOCK_HASH_SIZE;
unsigned state = block_state_get(block);
switch (state) {
case BLOCK_STATE_BLK :
case BLOCK_STATE_REP :
memcpy(hash, block->hash, BLOCK_HASH_SIZE);
break;
case BLOCK_STATE_CHG :
hash_invalid_set(hash);
break;
default :
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: State for dealloc block %u state %u\n", i, state);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
void dealloc_free(struct snapraid_dealloc* dealloc)
{
free(dealloc->sub);
free(dealloc->blockhash);
free(dealloc);
}
struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip_access)
{
struct snapraid_disk* disk;
@ -799,6 +848,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
tommy_hashdyn_init(&disk->linkset);
tommy_list_init(&disk->dirlist);
tommy_hashdyn_init(&disk->dirset);
tommy_list_init(&disk->dealloclist);
tommy_tree_init(&disk->fs_parity, extent_parity_compare);
tommy_tree_init(&disk->fs_file, extent_file_compare);
disk->fs_last = 0;
@ -818,6 +868,7 @@ void disk_free(struct snapraid_disk* disk)
tommy_hashdyn_done(&disk->linkset);
tommy_list_foreach(&disk->dirlist, (tommy_foreach_func*)dir_free);
tommy_hashdyn_done(&disk->dirset);
tommy_list_foreach(&disk->dealloclist, (tommy_foreach_func*)dealloc_free);
#if HAVE_THREAD
thread_mutex_destroy(&disk->fs_mutex);

59
cmdline/elem.h
View file

@ -308,6 +308,21 @@ struct snapraid_dir {
tommy_hashdyn_node nodeset;
};
/**
* Deallocated file.
*/
struct snapraid_dealloc {
char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
data_off_t size; /**< Size of the file. */
int64_t mtime_sec; /**< Modification time. */
int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or STAT_NSEC_INVALID if not present. */
block_off_t blockmax; /**< Number of blocks. */
unsigned char* blockhash; /**< Hash of all blocks. */
/* nodes for data structures */
tommy_node nodelist;
};
/**
* Chunk.
*
@ -322,7 +337,6 @@ struct snapraid_extent {
tommy_tree_node file_node; /**< Tree sorter by <file,file_pos>. */
};
/**
* Other disk.
*/
@ -346,9 +360,30 @@ struct snapraid_extra {
*/
struct snapraid_disk {
char name[PATH_MAX]; /**< Name of the disk. */
char mount_point[PATH_MAX]; /**< Configured mount point of the disk. It always terminates with /. */
char dir[PATH_MAX]; /**< Effective mount point of the disk. It could be either the real mount point or a snapshot. It always terminates with /. */
char snapshot_root[PATH_MAX]; /**< Subvolume root directory. It always terminates with /. Empty if not supported. */
/**
* Configured mount point of the disk.
*
* It always terminates with /
*/
char mount_point[PATH_MAX];
/**
* Effective mount point of the disk.
*
* It could be either the real mount point or a snapshot.
* It always terminates with /
*/
char dir[PATH_MAX];
/**
* Subvolumes root directory.
*
* Empty if disabled.
* It always terminates with /
*/
char snapshot_root[PATH_MAX];
char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */
char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */
@ -439,6 +474,7 @@ struct snapraid_disk {
tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
tommy_list dirlist; /**< List of all the empty dirs. */
tommy_hashdyn dirset; /**< Hashtable by name of all the empty dirs. */
tommy_list dealloclist; /**< List of all the deallocs. */
/* nodes for data structures */
tommy_node node;
@ -1006,6 +1042,21 @@ static inline tommy_uint32_t dir_name_hash(const char* name)
return tommy_hash_u32(0, name, strlen(name));
}
/**
* Allocate a dealloc.
*/
struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec);
/**
* Deallocate a dealloc.
*/
void dealloc_free(struct snapraid_dealloc* dealloc);
/**
* Import file hash in the dealloc.
*/
void dealloc_import(struct snapraid_dealloc* dealloc, struct snapraid_file* file);
/**
* Allocate a disk.
*/

76
cmdline/import.c
View file

@ -55,6 +55,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
file->is_runtime = 1;
buffer = malloc_nofail(block_size);
@ -126,6 +127,52 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
free(buffer);
}
static void import_dealloc(struct snapraid_state* state, const char* dir, struct snapraid_dealloc* dealloc)
{
char path[PATH_MAX];
struct snapraid_import_file* file;
block_off_t i;
data_off_t offset;
data_off_t size;
unsigned block_size = state->block_size;
pathcpy(path, sizeof(path), dir);
pathcat(path, sizeof(path), dealloc->sub);
size = dealloc->size;
file = malloc_nofail(sizeof(struct snapraid_import_file));
file->path = strdup_nofail(path);
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
file->is_runtime = 0;
offset = 0;
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_import_block* block = &file->blockimp[i];
unsigned read_size = block_size;
if (read_size > size)
read_size = size;
block->file = file;
block->offset = offset;
block->size = read_size;
memcpy(block->hash, dealloc->blockhash + i * BLOCK_HASH_SIZE, BLOCK_HASH_SIZE);
hash_invalid_set(block->prevhash);
/* do not insert invalid hashes */
if (!hash_is_invalid(block->hash))
tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash));
/* do not insert prevhash */
offset += read_size;
size -= read_size;
}
tommy_list_insert_tail(&state->importlist, &file->nodelist, file);
}
void import_file_free(struct snapraid_import_file* file)
{
free(file->path);
@ -157,6 +204,12 @@ int state_import_fetch(struct snapraid_state* state, int rehash, struct snapraid
f = open(path, O_RDONLY | O_BINARY);
if (f == -1) {
/* if no file, just skip it */
if (errno == ENOENT && !block->file->is_runtime) {
log_error(EUSER, "WARNING! Unexpected missing deallocated file '%s'.\n", path);
return -1;
}
/* LCOV_EXCL_START */
if (errno == ENOENT) {
log_fatal(errno, "DANGER! file '%s' disappeared.\n", path);
@ -196,6 +249,12 @@ int state_import_fetch(struct snapraid_state* state, int rehash, struct snapraid
memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size);
if (memcmp(buffer_hash, hash, BLOCK_HASH_SIZE) != 0) {
/* if hash mismatch, skip it */
if (!block->file->is_runtime) {
log_error(EUSER, "WARNING! Unexpected hash mismatch from deallocated file '%s'.\n", path);
return -1;
}
/* LCOV_EXCL_START */
log_fatal(EUSER, "Mismatch in data reading file '%s'.\n", path);
log_fatal(EUSER, "Please don't change imported files while running.\n");
@ -310,3 +369,20 @@ void state_import(struct snapraid_state* state, const char* dir)
import_dir(state, path);
}
void state_dealloc(struct snapraid_state* state, const char* dir, tommy_list* dealloclist)
{
/* snapshot should be enabled */
if (!state->snapshot)
return;
/* the hash must be full */
if (BLOCK_HASH_SIZE != HASH_MAX)
return;
for (tommy_node* i = tommy_list_head(dealloclist); i != 0; i = i->next) {
struct snapraid_dealloc* dealloc = i->data;
import_dealloc(state, dir, dealloc);
}
}

6
cmdline/import.h
View file

@ -35,6 +35,7 @@ struct snapraid_import_file {
struct snapraid_import_block* blockimp; /**< All the blocks of the file. */
block_off_t blockmax; /**< Number of blocks. */
char* path; /**< Full path of the file. */
int is_runtime; /**< If the hash was obtained at runtime */
/* nodes for data structures */
tommy_node nodelist;
@ -56,5 +57,10 @@ int state_import_fetch(struct snapraid_state* state, int prevhash, struct snapra
*/
void state_import(struct snapraid_state* state, const char* dir);
/**
* Import files from the dealloc list.
*/
void state_dealloc(struct snapraid_state* state, const char* dir, tommy_list* dealloclist);
#endif

22
cmdline/scan.c
View file

@ -650,7 +650,7 @@ static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* f
*
* File is then deleted.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file, int keep_track)
{
struct snapraid_disk* disk = scan->disk;
@ -659,6 +659,20 @@ static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* f
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
/*
* Keep track of the removed file (that won't be added later)
*
* This allows to record from where the parity was computed.
*/
if (keep_track) {
struct snapraid_dealloc* dealloc = dealloc_alloc(scan->state->block_size, file->sub, file->size, file->mtime_sec, file->mtime_nsec);
dealloc_import(dealloc, file);
/* insert the dealloc in the dealloc containers */
tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc);
}
stamp_lock(disk);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
stamp_unlock(disk);
@ -687,7 +701,7 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* fil
struct snapraid_file* copy = file_dup(file);
/* remove the file */
scan_file_remove(scan, file);
scan_file_remove(scan, file, 0);
/* reinsert the copy in the delayed list */
scan_file_insert(scan, copy);
@ -1014,7 +1028,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
is_original_file_size_different_than_zero = file->size != 0;
/* remove it, and continue to insert it again */
scan_file_remove(scan, file);
scan_file_remove(scan, file, 1);
/* and continue to insert it again */
} else {
@ -1790,7 +1804,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
}
}
scan_file_remove(scan, file);
scan_file_remove(scan, file, 1);
}
}

191
cmdline/state.c
View file

@ -2157,10 +2157,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
* The previous 'P' entry is now deprecated, but supported for importing.
* - SNAPCNT4/SnapRAID 15.0 Adds entry 'd' for dealloc file.
*/
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT4\n\3\0\0", 12) != 0
) {
/* LCOV_EXCL_START */
if (memcmp(buffer, "SNAPCNT", 7) != 0) {
@ -2595,6 +2597,105 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* LCOV_EXCL_STOP */
}
}
} else if (c == 'd') {
/* dealloc */
uint32_t v_count;
uint32_t v_pos;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Dealloc mapping index out of range\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
ret = sgetb32(f, &v_count);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
log_tag("content_info:dealloc:%s:%u\n", esc_tag(disk->name), v_count);
for (v_pos = 0; v_pos < v_count; ++v_pos) {
char sub[PATH_MAX];
uint64_t v_size;
uint64_t v_mtime_sec;
uint32_t v_mtime_nsec;
ret = sgetbs(f, sub, sizeof(sub));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*sub) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Null dealloc!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_mtime_sec);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_mtime_nsec);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* STAT_NSEC_INVALID is encoded as 0 */
if (v_mtime_nsec == 0)
v_mtime_nsec = STAT_NSEC_INVALID;
else
--v_mtime_nsec;
/* allocate the file */
struct snapraid_dealloc* dealloc = dealloc_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec);
log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", esc_tag(disk->name), esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec);
/* read all hashes */
for (block_off_t k = 0; k < dealloc->blockmax; ++k) {
unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE;
ret = sread(f, hash, BLOCK_HASH_SIZE);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
}
/* insert the dealloc in the dealloc containers */
tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc);
}
} else if (c == 's') {
/* symlink */
char sub[PATH_MAX];
@ -3358,9 +3459,22 @@ static void* state_write_thread(void* arg)
count_unscrubbed = 0;
tommy_hashdyn_init(&bucket_hash);
/* force version 3 a we want to always store the parity size */
/*
* Force at least version 3 a we want to always store the parity size
*
* If there is a dealloc list, force version 4
*/
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (!tommy_list_empty(&disk->dealloclist))
break;
}
/* write header */
swrite("SNAPCNT3\n\3\0\0", 12, f);
if (i == 0)
swrite("SNAPCNT3\n\3\0\0", 12, f);
else
swrite("SNAPCNT4\n\3\0\0", 12, f);
/* write block size and block max */
sputc('z', f);
@ -3706,6 +3820,49 @@ static void* state_write_thread(void* arg)
/* LCOV_EXCL_STOP */
}
}
/* deallocated files of the disk */
if (!tommy_list_empty(&disk->dealloclist)) {
sputc('d', f);
sputb32(disk->mapping_idx, f);
uint32_t v_count = tommy_list_count(&disk->dealloclist);
sputb32(v_count, f);
log_tag("content_info:dealloc:%s:%u\n", esc_tag(disk->name), v_count);
/* for each file */
for (j = tommy_list_head(&disk->dealloclist); j != 0; j = j->next) {
struct snapraid_dealloc* dealloc = j->data;
sputbs(dealloc->sub, f);
sputb64(dealloc->size, f);
sputb64(dealloc->mtime_sec, f);
/* encode STAT_NSEC_INVALID as 0 */
if (dealloc->mtime_nsec == STAT_NSEC_INVALID)
sputb32(0, f);
else
sputb32(dealloc->mtime_nsec + 1, f);
log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", esc_tag(disk->name), esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec);
/* write all hashes */
for (block_off_t k = 0; k < dealloc->blockmax; ++k) {
unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE;
swrite(hash, BLOCK_HASH_SIZE, f);
}
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
} else {
log_tag("content_info:dealloc:%s:0\n", esc_tag(disk->name));
}
}
/* write the info for each block */
@ -4650,6 +4807,20 @@ void state_write(struct snapraid_state* state)
state->written = 1;
}
void state_commit(struct snapraid_state* state)
{
tommy_node* i;
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
/* clear the dealloc list */
tommy_list_foreach(&disk->dealloclist, (tommy_foreach_func*)dealloc_free);
tommy_list_init(&disk->dealloclist);
}
}
void state_skip(struct snapraid_state* state)
{
tommy_node* i;
@ -5841,6 +6012,22 @@ void state_snapshot_write(struct snapraid_state* state, tommy_list* filterlist_d
pathcpy(disk->dir, sizeof(disk->dir), vol);
pathcat(disk->dir, sizeof(disk->dir), "/");
pathcat(disk->dir, sizeof(disk->dir), disk->mount_point + root_len);
/* if there is a dealloc list */
if (!tommy_list_empty(&disk->dealloclist)) {
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE);
/* if there is a previous snapshot */
if (is_dir(vol)) {
pathcat(vol, sizeof(vol), "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
msg_progress("Importing disk %s stable snapshot deallocated files...\n", disk->name);
state_dealloc(state, vol, &disk->dealloclist);
}
}
} else {
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE);

5
cmdline/state.h
View file

@ -231,6 +231,11 @@ void state_read(struct snapraid_state* state);
*/
void state_write(struct snapraid_state* state);
/**
* Signal that we reached a stable state with all parity computed.
*/
void state_commit(struct snapraid_state* state);
/**
* Diff all the disks.
*/

14
cmdline/status.c
View file

@ -51,6 +51,7 @@ int state_status(struct snapraid_state* state)
uint64_t file_size;
uint64_t file_block_count;
uint64_t file_block_free;
unsigned dealloc_count;
block_off_t parity_block_free;
uint64_t all_wasted;
int free_not_zero;
@ -84,6 +85,7 @@ int state_status(struct snapraid_state* state)
printf(" Files Fragments GB GB GB\n");
/* count fragments */
dealloc_count = 0;
file_count = 0;
file_size = 0;
file_block_count = 0;
@ -110,6 +112,8 @@ int state_status(struct snapraid_state* state)
uint64_t disk_free_bytes;
int64_t wasted;
dealloc_count += tommy_list_count(&disk->dealloclist);
/* for each file in the disk */
node = disk->filelist;
while (node) {
@ -377,6 +381,16 @@ int state_status(struct snapraid_state* state)
if (state->unsynced_blocks) {
printf("WARNING! The array is NOT fully synced.\n");
printf("You have a sync in progress at %u%%.\n", muldiv(blockmax - state->unsynced_blocks, 100, blockmax));
if (dealloc_count) {
printf("WARNING! There are %u files updated or deleted from the array that may reduce the recovery probability until the next sync.\n", dealloc_count);
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
for (tommy_node* node = tommy_list_head(&disk->dealloclist); node != 0; node = node->next) {
struct snapraid_dealloc* dealloc = node->data;
msg_verbose("\t%s\n", fmt_term(disk, dealloc->sub));
}
}
}
} else {
printf("No sync is in progress.\n");
}

3
cmdline/sync.c
View file

@ -1428,6 +1428,9 @@ end:
/* LCOV_EXCL_STOP */
}
/* now the parity is fully written, no need to keep deallocated files */
state_commit(state);
/* save the new state if required */
if (!state->opt.kill_after_sync) {
if ((state->need_write || state->opt.force_content_write))