mirror of
https://github.com/amadvance/snapraid.git
synced 2026-11-19 14:36:38 -05:00
Track deallocated files in the content file for parity recovery
Add a new 'dealloc' list per disk to record files that were removed or modified during a scan. This list is saved in the content file (new format version SNAPCNT4) so that past parity contributions of those files can be imported from previous snapshots.
This commit is contained in:
parent
cebc9b62f2
commit
1aae44a2a3
9 changed files with 417 additions and 10 deletions
191
cmdline/state.c
191
cmdline/state.c
|
|
@ -2157,10 +2157,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
|
|||
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
|
||||
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
|
||||
* The previous 'P' entry is now deprecated, but supported for importing.
|
||||
* - SNAPCNT4/SnapRAID 15.0 Adds entry 'd' for dealloc file.
|
||||
*/
|
||||
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
|
||||
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
|
||||
&& memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
|
||||
&& memcmp(buffer, "SNAPCNT4\n\3\0\0", 12) != 0
|
||||
) {
|
||||
/* LCOV_EXCL_START */
|
||||
if (memcmp(buffer, "SNAPCNT", 7) != 0) {
|
||||
|
|
@ -2595,6 +2597,105 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
} else if (c == 'd') {
|
||||
/* dealloc */
|
||||
uint32_t v_count;
|
||||
uint32_t v_pos;
|
||||
struct snapraid_disk* disk;
|
||||
uint32_t mapping;
|
||||
|
||||
ret = sgetb32(f, &mapping);
|
||||
if (ret < 0 || mapping >= mapping_max) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
log_fatal(EINTERNAL, "Internal inconsistency: Dealloc mapping index out of range\n");
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
disk = tommy_array_get(&disk_mapping, mapping);
|
||||
|
||||
ret = sgetb32(f, &v_count);
|
||||
if (ret < 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
log_tag("content_info:dealloc:%s:%u\n", esc_tag(disk->name), v_count);
|
||||
|
||||
for (v_pos = 0; v_pos < v_count; ++v_pos) {
|
||||
char sub[PATH_MAX];
|
||||
uint64_t v_size;
|
||||
uint64_t v_mtime_sec;
|
||||
uint32_t v_mtime_nsec;
|
||||
|
||||
ret = sgetbs(f, sub, sizeof(sub));
|
||||
if (ret < 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (!*sub) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
log_fatal(EINTERNAL, "Internal inconsistency: Null dealloc!\n");
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
ret = sgetb64(f, &v_size);
|
||||
if (ret < 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
ret = sgetb64(f, &v_mtime_sec);
|
||||
if (ret < 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
ret = sgetb32(f, &v_mtime_nsec);
|
||||
if (ret < 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
/* STAT_NSEC_INVALID is encoded as 0 */
|
||||
if (v_mtime_nsec == 0)
|
||||
v_mtime_nsec = STAT_NSEC_INVALID;
|
||||
else
|
||||
--v_mtime_nsec;
|
||||
|
||||
/* allocate the file */
|
||||
struct snapraid_dealloc* dealloc = dealloc_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec);
|
||||
|
||||
log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", esc_tag(disk->name), esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec);
|
||||
|
||||
/* read all hashes */
|
||||
for (block_off_t k = 0; k < dealloc->blockmax; ++k) {
|
||||
unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE;
|
||||
|
||||
ret = sread(f, hash, BLOCK_HASH_SIZE);
|
||||
if (ret < 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
decoding_error(path, f);
|
||||
os_abort();
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
/* insert the dealloc in the dealloc containers */
|
||||
tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc);
|
||||
}
|
||||
} else if (c == 's') {
|
||||
/* symlink */
|
||||
char sub[PATH_MAX];
|
||||
|
|
@ -3358,9 +3459,22 @@ static void* state_write_thread(void* arg)
|
|||
count_unscrubbed = 0;
|
||||
tommy_hashdyn_init(&bucket_hash);
|
||||
|
||||
/* force version 3 a we want to always store the parity size */
|
||||
/*
|
||||
* Force at least version 3 a we want to always store the parity size
|
||||
*
|
||||
* If there is a dealloc list, force version 4
|
||||
*/
|
||||
for (i = state->disklist; i != 0; i = i->next) {
|
||||
struct snapraid_disk* disk = i->data;
|
||||
if (!tommy_list_empty(&disk->dealloclist))
|
||||
break;
|
||||
}
|
||||
|
||||
/* write header */
|
||||
swrite("SNAPCNT3\n\3\0\0", 12, f);
|
||||
if (i == 0)
|
||||
swrite("SNAPCNT3\n\3\0\0", 12, f);
|
||||
else
|
||||
swrite("SNAPCNT4\n\3\0\0", 12, f);
|
||||
|
||||
/* write block size and block max */
|
||||
sputc('z', f);
|
||||
|
|
@ -3706,6 +3820,49 @@ static void* state_write_thread(void* arg)
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
/* deallocated files of the disk */
|
||||
if (!tommy_list_empty(&disk->dealloclist)) {
|
||||
sputc('d', f);
|
||||
sputb32(disk->mapping_idx, f);
|
||||
|
||||
uint32_t v_count = tommy_list_count(&disk->dealloclist);
|
||||
|
||||
sputb32(v_count, f);
|
||||
log_tag("content_info:dealloc:%s:%u\n", esc_tag(disk->name), v_count);
|
||||
|
||||
/* for each file */
|
||||
for (j = tommy_list_head(&disk->dealloclist); j != 0; j = j->next) {
|
||||
struct snapraid_dealloc* dealloc = j->data;
|
||||
sputbs(dealloc->sub, f);
|
||||
sputb64(dealloc->size, f);
|
||||
sputb64(dealloc->mtime_sec, f);
|
||||
|
||||
/* encode STAT_NSEC_INVALID as 0 */
|
||||
if (dealloc->mtime_nsec == STAT_NSEC_INVALID)
|
||||
sputb32(0, f);
|
||||
else
|
||||
sputb32(dealloc->mtime_nsec + 1, f);
|
||||
|
||||
log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", esc_tag(disk->name), esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec);
|
||||
|
||||
/* write all hashes */
|
||||
for (block_off_t k = 0; k < dealloc->blockmax; ++k) {
|
||||
unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE;
|
||||
|
||||
swrite(hash, BLOCK_HASH_SIZE, f);
|
||||
}
|
||||
}
|
||||
|
||||
if (serror(f)) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
} else {
|
||||
log_tag("content_info:dealloc:%s:0\n", esc_tag(disk->name));
|
||||
}
|
||||
}
|
||||
|
||||
/* write the info for each block */
|
||||
|
|
@ -4657,6 +4814,20 @@ void state_write(struct snapraid_state* state)
|
|||
state->written = 1;
|
||||
}
|
||||
|
||||
void state_commit(struct snapraid_state* state)
|
||||
{
|
||||
tommy_node* i;
|
||||
|
||||
/* for each disk */
|
||||
for (i = state->disklist; i != 0; i = i->next) {
|
||||
struct snapraid_disk* disk = i->data;
|
||||
|
||||
/* clear the dealloc list */
|
||||
tommy_list_foreach(&disk->dealloclist, (tommy_foreach_func*)dealloc_free);
|
||||
tommy_list_init(&disk->dealloclist);
|
||||
}
|
||||
}
|
||||
|
||||
void state_skip(struct snapraid_state* state)
|
||||
{
|
||||
tommy_node* i;
|
||||
|
|
@ -5848,6 +6019,22 @@ void state_snapshot_write(struct snapraid_state* state, tommy_list* filterlist_d
|
|||
pathcpy(disk->dir, sizeof(disk->dir), vol);
|
||||
pathcat(disk->dir, sizeof(disk->dir), "/");
|
||||
pathcat(disk->dir, sizeof(disk->dir), disk->mount_point + root_len);
|
||||
|
||||
/* if there is a dealloc list */
|
||||
if (!tommy_list_empty(&disk->dealloclist)) {
|
||||
pathcpy(vol, sizeof(vol), container);
|
||||
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE);
|
||||
|
||||
/* if there is a previous snapshot */
|
||||
if (is_dir(vol)) {
|
||||
pathcat(vol, sizeof(vol), "/");
|
||||
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
|
||||
|
||||
msg_progress("Importing disk %s stable snapshot deallocated files...\n", disk->name);
|
||||
|
||||
state_dealloc(state, vol, &disk->dealloclist);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
pathcpy(vol, sizeof(vol), container);
|
||||
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue